From 92f0ed67f4955cecf3a3d6d25f970119cd4ead0c Mon Sep 17 00:00:00 2001 From: Andrey Goncharov Date: Fri, 10 Dec 2021 06:34:37 -0800 Subject: [PATCH] Add download file API Summary: Changelog: Expose "downloadFile" API to Flipper plugins. Allow them to download files form the web to Flipper Server. Reviewed By: mweststrate Differential Revision: D32950685 fbshipit-source-id: 7b7f666e165ff7bf209230cdc96078272ede3616 --- desktop/flipper-common/src/index.tsx | 1 + desktop/flipper-common/src/server-types.tsx | 54 +++++++++ .../src/utils/assertNever.tsx | 0 .../flipper-plugin/src/__tests__/api.node.tsx | 1 + desktop/flipper-plugin/src/index.ts | 1 + .../flipper-plugin/src/plugin/FlipperLib.tsx | 17 +++ .../src/test-utils/test-utils.tsx | 1 + .../flipper-plugin/src/ui/FileSelector.tsx | 2 +- .../src/FlipperServerImpl.tsx | 6 + .../src/commands/DownloadFile.tsx | 106 ++++++++++++++++++ .../flipperLibImplementation/downloadFile.tsx | 72 ++++++++++++ .../index.tsx} | 20 ++-- 12 files changed, 271 insertions(+), 10 deletions(-) rename desktop/{flipper-plugin => flipper-common}/src/utils/assertNever.tsx (100%) create mode 100644 desktop/flipper-server-core/src/commands/DownloadFile.tsx create mode 100644 desktop/flipper-ui-core/src/utils/flipperLibImplementation/downloadFile.tsx rename desktop/flipper-ui-core/src/utils/{flipperLibImplementation.tsx => flipperLibImplementation/index.tsx} (83%) diff --git a/desktop/flipper-common/src/index.tsx b/desktop/flipper-common/src/index.tsx index 482e38399..019f30144 100644 --- a/desktop/flipper-common/src/index.tsx +++ b/desktop/flipper-common/src/index.tsx @@ -20,6 +20,7 @@ export * from './server-types'; export {sleep} from './utils/sleep'; export {timeout} from './utils/timeout'; export {isTest} from './utils/isTest'; +export {assertNever} from './utils/assertNever'; export { logPlatformSuccessRate, reportPlatformFailures, diff --git a/desktop/flipper-common/src/server-types.tsx b/desktop/flipper-common/src/server-types.tsx index 8c3e86bed..332cf6bfa 100644 --- a/desktop/flipper-common/src/server-types.tsx +++ b/desktop/flipper-common/src/server-types.tsx @@ -121,6 +121,7 @@ export type FlipperServerEvents = { id: string; message: string; }; + 'download-file-update': DownloadFileUpdate; }; export type IOSDeviceParams = { @@ -151,6 +152,11 @@ export type FlipperServerCommands = { command: string, options?: ExecOptions & {encoding?: BufferEncoding}, ) => Promise>; + 'download-file-start': ( + url: string, + dest: string, + options?: DownloadFileStartOptions, + ) => Promise; 'get-config': () => Promise; 'get-changelog': () => Promise; 'device-list': () => Promise; @@ -326,6 +332,54 @@ export interface MkdirOptions { mode?: string | number; } +export interface DownloadFileStartOptions { + method?: 'GET' | 'POST'; + timeout?: number; + maxRedirects?: number; + headers?: Record; + overwrite?: boolean; +} + +export type DownloadFileUpdate = { + id: string; + downloaded: number; + /** + * Set to 0 if unknown + */ + totalSize: number; +} & ( + | { + status: 'downloading'; + } + | { + status: 'success'; + } + | {status: 'error'; message: string; stack?: string} +); + +export interface DownloadFileStartResponse { + /** + * Download ID + */ + id: string; + /** + * Response status + */ + status: number; + /** + * Response status text + */ + statusText: string; + /** + * Response headers + */ + headers: Record; + /** + * Size of the file, being downloaded, in bytes. Inferred from the "Content-Length" header. Set to 0 if unknown. + */ + totalSize: number; +} + export type FlipperServerConfig = { gatekeepers: Record; env: Partial>; diff --git a/desktop/flipper-plugin/src/utils/assertNever.tsx b/desktop/flipper-common/src/utils/assertNever.tsx similarity index 100% rename from desktop/flipper-plugin/src/utils/assertNever.tsx rename to desktop/flipper-common/src/utils/assertNever.tsx diff --git a/desktop/flipper-plugin/src/__tests__/api.node.tsx b/desktop/flipper-plugin/src/__tests__/api.node.tsx index f10f2ba0a..b2119cd3e 100644 --- a/desktop/flipper-plugin/src/__tests__/api.node.tsx +++ b/desktop/flipper-plugin/src/__tests__/api.node.tsx @@ -91,6 +91,7 @@ test('Correct top level API exposed', () => { "DevicePluginClient", "DeviceType", "DialogResult", + "DownloadFileResponse", "Draft", "ElementAttribute", "ElementData", diff --git a/desktop/flipper-plugin/src/index.ts b/desktop/flipper-plugin/src/index.ts index 956bf0321..901e7f872 100644 --- a/desktop/flipper-plugin/src/index.ts +++ b/desktop/flipper-plugin/src/index.ts @@ -40,6 +40,7 @@ export { FileDescriptor, FileEncoding, RemoteServerContext, + DownloadFileResponse, } from './plugin/FlipperLib'; export { MenuEntry, diff --git a/desktop/flipper-plugin/src/plugin/FlipperLib.tsx b/desktop/flipper-plugin/src/plugin/FlipperLib.tsx index f73bd914a..ee6d619af 100644 --- a/desktop/flipper-plugin/src/plugin/FlipperLib.tsx +++ b/desktop/flipper-plugin/src/plugin/FlipperLib.tsx @@ -18,6 +18,9 @@ import { ExecOut, BufferEncoding, MkdirOptions, + DownloadFileStartOptions, + DownloadFileStartResponse, + DownloadFileUpdate, } from 'flipper-common'; export type FileEncoding = 'utf-8' | 'base64'; @@ -28,6 +31,13 @@ export interface FileDescriptor { path?: string; } +export interface DownloadFileResponse extends DownloadFileStartResponse { + /** + * Indicates whether a download is completed. Resolves with the number of downloaded bytes. Rejects if the download has errors. + */ + completed: Promise; +} + export type RemoteServerContext = { childProcess: { exec( @@ -49,6 +59,13 @@ export type RemoteServerContext = { ): Promise; copyFile(src: string, dest: string, flags?: number): Promise; }; + downloadFile( + url: string, + dest: string, + options?: DownloadFileStartOptions & { + onProgressUpdate?: (progressUpdate: DownloadFileUpdate) => void; + }, + ): Promise; }; /** diff --git a/desktop/flipper-plugin/src/test-utils/test-utils.tsx b/desktop/flipper-plugin/src/test-utils/test-utils.tsx index f7a1c3b81..5427f6586 100644 --- a/desktop/flipper-plugin/src/test-utils/test-utils.tsx +++ b/desktop/flipper-plugin/src/test-utils/test-utils.tsx @@ -400,6 +400,7 @@ export function createMockFlipperLib(options?: StartPluginOptions): FlipperLib { mkdir: jest.fn(), copyFile: jest.fn(), }, + downloadFile: jest.fn(), }, }; } diff --git a/desktop/flipper-plugin/src/ui/FileSelector.tsx b/desktop/flipper-plugin/src/ui/FileSelector.tsx index e1f8ba084..65a9ada5d 100644 --- a/desktop/flipper-plugin/src/ui/FileSelector.tsx +++ b/desktop/flipper-plugin/src/ui/FileSelector.tsx @@ -26,7 +26,7 @@ import { getFlipperLib, } from '../plugin/FlipperLib'; import {fromUint8Array} from 'js-base64'; -import {assertNever} from '../utils/assertNever'; +import {assertNever} from 'flipper-common'; export type FileSelectorProps = { /** diff --git a/desktop/flipper-server-core/src/FlipperServerImpl.tsx b/desktop/flipper-server-core/src/FlipperServerImpl.tsx index 316a0f0b1..972f4604a 100644 --- a/desktop/flipper-server-core/src/FlipperServerImpl.tsx +++ b/desktop/flipper-server-core/src/FlipperServerImpl.tsx @@ -45,6 +45,7 @@ import { } from './fb-stubs/internRequests'; import {commandNodeApiExec} from './commands/NodeApiExec'; import {access, copyFile, mkdir, unlink} from 'fs/promises'; +import {commandDownloadFileStartFactory} from './commands/DownloadFile'; export const SERVICE_FLIPPER = 'flipper.oAuthToken'; @@ -228,6 +229,11 @@ export class FlipperServerImpl implements FlipperServer { 'node-api-fs-unlink': unlink, 'node-api-fs-mkdir': mkdir, 'node-api-fs-copyFile': copyFile, + // TODO: Unit tests + // TODO: Do we need API to cancel an active download? + 'download-file-start': commandDownloadFileStartFactory( + this.emit.bind(this), + ), 'get-config': async () => this.config, 'get-changelog': getChangelog, 'device-list': async () => { diff --git a/desktop/flipper-server-core/src/commands/DownloadFile.tsx b/desktop/flipper-server-core/src/commands/DownloadFile.tsx new file mode 100644 index 000000000..fde80811d --- /dev/null +++ b/desktop/flipper-server-core/src/commands/DownloadFile.tsx @@ -0,0 +1,106 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +import {FlipperServerCommands, FlipperServerEvents} from 'flipper-common'; +import {pathExists} from 'fs-extra'; +import {promises, createWriteStream, ReadStream} from 'fs'; +import axios from 'axios'; +import {v4 as uuid} from 'uuid'; + +const {unlink} = promises; + +// Adapter which forces node.js implementation for axios instead of browser implementation +// used by default in Electron. Node.js implementation is better, because it +// supports streams which can be used for direct downloading to disk. +const axiosHttpAdapter = require('axios/lib/adapters/http'); // eslint-disable-line import/no-commonjs + +export const commandDownloadFileStartFactory = + ( + emit: ( + event: 'download-file-update', + payload: FlipperServerEvents['download-file-update'], + ) => void, + ): FlipperServerCommands['download-file-start'] => + async ( + url, + dest, + {method = 'GET', timeout, maxRedirects, headers, overwrite} = {}, + ) => { + const destExists = await pathExists(dest); + + if (destExists) { + if (!overwrite) { + throw new Error( + 'FlipperServerImpl -> executing "download-file" -> path already exists and overwrite set to false', + ); + } + + await unlink(dest); + } + + const downloadId = uuid(); + + const response = await axios.request({ + method, + url, + responseType: 'stream', + adapter: axiosHttpAdapter, + timeout, + maxRedirects, + headers, + }); + const totalSize = response.headers['content-length'] ?? 0; + + const writeStream = response.data.pipe( + createWriteStream(dest, {autoClose: true}), + ); + let downloaded = 0; + response.data.on('data', (data: any) => { + downloaded += Buffer.byteLength(data); + emit('download-file-update', { + id: downloadId, + downloaded, + totalSize, + status: 'downloading', + }); + }); + + response.data.on('error', (e: Error) => { + writeStream.destroy(e); + }); + + writeStream.on('finish', () => { + emit('download-file-update', { + id: downloadId, + downloaded, + totalSize, + status: 'success', + }); + }); + + writeStream.on('error', (e: Error) => { + response.data.destroy(); + emit('download-file-update', { + id: downloadId, + downloaded, + totalSize, + status: 'error', + message: e.message, + stack: e.stack, + }); + }); + + return { + id: downloadId, + headers: response.headers, + status: response.status, + statusText: response.statusText, + totalSize, + }; + }; diff --git a/desktop/flipper-ui-core/src/utils/flipperLibImplementation/downloadFile.tsx b/desktop/flipper-ui-core/src/utils/flipperLibImplementation/downloadFile.tsx new file mode 100644 index 000000000..75ea42d5a --- /dev/null +++ b/desktop/flipper-ui-core/src/utils/flipperLibImplementation/downloadFile.tsx @@ -0,0 +1,72 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +import {assertNever, DownloadFileUpdate} from 'flipper-common'; +import {FlipperLib, DownloadFileResponse} from 'flipper-plugin'; +import {RenderHost} from '../../RenderHost'; + +export const downloadFileFactory = + (renderHost: RenderHost): FlipperLib['remoteServerContext']['downloadFile'] => + async (url, dest, {onProgressUpdate, ...options} = {}) => { + const downloadDescriptor = (await renderHost.flipperServer.exec( + 'download-file-start', + url, + dest, + options, + // Casting to DownloadFileResponse to add `completed` field to `downloadDescriptor`. + )) as DownloadFileResponse; + + let onProgressUpdateWrapped: (progressUpdate: DownloadFileUpdate) => void; + const completed = new Promise((resolve, reject) => { + onProgressUpdateWrapped = (progressUpdate: DownloadFileUpdate) => { + if (progressUpdate.id === downloadDescriptor.id) { + const {status} = progressUpdate; + switch (status) { + case 'downloading': { + onProgressUpdate?.(progressUpdate); + break; + } + case 'success': { + resolve(progressUpdate.downloaded); + break; + } + case 'error': { + reject( + new Error( + `File download failed. Last message: ${JSON.stringify( + progressUpdate, + )}`, + ), + ); + break; + } + default: { + assertNever(status); + } + } + } + }; + renderHost.flipperServer.on( + 'download-file-update', + onProgressUpdateWrapped, + ); + }); + + // eslint-disable-next-line promise/catch-or-return + completed.finally(() => { + renderHost.flipperServer.off( + 'download-file-update', + onProgressUpdateWrapped, + ); + }); + + downloadDescriptor.completed = completed; + + return downloadDescriptor; + }; diff --git a/desktop/flipper-ui-core/src/utils/flipperLibImplementation.tsx b/desktop/flipper-ui-core/src/utils/flipperLibImplementation/index.tsx similarity index 83% rename from desktop/flipper-ui-core/src/utils/flipperLibImplementation.tsx rename to desktop/flipper-ui-core/src/utils/flipperLibImplementation/index.tsx index d1b87e9f3..0e5596a7f 100644 --- a/desktop/flipper-ui-core/src/utils/flipperLibImplementation.tsx +++ b/desktop/flipper-ui-core/src/utils/flipperLibImplementation/index.tsx @@ -11,21 +11,22 @@ import { _setFlipperLibImplementation, RemoteServerContext, } from 'flipper-plugin'; -import type { +import { BufferEncoding, ExecOptions, Logger, MkdirOptions, } from 'flipper-common'; -import type {Store} from '../reducers'; -import createPaste from '../fb-stubs/createPaste'; -import type BaseDevice from '../devices/BaseDevice'; -import constants from '../fb-stubs/constants'; -import {addNotification} from '../reducers/notifications'; +import type {Store} from '../../reducers'; +import createPaste from '../../fb-stubs/createPaste'; +import type BaseDevice from '../../devices/BaseDevice'; +import constants from '../../fb-stubs/constants'; +import {addNotification} from '../../reducers/notifications'; import {deconstructPluginKey} from 'flipper-common'; -import {DetailSidebarImpl} from '../sandy-chrome/DetailSidebarImpl'; -import {RenderHost} from '../RenderHost'; -import {setMenuEntries} from '../reducers/connections'; +import {DetailSidebarImpl} from '../../sandy-chrome/DetailSidebarImpl'; +import {RenderHost} from '../../RenderHost'; +import {setMenuEntries} from '../../reducers/connections'; +import {downloadFileFactory} from './downloadFile'; export function initializeFlipperLibImplementation( renderHost: RenderHost, @@ -102,6 +103,7 @@ export function initializeFlipperLibImplementation( flags, ), }, + downloadFile: downloadFileFactory(renderHost), }, }); }