Add download file API

Summary: Changelog: Expose "downloadFile" API to Flipper plugins. Allow them to download files form the web to Flipper Server.

Reviewed By: mweststrate

Differential Revision: D32950685

fbshipit-source-id: 7b7f666e165ff7bf209230cdc96078272ede3616

desktop/flipper-common/src/index.tsx

@@ -20,6 +20,7 @@ export * from './server-types';
 export {sleep} from './utils/sleep';
 export {timeout} from './utils/timeout';
 export {isTest} from './utils/isTest';
+export {assertNever} from './utils/assertNever';
 export {
   logPlatformSuccessRate,
   reportPlatformFailures,

desktop/flipper-common/src/server-types.tsx

@@ -121,6 +121,7 @@ export type FlipperServerEvents = {
     id: string;
     message: string;
   };
+  'download-file-update': DownloadFileUpdate;
 };
 
 export type IOSDeviceParams = {
@@ -151,6 +152,11 @@ export type FlipperServerCommands = {
     command: string,
     options?: ExecOptions & {encoding?: BufferEncoding},
   ) => Promise<ExecOut<string>>;
+  'download-file-start': (
+    url: string,
+    dest: string,
+    options?: DownloadFileStartOptions,
+  ) => Promise<DownloadFileStartResponse>;
   'get-config': () => Promise<FlipperServerConfig>;
   'get-changelog': () => Promise<string>;
   'device-list': () => Promise<DeviceDescription[]>;
@@ -326,6 +332,54 @@ export interface MkdirOptions {
   mode?: string | number;
 }
 
+export interface DownloadFileStartOptions {
+  method?: 'GET' | 'POST';
+  timeout?: number;
+  maxRedirects?: number;
+  headers?: Record<string, string>;
+  overwrite?: boolean;
+}
+
+export type DownloadFileUpdate = {
+  id: string;
+  downloaded: number;
+  /**
+   * Set to 0 if unknown
+   */
+  totalSize: number;
+} & (
+  | {
+      status: 'downloading';
+    }
+  | {
+      status: 'success';
+    }
+  | {status: 'error'; message: string; stack?: string}
+);
+
+export interface DownloadFileStartResponse {
+  /**
+   * Download ID
+   */
+  id: string;
+  /**
+   * Response status
+   */
+  status: number;
+  /**
+   * Response status text
+   */
+  statusText: string;
+  /**
+   * Response headers
+   */
+  headers: Record<string, string>;
+  /**
+   * Size of the file, being downloaded, in bytes. Inferred from the "Content-Length" header. Set to 0 if unknown.
+   */
+  totalSize: number;
+}
+
 export type FlipperServerConfig = {
   gatekeepers: Record<string, boolean>;
   env: Partial<Record<ENVIRONMENT_VARIABLES, string>>;

desktop/flipper-plugin/src/__tests__/api.node.tsx

@@ -91,6 +91,7 @@ test('Correct top level API exposed', () => {
       "DevicePluginClient",
       "DeviceType",
       "DialogResult",
+      "DownloadFileResponse",
       "Draft",
       "ElementAttribute",
       "ElementData",

desktop/flipper-plugin/src/index.ts

@@ -40,6 +40,7 @@ export {
   FileDescriptor,
   FileEncoding,
   RemoteServerContext,
+  DownloadFileResponse,
 } from './plugin/FlipperLib';
 export {
   MenuEntry,

desktop/flipper-plugin/src/plugin/FlipperLib.tsx

@@ -18,6 +18,9 @@ import {
   ExecOut,
   BufferEncoding,
   MkdirOptions,
+  DownloadFileStartOptions,
+  DownloadFileStartResponse,
+  DownloadFileUpdate,
 } from 'flipper-common';
 
 export type FileEncoding = 'utf-8' | 'base64';
@@ -28,6 +31,13 @@ export interface FileDescriptor {
   path?: string;
 }
 
+export interface DownloadFileResponse extends DownloadFileStartResponse {
+  /**
+   * Indicates whether a download is completed. Resolves with the number of downloaded bytes. Rejects if the download has errors.
+   */
+  completed: Promise<number>;
+}
+
 export type RemoteServerContext = {
   childProcess: {
     exec(
@@ -49,6 +59,13 @@ export type RemoteServerContext = {
     ): Promise<void>;
     copyFile(src: string, dest: string, flags?: number): Promise<void>;
   };
+  downloadFile(
+    url: string,
+    dest: string,
+    options?: DownloadFileStartOptions & {
+      onProgressUpdate?: (progressUpdate: DownloadFileUpdate) => void;
+    },
+  ): Promise<DownloadFileResponse>;
 };
 
 /**

desktop/flipper-plugin/src/test-utils/test-utils.tsx

@@ -400,6 +400,7 @@ export function createMockFlipperLib(options?: StartPluginOptions): FlipperLib {
         mkdir: jest.fn(),
         copyFile: jest.fn(),
       },
+      downloadFile: jest.fn(),
     },
   };
 }

desktop/flipper-plugin/src/ui/FileSelector.tsx

@@ -26,7 +26,7 @@ import {
   getFlipperLib,
 } from '../plugin/FlipperLib';
 import {fromUint8Array} from 'js-base64';
-import {assertNever} from '../utils/assertNever';
+import {assertNever} from 'flipper-common';
 
 export type FileSelectorProps = {
   /**

desktop/flipper-server-core/src/FlipperServerImpl.tsx

@@ -45,6 +45,7 @@ import {
 } from './fb-stubs/internRequests';
 import {commandNodeApiExec} from './commands/NodeApiExec';
 import {access, copyFile, mkdir, unlink} from 'fs/promises';
+import {commandDownloadFileStartFactory} from './commands/DownloadFile';
 
 export const SERVICE_FLIPPER = 'flipper.oAuthToken';
 
@@ -228,6 +229,11 @@ export class FlipperServerImpl implements FlipperServer {
     'node-api-fs-unlink': unlink,
     'node-api-fs-mkdir': mkdir,
     'node-api-fs-copyFile': copyFile,
+    // TODO: Unit tests
+    // TODO: Do we need API to cancel an active download?
+    'download-file-start': commandDownloadFileStartFactory(
+      this.emit.bind(this),
+    ),
     'get-config': async () => this.config,
     'get-changelog': getChangelog,
     'device-list': async () => {

desktop/flipper-server-core/src/commands/DownloadFile.tsx

@@ -0,0 +1,106 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+import {FlipperServerCommands, FlipperServerEvents} from 'flipper-common';
+import {pathExists} from 'fs-extra';
+import {promises, createWriteStream, ReadStream} from 'fs';
+import axios from 'axios';
+import {v4 as uuid} from 'uuid';
+
+const {unlink} = promises;
+
+// Adapter which forces node.js implementation for axios instead of browser implementation
+// used by default in Electron. Node.js implementation is better, because it
+// supports streams which can be used for direct downloading to disk.
+const axiosHttpAdapter = require('axios/lib/adapters/http'); // eslint-disable-line import/no-commonjs
+
+export const commandDownloadFileStartFactory =
+  (
+    emit: (
+      event: 'download-file-update',
+      payload: FlipperServerEvents['download-file-update'],
+    ) => void,
+  ): FlipperServerCommands['download-file-start'] =>
+  async (
+    url,
+    dest,
+    {method = 'GET', timeout, maxRedirects, headers, overwrite} = {},
+  ) => {
+    const destExists = await pathExists(dest);
+
+    if (destExists) {
+      if (!overwrite) {
+        throw new Error(
+          'FlipperServerImpl -> executing "download-file" -> path already exists and overwrite set to false',
+        );
+      }
+
+      await unlink(dest);
+    }
+
+    const downloadId = uuid();
+
+    const response = await axios.request<ReadStream>({
+      method,
+      url,
+      responseType: 'stream',
+      adapter: axiosHttpAdapter,
+      timeout,
+      maxRedirects,
+      headers,
+    });
+    const totalSize = response.headers['content-length'] ?? 0;
+
+    const writeStream = response.data.pipe(
+      createWriteStream(dest, {autoClose: true}),
+    );
+    let downloaded = 0;
+    response.data.on('data', (data: any) => {
+      downloaded += Buffer.byteLength(data);
+      emit('download-file-update', {
+        id: downloadId,
+        downloaded,
+        totalSize,
+        status: 'downloading',
+      });
+    });
+
+    response.data.on('error', (e: Error) => {
+      writeStream.destroy(e);
+    });
+
+    writeStream.on('finish', () => {
+      emit('download-file-update', {
+        id: downloadId,
+        downloaded,
+        totalSize,
+        status: 'success',
+      });
+    });
+
+    writeStream.on('error', (e: Error) => {
+      response.data.destroy();
+      emit('download-file-update', {
+        id: downloadId,
+        downloaded,
+        totalSize,
+        status: 'error',
+        message: e.message,
+        stack: e.stack,
+      });
+    });
+
+    return {
+      id: downloadId,
+      headers: response.headers,
+      status: response.status,
+      statusText: response.statusText,
+      totalSize,
+    };
+  };

desktop/flipper-ui-core/src/utils/flipperLibImplementation/downloadFile.tsx

@@ -0,0 +1,72 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+import {assertNever, DownloadFileUpdate} from 'flipper-common';
+import {FlipperLib, DownloadFileResponse} from 'flipper-plugin';
+import {RenderHost} from '../../RenderHost';
+
+export const downloadFileFactory =
+  (renderHost: RenderHost): FlipperLib['remoteServerContext']['downloadFile'] =>
+  async (url, dest, {onProgressUpdate, ...options} = {}) => {
+    const downloadDescriptor = (await renderHost.flipperServer.exec(
+      'download-file-start',
+      url,
+      dest,
+      options,
+      // Casting to DownloadFileResponse to add `completed` field to `downloadDescriptor`.
+    )) as DownloadFileResponse;
+
+    let onProgressUpdateWrapped: (progressUpdate: DownloadFileUpdate) => void;
+    const completed = new Promise<number>((resolve, reject) => {
+      onProgressUpdateWrapped = (progressUpdate: DownloadFileUpdate) => {
+        if (progressUpdate.id === downloadDescriptor.id) {
+          const {status} = progressUpdate;
+          switch (status) {
+            case 'downloading': {
+              onProgressUpdate?.(progressUpdate);
+              break;
+            }
+            case 'success': {
+              resolve(progressUpdate.downloaded);
+              break;
+            }
+            case 'error': {
+              reject(
+                new Error(
+                  `File download failed. Last message: ${JSON.stringify(
+                    progressUpdate,
+                  )}`,
+                ),
+              );
+              break;
+            }
+            default: {
+              assertNever(status);
+            }
+          }
+        }
+      };
+      renderHost.flipperServer.on(
+        'download-file-update',
+        onProgressUpdateWrapped,
+      );
+    });
+
+    // eslint-disable-next-line promise/catch-or-return
+    completed.finally(() => {
+      renderHost.flipperServer.off(
+        'download-file-update',
+        onProgressUpdateWrapped,
+      );
+    });
+
+    downloadDescriptor.completed = completed;
+
+    return downloadDescriptor;
+  };

desktop/flipper-ui-core/src/utils/flipperLibImplementation/index.tsx

@@ -11,21 +11,22 @@ import {
   _setFlipperLibImplementation,
   RemoteServerContext,
 } from 'flipper-plugin';
-import type {
+import {
   BufferEncoding,
   ExecOptions,
   Logger,
   MkdirOptions,
 } from 'flipper-common';
-import type {Store} from '../reducers';
+import type {Store} from '../../reducers';
-import createPaste from '../fb-stubs/createPaste';
+import createPaste from '../../fb-stubs/createPaste';
-import type BaseDevice from '../devices/BaseDevice';
+import type BaseDevice from '../../devices/BaseDevice';
-import constants from '../fb-stubs/constants';
+import constants from '../../fb-stubs/constants';
-import {addNotification} from '../reducers/notifications';
+import {addNotification} from '../../reducers/notifications';
 import {deconstructPluginKey} from 'flipper-common';
-import {DetailSidebarImpl} from '../sandy-chrome/DetailSidebarImpl';
+import {DetailSidebarImpl} from '../../sandy-chrome/DetailSidebarImpl';
-import {RenderHost} from '../RenderHost';
+import {RenderHost} from '../../RenderHost';
-import {setMenuEntries} from '../reducers/connections';
+import {setMenuEntries} from '../../reducers/connections';
+import {downloadFileFactory} from './downloadFile';
 
 export function initializeFlipperLibImplementation(
   renderHost: RenderHost,
@@ -102,6 +103,7 @@ export function initializeFlipperLibImplementation(
             flags,
           ),
       },
+      downloadFile: downloadFileFactory(renderHost),
     },
   });
 }