From 6e64f530462fdf5892b8633e5329c5ec9239c74d Mon Sep 17 00:00:00 2001 From: Luke De Feo Date: Thu, 2 Nov 2023 12:29:07 -0700 Subject: [PATCH] Support pixel copy on views attached directly to window manager Summary: As mentioned in the previous diff pixel copy only support copying a Window, Surface or SurfaceView, All of these underneath use surface. For views attached to the window manager there is no activity / window so we need another solution There is no official way to get a views underlying surface so we had to do some dirty hacks to get it from the window manager. See the inline comments for details. Additionally it turns out that the pixel copy api was actually made async in Android 34, so to prepare for this the snapshot method was made a suspend function and we wrap the callback based apit with suspendCoroutine. Reviewed By: lblasa Differential Revision: D50845281 fbshipit-source-id: 5ba8ed6f330c1e04549812a6493ae5f4cb629d1f --- .../plugins/uidebugger/core/ApplicationRef.kt | 2 +- .../uidebugger/core/DecorViewTracker.kt | 5 +- .../uidebugger/core/RootViewResolver.kt | 20 ++-- .../plugins/uidebugger/core/Snapshot.kt | 77 ++++++++++----- .../plugins/uidebugger/core/UpdateQueue.kt | 2 +- .../uidebugger/core/WindowManagerUtility.kt | 93 +++++++++++++++++++ .../plugins/uidebugger/util/StopWatch.kt | 8 ++ .../plugins/uidebugger/util/Throttler.kt | 2 +- .../uidebugger/util/WindowManagerCommon.kt | 37 ++++++++ 9 files changed, 205 insertions(+), 41 deletions(-) create mode 100644 android/src/main/java/com/facebook/flipper/plugins/uidebugger/core/WindowManagerUtility.kt create mode 100644 android/src/main/java/com/facebook/flipper/plugins/uidebugger/util/WindowManagerCommon.kt diff --git a/android/src/main/java/com/facebook/flipper/plugins/uidebugger/core/ApplicationRef.kt b/android/src/main/java/com/facebook/flipper/plugins/uidebugger/core/ApplicationRef.kt index 82663d432..69f4e307d 100644 --- a/android/src/main/java/com/facebook/flipper/plugins/uidebugger/core/ApplicationRef.kt +++ b/android/src/main/java/com/facebook/flipper/plugins/uidebugger/core/ApplicationRef.kt @@ -21,7 +21,7 @@ class ApplicationRef(val application: Application) { // kinds of custom overlays // 2. Dialog fragments val rootsResolver: RootViewResolver = RootViewResolver() - + val windowManagerUtility = WindowManagerUtility() val activitiesStack: List get() { return ActivityTracker.activitiesStack diff --git a/android/src/main/java/com/facebook/flipper/plugins/uidebugger/core/DecorViewTracker.kt b/android/src/main/java/com/facebook/flipper/plugins/uidebugger/core/DecorViewTracker.kt index 96f3cf0f6..24720ff89 100644 --- a/android/src/main/java/com/facebook/flipper/plugins/uidebugger/core/DecorViewTracker.kt +++ b/android/src/main/java/com/facebook/flipper/plugins/uidebugger/core/DecorViewTracker.kt @@ -81,13 +81,14 @@ class DecorViewTracker(private val context: UIDContext, private val snapshotter: preDrawListener = null } - private fun traverseSnapshotAndSend(decorView: View) { + private suspend fun traverseSnapshotAndSend(decorView: View) { val startTimestamp = System.currentTimeMillis() + val (nodes, traversalTime) = StopWatch.time { context.layoutTraversal.traverse(context.applicationRef) } - val (reusableBitmap, snapshotMs) = StopWatch.time { snapshotter.takeSnapshot(decorView) } + val (reusableBitmap, snapshotMs) = StopWatch.timeSuspend { snapshotter.takeSnapshot(decorView) } context.updateQueue.enqueueUpdate( Update( diff --git a/android/src/main/java/com/facebook/flipper/plugins/uidebugger/core/RootViewResolver.kt b/android/src/main/java/com/facebook/flipper/plugins/uidebugger/core/RootViewResolver.kt index dc5ea1a2f..653b6ea2e 100644 --- a/android/src/main/java/com/facebook/flipper/plugins/uidebugger/core/RootViewResolver.kt +++ b/android/src/main/java/com/facebook/flipper/plugins/uidebugger/core/RootViewResolver.kt @@ -10,12 +10,14 @@ package com.facebook.flipper.plugins.uidebugger.core import android.annotation.SuppressLint import android.os.Build import android.view.View +import com.facebook.flipper.plugins.uidebugger.util.WindowManagerCommon import java.lang.reflect.Field import java.lang.reflect.InvocationTargetException import java.lang.reflect.Modifier /** - * Provides access to all root views in an application. + * Provides access to all root views in an application, as well as the ability to listen to changes + * in root views * * 95% of the time this is unnecessary and we can operate solely on current Activity's root view as * indicated by getWindow().getDecorView(). However in the case of popup windows, menus, and dialogs @@ -67,15 +69,13 @@ class RootViewResolver { private fun initialize() { initialized = true - val accessClass = - if (Build.VERSION.SDK_INT > 16) WINDOW_MANAGER_GLOBAL_CLAZZ else WINDOW_MANAGER_IMPL_CLAZZ - val instanceMethod = if (Build.VERSION.SDK_INT > 16) GET_GLOBAL_INSTANCE else GET_DEFAULT_IMPL try { - val clazz = Class.forName(accessClass) - val getMethod = clazz.getMethod(instanceMethod) - windowManagerObj = getMethod.invoke(null) - val viewsField: Field = clazz.getDeclaredField(VIEWS_FIELD) + val (windowManager, windowManagerClas) = + WindowManagerCommon.getGlobalWindowManager() ?: return + windowManagerObj = windowManager + + val viewsField: Field = windowManagerClas.getDeclaredField(VIEWS_FIELD) viewsField.let { vf -> vf.isAccessible = true @@ -98,11 +98,7 @@ class RootViewResolver { } companion object { - private const val WINDOW_MANAGER_IMPL_CLAZZ = "android.view.WindowManagerImpl" - private const val WINDOW_MANAGER_GLOBAL_CLAZZ = "android.view.WindowManagerGlobal" private const val VIEWS_FIELD = "mViews" - private const val GET_DEFAULT_IMPL = "getDefault" - private const val GET_GLOBAL_INSTANCE = "getInstance" } class ObservableViewArrayList : ArrayList() { diff --git a/android/src/main/java/com/facebook/flipper/plugins/uidebugger/core/Snapshot.kt b/android/src/main/java/com/facebook/flipper/plugins/uidebugger/core/Snapshot.kt index 1467d73c9..b8335866a 100644 --- a/android/src/main/java/com/facebook/flipper/plugins/uidebugger/core/Snapshot.kt +++ b/android/src/main/java/com/facebook/flipper/plugins/uidebugger/core/Snapshot.kt @@ -11,6 +11,7 @@ import android.app.Activity import android.graphics.Canvas import android.os.Build import android.os.Handler +import android.os.HandlerThread import android.os.Looper import android.util.Log import android.view.PixelCopy @@ -18,19 +19,22 @@ import android.view.View import androidx.annotation.RequiresApi import com.facebook.flipper.plugins.uidebugger.LogTag import com.facebook.flipper.plugins.uidebugger.common.BitmapPool +import kotlin.coroutines.Continuation +import kotlin.coroutines.resume +import kotlin.coroutines.suspendCoroutine interface Snapshotter { - fun takeSnapshot(view: View): BitmapPool.ReusableBitmap? + suspend fun takeSnapshot(view: View): BitmapPool.ReusableBitmap? } /** * Takes a snapshot by redrawing the view into a bitmap backed canvas, Since this is software * rendering there can be discrepancies between the real image and the snapshot: * 1. It can be unreliable when snapshotting views that are added directly to window manager - * 2. It doesnt include certain types of content (video / images) + * 2. It doesn't include certain types of content (video / images) */ class CanvasSnapshotter(private val bitmapPool: BitmapPool) : Snapshotter { - override fun takeSnapshot(view: View): BitmapPool.ReusableBitmap? { + override suspend fun takeSnapshot(view: View): BitmapPool.ReusableBitmap? { if (view.width <= 0 || view.height <= 0) { return null @@ -54,8 +58,9 @@ class PixelCopySnapshotter( private val applicationRef: ApplicationRef, private val fallback: Snapshotter ) : Snapshotter { + private var handler = Handler(Looper.getMainLooper()) - override fun takeSnapshot(view: View): BitmapPool.ReusableBitmap? { + override suspend fun takeSnapshot(view: View): BitmapPool.ReusableBitmap? { if (view.width <= 0 || view.height <= 0) { return null @@ -63,34 +68,58 @@ class PixelCopySnapshotter( val bitmap = bitmapPool.getBitmap(view.width, view.height) try { - - val decorViewToActivity: Map = - applicationRef.activitiesStack.toList().associateBy { it.window.decorView } - - val activity = decorViewToActivity[view] - - // if this view belongs to an activity prefer this as it doesn't require private api hacks - if (activity != null) { - PixelCopy.request( - activity.window, - bitmap.bitmap, - { - // no-op this this api is actually synchronous despite how it looks - }, - Handler(Looper.getMainLooper())) + if (tryCopyViaActivityWindow(view, bitmap)) { + // if this view belongs to an activity prefer this as it doesn't require private api hacks + return bitmap + } else if (tryCopyViaInternalSurface(view, bitmap)) { return bitmap } } catch (e: OutOfMemoryError) { Log.e(LogTag, "OOM when taking snapshot") - null } catch (e: Exception) { // there was some problem with the pixel copy, fall back to canvas impl Log.e(LogTag, "Exception when taking snapshot", e) - Log.i(LogTag, "Using fallback snapshot", e) - bitmap.readyForReuse() - fallback.takeSnapshot(view) } - return null + // something went wrong, use fallback + Log.i(LogTag, "Using fallback snapshot method") + bitmap.readyForReuse() + return fallback.takeSnapshot(view) + } + + private suspend fun tryCopyViaActivityWindow( + view: View, + bitmap: BitmapPool.ReusableBitmap + ): Boolean { + + val decorViewToActivity: Map = + applicationRef.activitiesStack.toList().associateBy { it.window.decorView } + + val activityForDecorView = decorViewToActivity[view] ?: return false + + return suspendCoroutine { continuation -> + PixelCopy.request( + activityForDecorView.window, bitmap.bitmap, pixelCopyCallback(continuation), handler) + } + } + + private suspend fun tryCopyViaInternalSurface( + view: View, + bitmap: BitmapPool.ReusableBitmap + ): Boolean { + val surface = applicationRef.windowManagerUtility.surfaceForRootView(view) ?: return false + + return suspendCoroutine { continuation -> + PixelCopy.request(surface, bitmap.bitmap, pixelCopyCallback(continuation), handler) + } + } + + private fun pixelCopyCallback(continuation: Continuation) = { result: Int -> + if (result == PixelCopy.SUCCESS) { + continuation.resume(true) + } else { + Log.w(LogTag, "Pixel copy failed, code $result") + continuation.resume(false) + } } } diff --git a/android/src/main/java/com/facebook/flipper/plugins/uidebugger/core/UpdateQueue.kt b/android/src/main/java/com/facebook/flipper/plugins/uidebugger/core/UpdateQueue.kt index d768ea4f3..218ff2cd2 100644 --- a/android/src/main/java/com/facebook/flipper/plugins/uidebugger/core/UpdateQueue.kt +++ b/android/src/main/java/com/facebook/flipper/plugins/uidebugger/core/UpdateQueue.kt @@ -113,7 +113,7 @@ class UpdateQueue(val context: UIDContext) { if (update.snapshotBitmap != null) { val stream = ByteArrayOutputStream() val base64Stream = Base64OutputStream(stream, Base64.DEFAULT) - update.snapshotBitmap.bitmap?.compress(Bitmap.CompressFormat.PNG, 100, base64Stream) + update.snapshotBitmap.bitmap.compress(Bitmap.CompressFormat.PNG, 100, base64Stream) snapshot = Snapshot(update.snapshotNode, stream.toString()) update.snapshotBitmap.readyForReuse() } diff --git a/android/src/main/java/com/facebook/flipper/plugins/uidebugger/core/WindowManagerUtility.kt b/android/src/main/java/com/facebook/flipper/plugins/uidebugger/core/WindowManagerUtility.kt new file mode 100644 index 000000000..4f9cab789 --- /dev/null +++ b/android/src/main/java/com/facebook/flipper/plugins/uidebugger/core/WindowManagerUtility.kt @@ -0,0 +1,93 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +package com.facebook.flipper.plugins.uidebugger.core + +import android.annotation.SuppressLint +import android.util.Log +import android.view.Surface +import android.view.View +import com.facebook.flipper.plugins.uidebugger.LogTag +import com.facebook.flipper.plugins.uidebugger.util.WindowManagerCommon +import java.lang.reflect.Field + +/** + * This class is related to root view resolver, it also accesses parts of the global window manager + */ +class WindowManagerUtility { + + private var initialized = false + + // type is RootViewImpl + private var rootsImpls: ArrayList<*>? = null + private var mSurfaceField: Field? = null + private var mViewField: Field? = null + + /** + * Find the surface for a given root view to allow snapshotting with pixelcopy. In the window + * manager there exists 2 arrays that contain similar data + * 1. mViews (this is what we track in the observable array in the root view resolver), these are + * the root decor views + * 2. mRoots - this is an internal class that holds a reference to the decor view as well as other + * internal bits (including the surface). + * + * Therefore we go through the roots and check for a view that matches the target, if it + * matches we return the surface. It is possible for us to observe these 2 arrays slightly out + * of sync with each other which is why we do this equality matching on the view field + * + * The reason we do this and not just grab the last view is because sometimes there is a + * 'empty' root view at the top we need to ignore. The decision to decide which view to + * snapshot is done else where as it needs to be synced with the observation and traversal + */ + fun surfaceForRootView(rootView: View): Surface? { + if (!initialized) { + initialize() + } + + val roots = rootsImpls ?: return null + for (i in roots.size - 1 downTo 0) { + val rootViewImpl = roots[i] + val view = mViewField?.get(rootViewImpl) + if (view == rootView) { + return mSurfaceField?.get(rootViewImpl) as? Surface + } + } + + return null + } + + @SuppressLint("PrivateApi") + private fun initialize() { + + try { + val (windowManager, windowManagerClass) = + WindowManagerCommon.getGlobalWindowManager() ?: return + + val rootsField: Field = windowManagerClass.getDeclaredField(ROOTS_FIELD) + rootsField.isAccessible = true + rootsImpls = rootsField.get(windowManager) as ArrayList<*>? + + val rootViewImplClass = Class.forName(VIEW_ROOT_IMPL_CLAZZ) + mSurfaceField = rootViewImplClass.getDeclaredField(SURFACE_FIELD) + mSurfaceField?.isAccessible = true + + mViewField = rootViewImplClass.getDeclaredField(VIEW_FIELD) + mViewField?.isAccessible = true + + initialized = true + } catch (exception: Exception) { + Log.e(LogTag, "Failed to initialize WindowManagerUtility", exception) + } + } + + companion object { + private const val VIEW_ROOT_IMPL_CLAZZ = "android.view.ViewRootImpl" + private const val SURFACE_FIELD = "mSurface" + private const val VIEW_FIELD = "mView" + private const val ROOTS_FIELD = "mRoots" + } +} diff --git a/android/src/main/java/com/facebook/flipper/plugins/uidebugger/util/StopWatch.kt b/android/src/main/java/com/facebook/flipper/plugins/uidebugger/util/StopWatch.kt index 12698ebac..16fa37c66 100644 --- a/android/src/main/java/com/facebook/flipper/plugins/uidebugger/util/StopWatch.kt +++ b/android/src/main/java/com/facebook/flipper/plugins/uidebugger/util/StopWatch.kt @@ -21,6 +21,14 @@ class StopWatch() { companion object { fun time(fn: () -> T): Pair { + + val start = System.currentTimeMillis() + val result = fn() + val elapsed = System.currentTimeMillis() - start + return Pair(result, elapsed) + } + + suspend fun timeSuspend(fn: suspend () -> T): Pair { val start = System.currentTimeMillis() val result = fn() val elapsed = System.currentTimeMillis() - start diff --git a/android/src/main/java/com/facebook/flipper/plugins/uidebugger/util/Throttler.kt b/android/src/main/java/com/facebook/flipper/plugins/uidebugger/util/Throttler.kt index 1069ddd80..7856e95ce 100644 --- a/android/src/main/java/com/facebook/flipper/plugins/uidebugger/util/Throttler.kt +++ b/android/src/main/java/com/facebook/flipper/plugins/uidebugger/util/Throttler.kt @@ -17,7 +17,7 @@ import kotlinx.coroutines.launch * This class will throttle calls into a callback. E.g if interval is 500ms and you receive triggers * at t=0, 100, 300 400, the callback will only be triggered at t=500 */ -class Throttler(private val intervalMs: Long, val callback: () -> T) { +class Throttler(private val intervalMs: Long, val callback: suspend () -> T) { private val executionScope: CoroutineScope = CoroutineScope(Dispatchers.Main) private var throttleJob: Job? = null diff --git a/android/src/main/java/com/facebook/flipper/plugins/uidebugger/util/WindowManagerCommon.kt b/android/src/main/java/com/facebook/flipper/plugins/uidebugger/util/WindowManagerCommon.kt new file mode 100644 index 000000000..6170be3e2 --- /dev/null +++ b/android/src/main/java/com/facebook/flipper/plugins/uidebugger/util/WindowManagerCommon.kt @@ -0,0 +1,37 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +package com.facebook.flipper.plugins.uidebugger.util + +import android.os.Build +import android.util.Log +import com.facebook.flipper.plugins.uidebugger.LogTag + +object WindowManagerCommon { + + // provides access to + // https://android.googlesource.com/platform/frameworks/base/+/refs/heads/main/core/java/android/view/WindowManagerGlobal.java + fun getGlobalWindowManager(): Pair>? { + val accessClass = + if (Build.VERSION.SDK_INT > 16) WINDOW_MANAGER_GLOBAL_CLAZZ else WINDOW_MANAGER_IMPL_CLAZZ + val instanceMethod = if (Build.VERSION.SDK_INT > 16) GET_GLOBAL_INSTANCE else GET_DEFAULT_IMPL + + try { + val clazz = Class.forName(accessClass) + val getMethod = clazz.getMethod(instanceMethod) + return Pair(getMethod.invoke(null), clazz) + } catch (exception: Exception) { + Log.e(LogTag, "Unable to get global window manager handle", exception) + return null + } + } + + private const val GET_DEFAULT_IMPL = "getDefault" + private const val GET_GLOBAL_INSTANCE = "getInstance" + private const val WINDOW_MANAGER_IMPL_CLAZZ = "android.view.WindowManagerImpl" + private const val WINDOW_MANAGER_GLOBAL_CLAZZ = "android.view.WindowManagerGlobal" +}