Support pixel copy on views attached directly to window manager

Summary:
As mentioned in the previous diff pixel copy only support copying a Window, Surface or SurfaceView, All of these underneath use surface.

For views attached to the window manager there is no activity / window so we need another solution

There is no official way to get a views underlying surface so we had to do some dirty hacks to get it from the window manager. See the inline comments for details.

Additionally it turns out that the pixel copy api was actually made async in Android 34, so to prepare for this the snapshot method was made a suspend function and we wrap the callback based apit with suspendCoroutine.

Reviewed By: lblasa

Differential Revision: D50845281

fbshipit-source-id: 5ba8ed6f330c1e04549812a6493ae5f4cb629d1f

android/src/main/java/com/facebook/flipper/plugins/uidebugger/core/ApplicationRef.kt

@@ -21,7 +21,7 @@ class ApplicationRef(val application: Application) {
   // kinds of custom overlays
   // 2. Dialog fragments
   val rootsResolver: RootViewResolver = RootViewResolver()
-
+  val windowManagerUtility = WindowManagerUtility()
   val activitiesStack: List<Activity>
     get() {
       return ActivityTracker.activitiesStack

android/src/main/java/com/facebook/flipper/plugins/uidebugger/core/DecorViewTracker.kt

@@ -81,13 +81,14 @@ class DecorViewTracker(private val context: UIDContext, private val snapshotter:
     preDrawListener = null
   }
 
-  private fun traverseSnapshotAndSend(decorView: View) {
+  private suspend fun traverseSnapshotAndSend(decorView: View) {
 
     val startTimestamp = System.currentTimeMillis()
+
     val (nodes, traversalTime) =
         StopWatch.time { context.layoutTraversal.traverse(context.applicationRef) }
 
-    val (reusableBitmap, snapshotMs) = StopWatch.time { snapshotter.takeSnapshot(decorView) }
+    val (reusableBitmap, snapshotMs) = StopWatch.timeSuspend { snapshotter.takeSnapshot(decorView) }
 
     context.updateQueue.enqueueUpdate(
         Update(

android/src/main/java/com/facebook/flipper/plugins/uidebugger/core/RootViewResolver.kt

@@ -10,12 +10,14 @@ package com.facebook.flipper.plugins.uidebugger.core
 import android.annotation.SuppressLint
 import android.os.Build
 import android.view.View
+import com.facebook.flipper.plugins.uidebugger.util.WindowManagerCommon
 import java.lang.reflect.Field
 import java.lang.reflect.InvocationTargetException
 import java.lang.reflect.Modifier
 
 /**
- * Provides access to all root views in an application.
+ * Provides access to all root views in an application, as well as the ability to listen to changes
+ * in root views
  *
  * 95% of the time this is unnecessary and we can operate solely on current Activity's root view as
  * indicated by getWindow().getDecorView(). However in the case of popup windows, menus, and dialogs
@@ -67,15 +69,13 @@ class RootViewResolver {
   private fun initialize() {
 
     initialized = true
-    val accessClass =
-        if (Build.VERSION.SDK_INT > 16) WINDOW_MANAGER_GLOBAL_CLAZZ else WINDOW_MANAGER_IMPL_CLAZZ
-    val instanceMethod = if (Build.VERSION.SDK_INT > 16) GET_GLOBAL_INSTANCE else GET_DEFAULT_IMPL
     try {
-      val clazz = Class.forName(accessClass)
-      val getMethod = clazz.getMethod(instanceMethod)
-      windowManagerObj = getMethod.invoke(null)
 
-      val viewsField: Field = clazz.getDeclaredField(VIEWS_FIELD)
+      val (windowManager, windowManagerClas) =
+          WindowManagerCommon.getGlobalWindowManager() ?: return
+      windowManagerObj = windowManager
+
+      val viewsField: Field = windowManagerClas.getDeclaredField(VIEWS_FIELD)
 
       viewsField.let { vf ->
         vf.isAccessible = true
@@ -98,11 +98,7 @@ class RootViewResolver {
   }
 
   companion object {
-    private const val WINDOW_MANAGER_IMPL_CLAZZ = "android.view.WindowManagerImpl"
-    private const val WINDOW_MANAGER_GLOBAL_CLAZZ = "android.view.WindowManagerGlobal"
     private const val VIEWS_FIELD = "mViews"
-    private const val GET_DEFAULT_IMPL = "getDefault"
-    private const val GET_GLOBAL_INSTANCE = "getInstance"
   }
 
   class ObservableViewArrayList : ArrayList<View>() {

android/src/main/java/com/facebook/flipper/plugins/uidebugger/core/Snapshot.kt

@@ -11,6 +11,7 @@ import android.app.Activity
 import android.graphics.Canvas
 import android.os.Build
 import android.os.Handler
+import android.os.HandlerThread
 import android.os.Looper
 import android.util.Log
 import android.view.PixelCopy
@@ -18,19 +19,22 @@ import android.view.View
 import androidx.annotation.RequiresApi
 import com.facebook.flipper.plugins.uidebugger.LogTag
 import com.facebook.flipper.plugins.uidebugger.common.BitmapPool
+import kotlin.coroutines.Continuation
+import kotlin.coroutines.resume
+import kotlin.coroutines.suspendCoroutine
 
 interface Snapshotter {
-  fun takeSnapshot(view: View): BitmapPool.ReusableBitmap?
+  suspend fun takeSnapshot(view: View): BitmapPool.ReusableBitmap?
 }
 
 /**
  * Takes a snapshot by redrawing the view into a bitmap backed canvas, Since this is software
  * rendering there can be discrepancies between the real image and the snapshot:
  * 1. It can be unreliable when snapshotting views that are added directly to window manager
- * 2. It doesnt include certain types of content (video / images)
+ * 2. It doesn't include certain types of content (video / images)
  */
 class CanvasSnapshotter(private val bitmapPool: BitmapPool) : Snapshotter {
-  override fun takeSnapshot(view: View): BitmapPool.ReusableBitmap? {
+  override suspend fun takeSnapshot(view: View): BitmapPool.ReusableBitmap? {
 
     if (view.width <= 0 || view.height <= 0) {
       return null
@@ -54,8 +58,9 @@ class PixelCopySnapshotter(
     private val applicationRef: ApplicationRef,
     private val fallback: Snapshotter
 ) : Snapshotter {
+  private var handler = Handler(Looper.getMainLooper())
 
-  override fun takeSnapshot(view: View): BitmapPool.ReusableBitmap? {
+  override suspend fun takeSnapshot(view: View): BitmapPool.ReusableBitmap? {
 
     if (view.width <= 0 || view.height <= 0) {
       return null
@@ -63,34 +68,58 @@ class PixelCopySnapshotter(
 
     val bitmap = bitmapPool.getBitmap(view.width, view.height)
     try {
-
+      if (tryCopyViaActivityWindow(view, bitmap)) {
-      val decorViewToActivity: Map<View, Activity> =
+        // if this view belongs to an activity prefer this as it doesn't require private api hacks
-          applicationRef.activitiesStack.toList().associateBy { it.window.decorView }
+        return bitmap
-
+      } else if (tryCopyViaInternalSurface(view, bitmap)) {
-      val activity = decorViewToActivity[view]
-
-      // if this view belongs to an activity prefer this as it doesn't require private api hacks
-      if (activity != null) {
-        PixelCopy.request(
-            activity.window,
-            bitmap.bitmap,
-            {
-              // no-op this this api is actually synchronous despite how it looks
-            },
-            Handler(Looper.getMainLooper()))
         return bitmap
       }
     } catch (e: OutOfMemoryError) {
       Log.e(LogTag, "OOM when taking snapshot")
-      null
     } catch (e: Exception) {
       // there was some problem with the pixel copy, fall back to canvas impl
       Log.e(LogTag, "Exception when taking snapshot", e)
-      Log.i(LogTag, "Using fallback snapshot", e)
-      bitmap.readyForReuse()
-      fallback.takeSnapshot(view)
     }
 
-    return null
+    // something went wrong, use fallback
+    Log.i(LogTag, "Using fallback snapshot method")
+    bitmap.readyForReuse()
+    return fallback.takeSnapshot(view)
+  }
+
+  private suspend fun tryCopyViaActivityWindow(
+      view: View,
+      bitmap: BitmapPool.ReusableBitmap
+  ): Boolean {
+
+    val decorViewToActivity: Map<View, Activity> =
+        applicationRef.activitiesStack.toList().associateBy { it.window.decorView }
+
+    val activityForDecorView = decorViewToActivity[view] ?: return false
+
+    return suspendCoroutine { continuation ->
+      PixelCopy.request(
+          activityForDecorView.window, bitmap.bitmap, pixelCopyCallback(continuation), handler)
+    }
+  }
+
+  private suspend fun tryCopyViaInternalSurface(
+      view: View,
+      bitmap: BitmapPool.ReusableBitmap
+  ): Boolean {
+    val surface = applicationRef.windowManagerUtility.surfaceForRootView(view) ?: return false
+
+    return suspendCoroutine { continuation ->
+      PixelCopy.request(surface, bitmap.bitmap, pixelCopyCallback(continuation), handler)
+    }
+  }
+
+  private fun pixelCopyCallback(continuation: Continuation<Boolean>) = { result: Int ->
+    if (result == PixelCopy.SUCCESS) {
+      continuation.resume(true)
+    } else {
+      Log.w(LogTag, "Pixel copy failed, code $result")
+      continuation.resume(false)
+    }
   }
 }

android/src/main/java/com/facebook/flipper/plugins/uidebugger/core/UpdateQueue.kt

@@ -113,7 +113,7 @@ class UpdateQueue(val context: UIDContext) {
     if (update.snapshotBitmap != null) {
       val stream = ByteArrayOutputStream()
       val base64Stream = Base64OutputStream(stream, Base64.DEFAULT)
-      update.snapshotBitmap.bitmap?.compress(Bitmap.CompressFormat.PNG, 100, base64Stream)
+      update.snapshotBitmap.bitmap.compress(Bitmap.CompressFormat.PNG, 100, base64Stream)
       snapshot = Snapshot(update.snapshotNode, stream.toString())
       update.snapshotBitmap.readyForReuse()
     }

android/src/main/java/com/facebook/flipper/plugins/uidebugger/core/WindowManagerUtility.kt

@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+package com.facebook.flipper.plugins.uidebugger.core
+
+import android.annotation.SuppressLint
+import android.util.Log
+import android.view.Surface
+import android.view.View
+import com.facebook.flipper.plugins.uidebugger.LogTag
+import com.facebook.flipper.plugins.uidebugger.util.WindowManagerCommon
+import java.lang.reflect.Field
+
+/**
+ * This class is related to root view resolver, it also accesses parts of the global window manager
+ */
+class WindowManagerUtility {
+
+  private var initialized = false
+
+  // type is RootViewImpl
+  private var rootsImpls: ArrayList<*>? = null
+  private var mSurfaceField: Field? = null
+  private var mViewField: Field? = null
+
+  /**
+   * Find the surface for a given root view to allow snapshotting with pixelcopy. In the window
+   * manager there exists 2 arrays that contain similar data
+   * 1. mViews (this is what we track in the observable array in the root view resolver), these are
+   *    the root decor views
+   * 2. mRoots - this is an internal class that holds a reference to the decor view as well as other
+   *    internal bits (including the surface).
+   *
+   *    Therefore we go through the roots and check for a view that matches the target, if it
+   *    matches we return the surface. It is possible for us to observe these 2 arrays slightly out
+   *    of sync with each other which is why we do this equality matching on the view field
+   *
+   *    The reason we do this and not just grab the last view is because sometimes there is a
+   *    'empty' root view at the top we need to ignore. The decision to decide which view to
+   *    snapshot is done else where as it needs to be synced with the observation and traversal
+   */
+  fun surfaceForRootView(rootView: View): Surface? {
+    if (!initialized) {
+      initialize()
+    }
+
+    val roots = rootsImpls ?: return null
+    for (i in roots.size - 1 downTo 0) {
+      val rootViewImpl = roots[i]
+      val view = mViewField?.get(rootViewImpl)
+      if (view == rootView) {
+        return mSurfaceField?.get(rootViewImpl) as? Surface
+      }
+    }
+
+    return null
+  }
+
+  @SuppressLint("PrivateApi")
+  private fun initialize() {
+
+    try {
+      val (windowManager, windowManagerClass) =
+          WindowManagerCommon.getGlobalWindowManager() ?: return
+
+      val rootsField: Field = windowManagerClass.getDeclaredField(ROOTS_FIELD)
+      rootsField.isAccessible = true
+      rootsImpls = rootsField.get(windowManager) as ArrayList<*>?
+
+      val rootViewImplClass = Class.forName(VIEW_ROOT_IMPL_CLAZZ)
+      mSurfaceField = rootViewImplClass.getDeclaredField(SURFACE_FIELD)
+      mSurfaceField?.isAccessible = true
+
+      mViewField = rootViewImplClass.getDeclaredField(VIEW_FIELD)
+      mViewField?.isAccessible = true
+
+      initialized = true
+    } catch (exception: Exception) {
+      Log.e(LogTag, "Failed to initialize WindowManagerUtility", exception)
+    }
+  }
+
+  companion object {
+    private const val VIEW_ROOT_IMPL_CLAZZ = "android.view.ViewRootImpl"
+    private const val SURFACE_FIELD = "mSurface"
+    private const val VIEW_FIELD = "mView"
+    private const val ROOTS_FIELD = "mRoots"
+  }
+}

android/src/main/java/com/facebook/flipper/plugins/uidebugger/util/StopWatch.kt

@@ -21,6 +21,14 @@ class StopWatch() {
   companion object {
 
     fun <T> time(fn: () -> T): Pair<T, Long> {
+
+      val start = System.currentTimeMillis()
+      val result = fn()
+      val elapsed = System.currentTimeMillis() - start
+      return Pair(result, elapsed)
+    }
+
+    suspend fun <T> timeSuspend(fn: suspend () -> T): Pair<T, Long> {
       val start = System.currentTimeMillis()
       val result = fn()
       val elapsed = System.currentTimeMillis() - start

android/src/main/java/com/facebook/flipper/plugins/uidebugger/util/Throttler.kt

@@ -17,7 +17,7 @@ import kotlinx.coroutines.launch
  * This class will throttle calls into a callback. E.g if interval is 500ms and you receive triggers
  * at t=0, 100, 300 400, the callback will only be triggered at t=500
  */
-class Throttler<T>(private val intervalMs: Long, val callback: () -> T) {
+class Throttler<T>(private val intervalMs: Long, val callback: suspend () -> T) {
 
   private val executionScope: CoroutineScope = CoroutineScope(Dispatchers.Main)
   private var throttleJob: Job? = null

android/src/main/java/com/facebook/flipper/plugins/uidebugger/util/WindowManagerCommon.kt

@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+package com.facebook.flipper.plugins.uidebugger.util
+
+import android.os.Build
+import android.util.Log
+import com.facebook.flipper.plugins.uidebugger.LogTag
+
+object WindowManagerCommon {
+
+  // provides access to
+  // https://android.googlesource.com/platform/frameworks/base/+/refs/heads/main/core/java/android/view/WindowManagerGlobal.java
+  fun getGlobalWindowManager(): Pair<Any, Class<*>>? {
+    val accessClass =
+        if (Build.VERSION.SDK_INT > 16) WINDOW_MANAGER_GLOBAL_CLAZZ else WINDOW_MANAGER_IMPL_CLAZZ
+    val instanceMethod = if (Build.VERSION.SDK_INT > 16) GET_GLOBAL_INSTANCE else GET_DEFAULT_IMPL
+
+    try {
+      val clazz = Class.forName(accessClass)
+      val getMethod = clazz.getMethod(instanceMethod)
+      return Pair(getMethod.invoke(null), clazz)
+    } catch (exception: Exception) {
+      Log.e(LogTag, "Unable to get global window manager handle", exception)
+      return null
+    }
+  }
+
+  private const val GET_DEFAULT_IMPL = "getDefault"
+  private const val GET_GLOBAL_INSTANCE = "getInstance"
+  private const val WINDOW_MANAGER_IMPL_CLAZZ = "android.view.WindowManagerImpl"
+  private const val WINDOW_MANAGER_GLOBAL_CLAZZ = "android.view.WindowManagerGlobal"
+}