Coordinate update event when litho scrolls or is shifted

Summary: See doc comment for explanation

Reviewed By: lblasa

Differential Revision: D40587610

fbshipit-source-id: f0909440c4e6e3cc9f5c7b557198a93ba8809bd9

android/plugins/litho/src/main/java/com/facebook/flipper/plugins/uidebugger/litho/LithoObserver.kt

@@ -7,11 +7,17 @@
 
 package com.facebook.flipper.plugins.uidebugger.litho
 
+import android.annotation.SuppressLint
 import android.util.Log
 import android.view.ViewTreeObserver
 import com.facebook.flipper.plugins.uidebugger.LogTag
 import com.facebook.flipper.plugins.uidebugger.core.Context
+import com.facebook.flipper.plugins.uidebugger.descriptors.ViewDescriptor
 import com.facebook.flipper.plugins.uidebugger.descriptors.nodeId
+import com.facebook.flipper.plugins.uidebugger.litho.descriptors.LithoViewDescriptor
+import com.facebook.flipper.plugins.uidebugger.model.Bounds
+import com.facebook.flipper.plugins.uidebugger.model.Coordinate
+import com.facebook.flipper.plugins.uidebugger.observers.CoordinateUpdate
 import com.facebook.flipper.plugins.uidebugger.observers.TreeObserver
 import com.facebook.flipper.plugins.uidebugger.observers.TreeObserverBuilder
 import com.facebook.flipper.plugins.uidebugger.scheduler.throttleLatest
@@ -22,30 +28,32 @@ import kotlinx.coroutines.*
 
 /**
  * There are 2 ways a litho view can update:
- * 1. a view was added / updated / removed through a mount ,we use the mount extension to capture
+ * 1. A view was added / updated / removed through a mount, This should be refelected in a change in
- * these
+ * props / state so we use the mount extension to capture these including the entire component tree
- * 2. The user scrolled. This does not cause a mount to the litho view but it may cause new
+ * 2. The coordinate of the litho view changes externally and doesn't cause a mount, examples:
- * components to mount as they come on screen On the native side we capture scrolls as it causes the
+ * - Sibling changed size or position and shifted this view
- * draw listener to first but but the layout traversal would stop once it sees the lithoview.
+ * - User scrolled
  *
- * Therefore we need a way to capture the changes in the position of views in a litho view hierarchy
+ * These are not interesting from UI debugger perspective, we don't want to send the whole subtree
- * as they are scrolled. A property that seems to hold for litho is if there is a scrolling view in
+ * as only the Coordinate of the root litho view has changed. For this situation we send a
- * the heierachy, its direct children are lithoview.
+ * lightweight coordinate update event to distinguish these 2 cases
  *
- * Given that we are observing a litho view in this class for mount extension we can also attach a
+ * If an external event such as a scroll does does lead to a mount (new view in recycler view) this
- * on scroll changed listener to it to be notified by android when it is scrolled. We just need to
+ * will be picked up by the mount extension
- * then update the bounds for this view as nothing else has changed. If this scroll does lead to a
- * mount this will be picked up by the mount extension
  */
 class LithoViewTreeObserver(val context: Context) : TreeObserver<LithoView>() {
 
   override val type = "Litho"
-  private val throttleTimeMs = 500L
+  private val throttleTimeMs = 100L
 
   private val waitScope = CoroutineScope(Dispatchers.IO)
   private val mainScope = CoroutineScope(Dispatchers.Main)
+
+  var lastBounds: Bounds? = null
+
   var nodeRef: LithoView? = null
   private var preDrawListener: ViewTreeObserver.OnPreDrawListener? = null
+  @SuppressLint("PrivateApi")
   override fun subscribe(node: Any) {
 
     Log.d(LogTag, "Subscribing to litho view ${node.nodeId()}")
@@ -55,21 +63,28 @@ class LithoViewTreeObserver(val context: Context) : TreeObserver<LithoView>() {
     val lithoDebuggerExtension = LithoDebuggerExtension(this)
     node.registerUIDebugger(lithoDebuggerExtension)
 
-    val throttledUpdate =
+    val throttledCordinateUpdate =
-        throttleLatest<Any>(throttleTimeMs, waitScope, mainScope) { node ->
+        throttleLatest<LithoView>(throttleTimeMs, waitScope, mainScope) { node ->
-          // todo only send bounds for the view rather than the entire hierachy
+          // use the descriptor to get the bounds since we do some magic in there
-          processUpdate(context, node)
+          val bounds = ViewDescriptor.onGetBounds(node)
+          if (bounds != lastBounds) {
+            context.treeObserverManager.enqueueUpdate(
+                CoordinateUpdate(this.type, node.nodeId(), Coordinate(bounds.x, bounds.y)))
+            lastBounds = bounds
+          }
         }
 
     preDrawListener =
         ViewTreeObserver.OnPreDrawListener {
-          throttledUpdate(node)
+          // this cases case 2
+          throttledCordinateUpdate(node)
           true
         }
 
     node.viewTreeObserver.addOnPreDrawListener(preDrawListener)
 
     // we have already missed the first mount so we trigger it manually on subscribe
+    lastBounds = LithoViewDescriptor.onGetBounds(node)
     processUpdate(context, node)
   }
 
@@ -93,7 +108,7 @@ class LithoDebuggerExtension(val observer: LithoViewTreeObserver) : MountExtensi
    */
   override fun afterMount(state: ExtensionState<Void?>) {
     Log.i(LogTag, "After mount called for litho view ${observer.nodeRef?.nodeId()}")
-    // todo sparse update
+    observer.lastBounds = ViewDescriptor.onGetBounds(state.rootHost)
     observer.processUpdate(observer.context, state.rootHost as Any)
   }
 }

android/src/main/java/com/facebook/flipper/plugins/uidebugger/model/Events.kt

@@ -29,6 +29,17 @@ data class SubtreeUpdateEvent(
   }
 }
 
+@kotlinx.serialization.Serializable
+data class CoordinateUpdateEvent(
+    val observerType: String,
+    val nodeId: Id,
+    val coordinate: Coordinate
+) {
+  companion object {
+    const val name = "coordinateUpdate"
+  }
+}
+
 /** Separate optional performance statistics event */
 @kotlinx.serialization.Serializable
 data class PerfStatsEvent(

android/src/main/java/com/facebook/flipper/plugins/uidebugger/observers/TreeObserverManager.kt

@@ -16,6 +16,8 @@ import com.facebook.flipper.plugins.uidebugger.LogTag
 import com.facebook.flipper.plugins.uidebugger.common.BitmapPool
 import com.facebook.flipper.plugins.uidebugger.core.Context
 import com.facebook.flipper.plugins.uidebugger.descriptors.Id
+import com.facebook.flipper.plugins.uidebugger.model.Coordinate
+import com.facebook.flipper.plugins.uidebugger.model.CoordinateUpdateEvent
 import com.facebook.flipper.plugins.uidebugger.model.Node
 import com.facebook.flipper.plugins.uidebugger.model.PerfStatsEvent
 import com.facebook.flipper.plugins.uidebugger.model.SubtreeUpdateEvent
@@ -25,6 +27,11 @@ import kotlinx.coroutines.*
 import kotlinx.coroutines.channels.Channel
 import kotlinx.serialization.json.Json
 
+sealed interface Update
+
+data class CoordinateUpdate(val observerType: String, val nodeId: Id, val coordinate: Coordinate) :
+    Update
+
 data class SubtreeUpdate(
     val observerType: String,
     val rootId: Id,
@@ -33,19 +40,19 @@ data class SubtreeUpdate(
     val traversalCompleteTime: Long,
     val snapshotComplete: Long,
     val snapshot: BitmapPool.ReusableBitmap?
-)
+) : Update
 
 /** Holds the root observer and manages sending updates to desktop */
 class TreeObserverManager(val context: Context) {
 
   private val rootObserver = ApplicationTreeObserver(context)
-  private lateinit var treeUpdates: Channel<SubtreeUpdate>
+  private lateinit var updates: Channel<Update>
   private var job: Job? = null
   private val workerScope = CoroutineScope(Dispatchers.IO)
   private val txId = AtomicInteger()
 
-  fun enqueueUpdate(update: SubtreeUpdate) {
+  fun enqueueUpdate(update: Update) {
-    treeUpdates.trySend(update)
+    updates.trySend(update)
   }
 
   /**
@@ -55,64 +62,24 @@ class TreeObserverManager(val context: Context) {
   @SuppressLint("NewApi")
   fun start() {
 
-    treeUpdates = Channel(Channel.UNLIMITED)
+    updates = Channel(Channel.UNLIMITED)
     rootObserver.subscribe(context.applicationRef)
 
     job =
         workerScope.launch {
           while (isActive) {
             try {
-              val treeUpdate = treeUpdates.receive()
-              val onWorkerThread = System.currentTimeMillis()
-              val txId = txId.getAndIncrement().toLong()
 
-              var serialized: String?
+              val update = updates.receive()
-              if (treeUpdate.snapshot == null) {
+              when (update) {
-                serialized =
+                is SubtreeUpdate -> sendSubtreeUpdate(update)
-                    Json.encodeToString(
+                is CoordinateUpdate -> {
-                        SubtreeUpdateEvent.serializer(),
+                  val event =
-                        SubtreeUpdateEvent(
+                      CoordinateUpdateEvent(update.observerType, update.nodeId, update.coordinate)
-                            txId, treeUpdate.observerType, treeUpdate.rootId, treeUpdate.nodes))
+                  val serialized = Json.encodeToString(CoordinateUpdateEvent.serializer(), event)
-              } else {
+                  context.connectionRef.connection?.send(CoordinateUpdateEvent.name, serialized)
-                val stream = ByteArrayOutputStream()
+                }
-                val base64Stream = Base64OutputStream(stream, Base64.DEFAULT)
-                treeUpdate.snapshot.bitmap?.compress(Bitmap.CompressFormat.JPEG, 100, base64Stream)
-                val snapshot = stream.toString()
-                serialized =
-                    Json.encodeToString(
-                        SubtreeUpdateEvent.serializer(),
-                        SubtreeUpdateEvent(
-                            txId,
-                            treeUpdate.observerType,
-                            treeUpdate.rootId,
-                            treeUpdate.nodes,
-                            snapshot))
-
-                treeUpdate.snapshot.readyForReuse()
               }
-
-              val serializationEnd = System.currentTimeMillis()
-
-              context.connectionRef.connection?.send(SubtreeUpdateEvent.name, serialized)
-              val socketEnd = System.currentTimeMillis()
-              Log.i(
-                  LogTag,
-                  "Sent event for ${treeUpdate.observerType} root ID ${treeUpdate.rootId} nodes ${treeUpdate.nodes.size}")
-
-              val perfStats =
-                  PerfStatsEvent(
-                      txId = txId,
-                      observerType = treeUpdate.observerType,
-                      start = treeUpdate.startTime,
-                      traversalComplete = treeUpdate.traversalCompleteTime,
-                      snapshotComplete = treeUpdate.snapshotComplete,
-                      queuingComplete = onWorkerThread,
-                      serializationComplete = serializationEnd,
-                      socketComplete = socketEnd,
-                      nodesCount = treeUpdate.nodes.size)
-
-              context.connectionRef.connection?.send(
-                  PerfStatsEvent.name, Json.encodeToString(PerfStatsEvent.serializer(), perfStats))
             } catch (e: CancellationException) {} catch (e: java.lang.Exception) {
               Log.e(LogTag, "Unexpected Error in channel ", e)
             }
@@ -121,9 +88,58 @@ class TreeObserverManager(val context: Context) {
         }
   }
 
+  fun sendSubtreeUpdate(treeUpdate: SubtreeUpdate) {
+    val onWorkerThread = System.currentTimeMillis()
+    val txId = txId.getAndIncrement().toLong()
+
+    var serialized: String?
+    if (treeUpdate.snapshot == null) {
+      serialized =
+          Json.encodeToString(
+              SubtreeUpdateEvent.serializer(),
+              SubtreeUpdateEvent(
+                  txId, treeUpdate.observerType, treeUpdate.rootId, treeUpdate.nodes))
+    } else {
+      val stream = ByteArrayOutputStream()
+      val base64Stream = Base64OutputStream(stream, Base64.DEFAULT)
+      treeUpdate.snapshot.bitmap?.compress(Bitmap.CompressFormat.JPEG, 100, base64Stream)
+      val snapshot = stream.toString()
+      serialized =
+          Json.encodeToString(
+              SubtreeUpdateEvent.serializer(),
+              SubtreeUpdateEvent(
+                  txId, treeUpdate.observerType, treeUpdate.rootId, treeUpdate.nodes, snapshot))
+
+      treeUpdate.snapshot.readyForReuse()
+    }
+
+    val serializationEnd = System.currentTimeMillis()
+
+    context.connectionRef.connection?.send(SubtreeUpdateEvent.name, serialized)
+    val socketEnd = System.currentTimeMillis()
+    Log.i(
+        LogTag,
+        "Sent event for ${treeUpdate.observerType} root ID ${treeUpdate.rootId} nodes ${treeUpdate.nodes.size}")
+
+    val perfStats =
+        PerfStatsEvent(
+            txId = txId,
+            observerType = treeUpdate.observerType,
+            start = treeUpdate.startTime,
+            traversalComplete = treeUpdate.traversalCompleteTime,
+            snapshotComplete = treeUpdate.snapshotComplete,
+            queuingComplete = onWorkerThread,
+            serializationComplete = serializationEnd,
+            socketComplete = socketEnd,
+            nodesCount = treeUpdate.nodes.size)
+
+    context.connectionRef.connection?.send(
+        PerfStatsEvent.name, Json.encodeToString(PerfStatsEvent.serializer(), perfStats))
+  }
+
   fun stop() {
     rootObserver.cleanUpRecursive()
     job?.cancel()
-    treeUpdates.cancel()
+    updates.cancel()
   }
 }

desktop/plugins/public/ui-debugger/index.tsx

@@ -24,6 +24,18 @@ export function plugin(client: PluginClient<Events>) {
 
   const nodesAtom = createState<Map<Id, UINode>>(new Map());
   const snapshotsAtom = createState<Map<Id, Snapshot>>(new Map());
+
+  client.onMessage('coordinateUpdate', (event) => {
+    nodesAtom.update((draft) => {
+      const node = draft.get(event.nodeId);
+      if (!node) {
+        console.warn(`Coordinate update for non existing node `, event);
+      } else {
+        node.bounds.x = event.coordinate.x;
+        node.bounds.y = event.coordinate.y;
+      }
+    });
+  });
   client.onMessage('subtreeUpdate', (event) => {
     snapshotsAtom.update((draft) => {
       draft.set(event.rootId, event.snapshot);

desktop/plugins/public/ui-debugger/types.tsx

@@ -10,9 +10,16 @@
 export type Events = {
   init: InitEvent;
   subtreeUpdate: SubtreeUpdateEvent;
+  coordinateUpdate: CoordinateUpdateEvent;
   perfStats: PerfStatsEvent;
 };
 
+export type CoordinateUpdateEvent = {
+  observerType: String;
+  nodeId: Id;
+  coordinate: Coordinate;
+};
+
 export type SubtreeUpdateEvent = {
   txId: number;
   rootId: Id;
@@ -39,7 +46,7 @@ export type UINode = {
   name: string;
   attributes: Record<string, Inspectable>;
   children: Id[];
-  bounds?: Bounds;
+  bounds: Bounds;
   tags: Tag[];
   activeChild?: Id;
 };