Fix build caching (#242)

Summary:
See https://github.com/facebook/flipper/issues/230 for a longer
explanation.

It turns out that many of the tasks in `native.gradle` (at least
`prepareGlog` which modifies files inside the source directory, but also
others) cause the cmake cache to invalidate.

Because we run those tasks unconditionally, we need to recompile all our
C++ code even when unmodified.

I fix this by adding a manual cache key to this which simply skips all
the tasks unless an up-to-date `external_cache_revision.txt` is found.
This is a bit hacky, but the alternatives are a) convincing cmake that
these modifications are not recompile-worthy which I have no idea how to
do and which might be impossible to do `mtime` on the files touches, or
b) having fine-grained `onlyIf` rules for every task which can easily
break. This seems the most straight-forward option.

Fixes #230.
Pull Request resolved: https://github.com/facebook/flipper/pull/242

Reviewed By: jknoxville

Differential Revision: D9381340

Pulled By: passy

fbshipit-source-id: a7db55aacac4a29076c29298d6c5b97d8bc91f66

android/third-party/native.gradle

@@ -1,13 +1,29 @@
 import org.apache.tools.ant.filters.ReplaceTokens
 
-final def downloadsDir = new File("$buildDir/downloads")
+// !!!
+// Increment this when making changes to any of the native
+// dependencies.
+// !!!
+final def CACHE_REVISION = 0
+
 final def externalDir = new File("$projectDir/external")
+final def downloadsDir = new File("$buildDir/downloads")
+final def revisionFile = new File("$buildDir/external_cache_revision.txt")
 
 final def getDownloadFileName = { final URL src ->
     final def i = src.file.lastIndexOf('/')
     return src.file.substring(i + 1)
 }
 
+final def isCacheOutOfDate = { final Integer revision ->
+    if (!revisionFile.exists()) {
+        return true
+    } else {
+        final def content = revisionFile.text
+        return !content.isInteger() || content as Integer != revision
+    }
+}
+
 task createNativeDepsDirectories {
     downloadsDir.mkdirs()
     externalDir.mkdirs()
@@ -25,6 +41,7 @@ task downloadGlog(dependsOn: createNativeDepsDirectories, type: Download) {
 }
 
 task prepareGlog(dependsOn: [downloadGlog], type: Copy) {
+    onlyIf { isCacheOutOfDate(CACHE_REVISION) }
     from tarTree(downloadGlog.dest)
     from './overrides/glog/'
     include 'glog-0.3.5/src/**/*', 'Android.mk', 'config.h', 'build.gradle', 'CMakeLists.txt', 'ApplicationManifest.xml'
@@ -99,6 +116,7 @@ task downloadFolly(dependsOn: createNativeDepsDirectories, type: Download) {
 }
 
 task prepareFolly(dependsOn: [downloadFolly], type: Copy) {
+    onlyIf { isCacheOutOfDate(CACHE_REVISION) }
     from tarTree(downloadFolly.dest)
     from './overrides/Folly/'
     include 'folly-2018.06.18.00/folly/**/*', 'build.gradle', 'CMakeLists.txt', 'ApplicationManifest.xml'
@@ -122,27 +140,21 @@ task downloadLibEvent(dependsOn: [], type: Download) {
 }
 
 task prepareLibEvent(dependsOn: [downloadLibEvent], type: Copy) {
+    onlyIf { isCacheOutOfDate(CACHE_REVISION) }
     from tarTree(downloadLibEvent.dest)
     from './overrides/LibEvent/'
-    include 'libevent-release-2.1.9/**/*', 'build.gradle', 'ApplicationManifest.xml'
+    include 'libevent-release-2.1.9/**/*', 'build.gradle', 'ApplicationManifest.xml', 'CMakeLists.txt'
     includeEmptyDirs = false
     into "$externalDir/LibEvent"
 }
 
-task finalizeEvent(dependsOn: [prepareLibEvent], type: Copy) {
+task finalizeLibEvent(dependsOn: [prepareLibEvent], type: Copy) {
     from './overrides/LibEvent/'
     include 'event-config.h'
     includeEmptyDirs = false
     into "$externalDir/LibEvent/libevent-release-2.1.9/include/event2/"
 }
 
-task finalizeEvent2(dependsOn: [finalizeEvent], type: Copy) {
-    from './overrides/LibEvent/'
-    include 'libs/**/*'
-    includeEmptyDirs = false
-    into "$externalDir/LibEvent/"
-}
-
 task downloadOpenSSLSource(dependsOn: [], type: Download) {
     src 'https://github.com/passy/openssl-android/releases/download/1.1.0h/openssl-1.1.0h.tar.gz'
     onlyIfNewer true
@@ -158,6 +170,7 @@ task downloadOpenSSLLibs(dependsOn: [], type: Download) {
 }
 
 task prepareOpenSSL(dependsOn: [downloadOpenSSLSource, downloadOpenSSLLibs], type: Copy) {
+    onlyIf { isCacheOutOfDate(CACHE_REVISION) }
     from tarTree(downloadOpenSSLSource.dest)
     from tarTree(downloadOpenSSLLibs.dest)
     from './overrides/OpenSSL/'
@@ -168,6 +181,7 @@ task prepareOpenSSL(dependsOn: [downloadOpenSSLSource, downloadOpenSSLLibs], typ
 }
 
 task configureOpenSSL(dependsOn: [prepareOpenSSL], type: Exec) {
+    onlyIf { isCacheOutOfDate(CACHE_REVISION) }
     workingDir "$externalDir/OpenSSL/openssl-1.1.0h/"
     // This is only to generate a buildconfig.h in the next step. I **believe**
     // that the options here don't really matter for that file.
@@ -195,12 +209,28 @@ task prepareRSocket(dependsOn: [downloadRSocket], type: Copy) {
     into "$externalDir/RSocket"
 }
 
-task prepareAllLibs() {
-    dependsOn finalizeGlog
-    dependsOn prepareDoubleConversion
-    dependsOn prepareBoost
-    dependsOn finalizeFolly
-    dependsOn finalizeEvent
-    dependsOn finalizeOpenSSL
-    dependsOn prepareRSocket
+task writeCacheRevision() {
+    doLast {
+        revisionFile.text = CACHE_REVISION.toString()
+    }
+}
+
+def allTasks = [
+    finalizeGlog,
+    prepareDoubleConversion,
+    prepareBoost,
+    finalizeFolly,
+    finalizeLibEvent,
+    finalizeOpenSSL,
+    prepareRSocket,
+    writeCacheRevision,
+]
+
+allTasks.each { i ->
+    i.onlyIf { isCacheOutOfDate(CACHE_REVISION) }
+}
+
+task prepareAllLibs() {
+    onlyIf { isCacheOutOfDate(CACHE_REVISION) }
+    dependsOn allTasks
 }

android/third-party/overrides/LibEvent/CMakeLists.txt

@@ -1,37 +0,0 @@
-cmake_minimum_required (VERSION 3.6.0)
-
-PROJECT(libevent C)
-enable_language(C)
-set(PACKAGE_NAME      libevent)
-set(LIBEVENT_DIR  libevent-release-2.1.8-stable)
-list(APPEND DIR_LIST ${LIBEVENT_DIR}/)
-list(APPEND DIR_LIST ${LIBEVENT_DIR}/include)
-include_directories(${DIR_LIST})
-list(APPEND SRCFILES ${LIBEVENT_DIR}/event.c
-                     ${LIBEVENT_DIR}/buffer.c
-                     ${LIBEVENT_DIR}/bufferevent.c
-                     ${LIBEVENT_DIR}/bufferevent_filter.c
-                     ${LIBEVENT_DIR}/bufferevent_ratelim.c
-			 ${LIBEVENT_DIR}/bufferevent_sock.c
-                     ${LIBEVENT_DIR}/epoll.c
-			 ${LIBEVENT_DIR}/epoll_sub.c
-			 ${LIBEVENT_DIR}/evdns.c
-			 ${LIBEVENT_DIR}/event_tagging.c
-			 ${LIBEVENT_DIR}/evmap.c
-			 ${LIBEVENT_DIR}/evrpc.c
-			 ${LIBEVENT_DIR}/evthread.c
-			 ${LIBEVENT_DIR}/evthread_pthread.c
-			 ${LIBEVENT_DIR}/evutil.c
-			 ${LIBEVENT_DIR}/evutil_rand.c
-			 ${LIBEVENT_DIR}/http.c
-			 ${LIBEVENT_DIR}/listener.c
-			 ${LIBEVENT_DIR}/log.c
-			 ${LIBEVENT_DIR}/poll.c
-			 ${LIBEVENT_DIR}/select.c
-			 ${LIBEVENT_DIR}/signal.c
-			 ${LIBEVENT_DIR}/strlcpy.c
-)
-
-add_library(${PACKAGE_NAME} SHARED ${SRCFILES})
-install(TARGETS ${PACKAGE_NAME}  DESTINATION ./build/)
-target_link_libraries(${PACKAGE_NAME})

xplat/CMakeLists.txt

@@ -48,7 +48,6 @@ set(rsocket_build_DIR ${build_DIR}/rsocket/${ANDROID_ABI})
 
 file(MAKE_DIRECTORY ${build_DIR})
 
-#add_subdirectory(${libfolly_DIR} ${libfolly_build_DIR})
 add_subdirectory(${rsocket_DIR} ${rsocket_build_DIR})
 
 message(STATUS "RSocket DIR:- " ${rsocket_DIR})