From 6f95ad512f31508ff01ad5a10299e1bb19e88e94 Mon Sep 17 00:00:00 2001 From: Daniel Buchele Date: Fri, 15 Jun 2018 02:23:48 -0700 Subject: [PATCH] fbshipit-source-id: b14273e883aba6de7b817801a1b04e54a29a6366 --- android/build.gradle | 4 +- android/sample/build.gradle | 2 - build.gradle | 25 +--- docs/getting-started.md | 1 + .../CppBridge/SonarCppBridgingConnection.mm | 2 +- package.json | 4 +- scripts/public-build.json | 70 ++++++++-- scripts/public-build.sh | 56 +++----- src/server.js | 51 ++++++- src/utils/CertificateProvider.js | 129 +++++++++++------- src/utils/promise.js | 0 xplat/Sonar/SonarWebSocketImpl.cpp | 3 +- 12 files changed, 217 insertions(+), 130 deletions(-) create mode 100644 src/utils/promise.js diff --git a/android/build.gradle b/android/build.gradle index 92f11a0f5..cd641166d 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -121,11 +121,13 @@ android { buildConfigField "boolean", "IS_INTERNAL_BUILD", 'true' ndk { abiFilters 'x86', 'x86_64', 'armeabi-v7a', 'arm64-v8a' + stl 'c++_shared' } externalNativeBuild { cmake { arguments '-DANDROID_TOOLCHAIN=clang' + arguments '-DANDROID_STL=c++_shared' } } } @@ -163,7 +165,7 @@ android { implementation deps.okhttp3 implementation 'com.facebook.litho:litho-core:0.15.0' implementation 'com.facebook.litho:litho-widget:0.15.0' - implementation fileTree(dir: 'plugins/console/dependencies', include: ['*.jar']) + implementation 'org.mozilla:rhino:1.7.10' } } diff --git a/android/sample/build.gradle b/android/sample/build.gradle index 5d0794a74..b6f141cfc 100644 --- a/android/sample/build.gradle +++ b/android/sample/build.gradle @@ -40,7 +40,6 @@ dependencies { testImplementation 'junit:junit:4.12' androidTestImplementation 'com.android.support.test:runner:1.0.2' androidTestImplementation 'com.android.support.test.espresso:espresso-core:3.0.2' - // ... // Litho implementation 'com.facebook.litho:litho-core:0.15.0' implementation 'com.facebook.litho:litho-widget:0.15.0' @@ -59,5 +58,4 @@ dependencies { implementation 'com.squareup.okhttp3:okhttp:3.10.0' implementation project(':android') - //implementation project(':sonar') } diff --git a/build.gradle b/build.gradle index 66b141600..1359edbf8 100644 --- a/build.gradle +++ b/build.gradle @@ -1,10 +1,3 @@ -/* - * This file was generated by the Gradle 'init' task. - * - * This is a general purpose Gradle build. - * Learn how to create Gradle builds at https://guides.gradle.org/creating-new-gradle-builds/ - */ - buildscript { repositories { jcenter() @@ -17,9 +10,6 @@ buildscript { classpath "com.github.ben-manes:gradle-versions-plugin:${GRADLE_VERSIONS_PLUGIN_VERSION}" classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:${KOTLIN_VERSION}" classpath 'de.undercouch:gradle-download-task:3.1.2' - - // NOTE: Do not place your application dependencies here; they belong - // in the individual module build.gradle files } } @@ -33,7 +23,7 @@ subprojects { ext { minSdkVersion = 15 targetSdkVersion = 25 - compileSdkVersion = 26 + compileSdkVersion = 27 buildToolsVersion = '27.0.3' sourceCompatibilityVersion = JavaVersion.VERSION_1_7 targetCompatibilityVersion = JavaVersion.VERSION_1_7 @@ -41,15 +31,15 @@ ext { ext.deps = [ // Android support - supportAnnotations : 'com.android.support:support-annotations:27.0.2', - supportAppCompat : 'com.android.support:appcompat-v7:26.1.0', - supportCoreUi : 'com.android.support:support-core-ui:26.1.0', - supportRecyclerView: 'com.android.support:recyclerview-v7:26.1.0', + supportAnnotations : 'com.android.support:support-annotations:27.1.1', + supportAppCompat : 'com.android.support:appcompat-v7:27.1.1', + supportCoreUi : 'com.android.support:support-core-ui:27.1.1', + supportRecyclerView: 'com.android.support:recyclerview-v7:27.1.1', supportEspresso : 'com.android.support.test.espresso:espresso-core:2.2.2', supportEspressoIntents : 'com.android.support.test.espresso:espresso-intents:2.2.2', - supportTestRunner : 'com.android.support.test:runner:1.0.1', + supportTestRunner : 'com.android.support.test:runner:1.0.2', // Arch - archPaging : 'android.arch.paging:runtime:1.0.0-alpha3', + archPaging : 'android.arch.paging:runtime:1.0.0', // First-party soloader : 'com.facebook.soloader:soloader:0.4.1', screenshot : 'com.facebook.testing.screenshot:core:0.5.0', @@ -61,7 +51,6 @@ ext.deps = [ guava : 'com.google.guava:guava:20.0', robolectric : 'org.robolectric:robolectric:3.0', junit : 'junit:junit:4.12', - guava : 'com.google.guava:guava:20.0', stetho : 'com.facebook.stetho:stetho:1.5.0', okhttp3 : 'com.squareup.okhttp3:okhttp:3.10.0' diff --git a/docs/getting-started.md b/docs/getting-started.md index b8e736b0b..0cf2611e7 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -110,6 +110,7 @@ and install the dependencies by running `pod install`. When you open the Xcode w * We haven't released the dependency to CocoaPods, because we weren't able to successfully validate the podspec of SonarKit. You could help us out by fixing this [issue](https://github.com/facebook/Sonar/issues/11) by submitting a PR to the repo. * If you do not use CocoaPods as a dependency management tool then currently there is no way to integrate SonarKit other than manually including all the dependencies and building it. +* For Android, Sonar works with both emulators and physical devices connected through USB. However on iOS, we don't yet support physical devices. * Also Sonar doesn't work with swift projects as its written in C++ and had C++ dependencies. But we are working on supporting sonar for swift projects. You can find this issue [here](https://github.com/facebook/Sonar/issues/13) diff --git a/iOS/SonarKit/CppBridge/SonarCppBridgingConnection.mm b/iOS/SonarKit/CppBridge/SonarCppBridgingConnection.mm index 031cdbca7..48a9eadcd 100644 --- a/iOS/SonarKit/CppBridge/SonarCppBridgingConnection.mm +++ b/iOS/SonarKit/CppBridge/SonarCppBridgingConnection.mm @@ -28,7 +28,7 @@ - (void)send:(NSString *)method withParams:(NSDictionary *)params { - conn_->send([method UTF8String], facebook::cxxutils::convertIdToFollyDynamic(params)); + conn_->send([method UTF8String], facebook::cxxutils::convertIdToFollyDynamic(params, true)); } - (void)receive:(NSString *)method withBlock:(SonarReceiver)receiver diff --git a/package.json b/package.json index bf2a9026c..2fa4fcc9f 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,6 @@ } }, "devDependencies": { - "7zip-bin-mac": "^1.0.1", "babel-eslint": "^8.2.1", "electron": "^2.0.1", "electron-builder": "^19.49.0", @@ -83,5 +82,8 @@ "build": "yarn rm-dist && NODE_ENV=production node scripts/build-release.js $@", "fix": "eslint . --fix", "lint": "eslint . && flow check" + }, + "optionalDependencies": { + "7zip-bin-mac": "^1.0.1" } } diff --git a/scripts/public-build.json b/scripts/public-build.json index 40db5cde3..b5fe995d5 100644 --- a/scripts/public-build.json +++ b/scripts/public-build.json @@ -1,20 +1,68 @@ { "command": "SandcastleUniversalCommand", "args": { - "name": "Release public Sonar build", - "oncall": "danielbuechele", - "steps": [ - { - "name": "sonar_release_public_build", - "required": true, - "shell": "cd ../xplat/sonar/scripts && ./public-build.sh" - } - ] + "name": "Release public Sonar build", + "oncall": "danielbuechele", + "steps": [ + { + "name": "Clone from GitHub", + "required": true, + "shell": "git -c http.proxy=fwdproxy:8080 -c https.proxy=fwdproxy:8080 clone https://github.com/facebook/Sonar.git ../xplat/sonar-public" + }, + { + "name": "Create Version", + "required": true, + "shell": "cat ../xplat/sonar-public/package.json | jq -r '.version' | sed -E 's/[0-9]+$/__VERSION__/g' > $SANDCASTLE_NEXUS/VERSION" + }, + { + "name": "Copy Sandcastle script", + "required": true, + "shell": "cp ../xplat/sonar/scripts/sandcastle-build.sh ../xplat/sonar-public/scripts/sandcastle-build.sh" + }, + { + "name": "Create build", + "required": true, + "shell": "cd ../xplat/sonar-public/scripts && ./sandcastle-build.sh $(cat $SANDCASTLE_NEXUS/VERSION)" + }, + { + "name": "Copy artifacts for syncing", + "required": true, + "shell": "cp -R ../xplat/sonar-public/dist/Sonar-mac.zip $SANDCASTLE_NEXUS/Sonar-mac.zip" + }, + { + "name": "Sync to local", + "step_type": "sync_step", + "from": "remote", + "to": "local", + "paths": [ + "Sonar-mac.zip", + "VERSION" + ] + }, + { + "name": "Publish to github", + "required": true, + "shell": "curl -o RELEASE.json -x fwdproxy:8080 --silent --data '{ \"tag_name\": \"v'$(cat $SANDCASTLE_NEXUS/VERSION)'\", \"target_commitish\": \"master\", \"name\": \"v'$(cat $SANDCASTLE_NEXUS/VERSION)'\", \"body\": \"\", \"draft\": false, \"prerelease\": false}' https://api.github.com/repos/facebook/Sonar/releases?access_token=$(secrets_tool get SONAR_GITHUB_TOKEN)", + "shell_type": "SandcastleLocalShell" + }, + { + "name": "Upload", + "required": true, + "shell": "curl -x fwdproxy:8080 $(cat RELEASE.json | jq -r '.upload_url' | sed -e 's#{?name,label}##')'?access_token='$(secrets_tool get SONAR_GITHUB_TOKEN)'&name=Sonar.zip' --header 'Content-Type: application/zip' --upload-file $SANDCASTLE_NEXUS'/Sonar-mac.zip' -X POST", + "shell_type": "SandcastleLocalShell" + } + ] }, "alias": "sonar_release_public_build", "capabilities": { "vcs": "fbcode-fbsource", - "type": "lego" + "type": "lego-mac" }, - "hash": "master" + "hash": "master", + "report": [ + { + "type": "chirp", + "users": ["__USER__"] + } + ] } diff --git a/scripts/public-build.sh b/scripts/public-build.sh index aae5f60e9..4702a055f 100755 --- a/scripts/public-build.sh +++ b/scripts/public-build.sh @@ -1,48 +1,24 @@ #!/bin/bash -TOKEN=$(secrets_tool get SONAR_GITHUB_TOKEN) -GITHUB_ORG="facebook" -GITHUB_REPO="Sonar" -cd ../../ || exit +echo "✨ Creating new Sonar release on GitHub..." +MAJOR=$(curl -x fwdproxy:8080 --silent https://raw.githubusercontent.com/facebook/Sonar/master/package.json | jq -r '.version' | sed -E 's/[0-9]+$//g') -function jsonValue() { - python -c 'import json,sys;obj=json.load(sys.stdin);print obj["'$1'"]' || echo '' -} +echo "What should the patch version of the next release be? (v${MAJOR}_)" -git -c http.proxy=fwdproxy:8080 -c https.proxy=fwdproxy:8080 clone https://github.com/facebook/Sonar.git sonar-public -cp sonar/scripts/sandcastle-build.sh sonar-public/scripts/sandcastle-build.sh -# third-party dependencies are not on github, so we need to copy them in place -cp -r sonar/third-party sonar-public/third-party -cd sonar-public/scripts && ./sandcastle-build.sh "$(git rev-list HEAD --count || echo 0)" - -VERSION=$(plutil -p ./sonar-public/dist/mac/Sonar.app/Contents/Info.plist | awk '/CFBundleShortVersionString/ {print substr($3, 2, length($3)-2)}') - -RELEASE_JSON=$(curl -x fwdproxy:8080 --silent --data '{ - "tag_name": "v'$VERSION'", - "target_commitish": "master", - "name": "v'$VERSION'", - "body": "", - "draft": false, - "prerelease": false -}' https://api.github.com/repos/$GITHUB_ORG/$GITHUB_REPO/releases?access_token=$TOKEN) - -RELEASE_ID=$(echo $RELEASE_JSON | jsonValue id) - -if [ -z "${RELEASE_ID}" ]; then - echo $RELEASE_JSON - exit 1 +read -r VERSION +if ! [[ $VERSION =~ ^[0-9]+$ ]] ; then + echo "error: Version needs to be a number" >&2; exit 1 fi -echo "Created GitHub release ID: $RELEASE_ID" -UPLOAD_URL=$(echo $RELEASE_JSON | jsonValue upload_url| sed -e 's#{?name,label}##') -ASSET_JSON=$(curl -x fwdproxy:8080 --silent $UPLOAD_URL'?access_token='$TOKEN'&name=Sonar.zip' --header 'Content-Type: application/zip' --upload-file ./sonar-public/dist/Sonar.zip -X POST) +echo "Creating version $MAJOR$VERSION and releasing to GitHub..." +TMP_DIR=$(mktemp -d) +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +JSON=$(cat "$DIR/public-build.json") +JSON=${JSON/__VERSION__/$VERSION} +JSON=${JSON/__USER__/$USER} +echo "$JSON" > "$TMP_DIR/job.json" -DOWNLOAD_URL=$(echo $ASSET_JSON | jsonValue browser_download_url) +scutil create "$TMP_DIR/job.json" +echo "✅ GitHub release will be automatically created once the Sandcastle job finishes." -if [ -z "${DOWNLOAD_URL}" ]; then - echo $ASSET_JSON - exit 1 -fi - -echo "Released Sonar v$VERSION" -echo "Download: $DOWNLOAD_URL" +rm -rf "$TMP_DIR" diff --git a/src/server.js b/src/server.js index e98da0e5f..cec54bf44 100644 --- a/src/server.js +++ b/src/server.js @@ -11,6 +11,7 @@ import type {SonarPlugin} from './plugin.js'; import plugins from './plugins/index.js'; import CertificateProvider from './utils/CertificateProvider'; import type {SecureServerConfig} from './utils/CertificateProvider'; +import type Logger from './fb-stubs/Logger'; import {RSocketServer, ReactiveSocket, PartialResponder} from 'rsocket-core'; import RSocketTCPServer from 'rsocket-tcp-server'; @@ -315,6 +316,7 @@ export class Server extends EventEmitter { secureServer: RSocketServer; insecureServer: RSocketServer; certificateProvider: CertificateProvider; + connectionTracker: ConnectionTracker; app: App; constructor(app: App) { @@ -322,6 +324,7 @@ export class Server extends EventEmitter { this.app = app; this.connections = new Map(); this.certificateProvider = new CertificateProvider(this, app.logger); + this.connectionTracker = new ConnectionTracker(app.logger); this.init(); } @@ -389,7 +392,10 @@ export class Server extends EventEmitter { _trustedRequestHandler = (conn: RSocket, connectRequest: {data: string}) => { const server = this; - const client = this.addConnection(conn, connectRequest.data); + const clientData: ClientQuery = JSON.parse(connectRequest.data); + this.connectionTracker.logConnectionAttempt(clientData); + + const client = this.addConnection(conn, clientData); conn.connectionStatus().subscribe({ onNext(payload) { @@ -413,7 +419,8 @@ export class Server extends EventEmitter { conn: RSocket, connectRequest: {data: string}, ) => { - const connectionParameters = JSON.parse(connectRequest.data); + const clientData = JSON.parse(connectRequest.data); + this.connectionTracker.logConnectionAttempt(clientData); return { fireAndForget: (payload: {data: string}) => { @@ -442,7 +449,7 @@ export class Server extends EventEmitter { const {csr, destination} = json; this.certificateProvider.processCertificateSigningRequest( csr, - connectionParameters.os, + clientData.os, destination, ); } @@ -459,13 +466,12 @@ export class Server extends EventEmitter { return null; } - addConnection(conn: ReactiveSocket, queryString: string): Client { - const query = JSON.parse(queryString); + addConnection(conn: ReactiveSocket, query: ClientQuery): Client { invariant(query, 'expected query'); - this.app.logger.warn(`Device connected: ${queryString}`, 'connection'); - const id = `${query.app}-${query.os}-${query.device}`; + this.app.logger.warn(`Device connected: ${id}`, 'connection'); + const client = new Client(this.app, id, query, conn); const info = { @@ -518,3 +524,34 @@ export class Server extends EventEmitter { } } } + +class ConnectionTracker { + timeWindowMillis = 20 * 1000; + connectionProblemThreshold = 4; + + // "${device}.${app}" -> [timestamp1, timestamp2...] + connectionAttempts: Map> = new Map(); + logger: Logger; + + constructor(logger: Logger) { + this.logger = logger; + } + + logConnectionAttempt(client: ClientQuery) { + const key = `${client.os}-${client.device}-${client.app}`; + const time = Date.now(); + var entry = this.connectionAttempts.get(key) || []; + entry.push(time); + entry = entry.filter(t => t >= time - this.timeWindowMillis); + + this.connectionAttempts.set(key, entry); + if (entry.length >= this.connectionProblemThreshold) { + this.logger.error( + `Connection loop detected with ${key}. Connected ${ + entry.length + } times in ${(time - entry[0]) / 1000}s.`, + 'ConnectionTracker', + ); + } + } +} diff --git a/src/utils/CertificateProvider.js b/src/utils/CertificateProvider.js index 93b5d980e..c109ba0e6 100644 --- a/src/utils/CertificateProvider.js +++ b/src/utils/CertificateProvider.js @@ -33,6 +33,7 @@ const minCertExpiryWindowSeconds = 24 * 60 * 60; const appNotDebuggableRegex = /debuggable/; const allowedAppNameRegex = /^[a-zA-Z0-9.\-]+$/; const allowedAppDirectoryRegex = /^\/[ a-zA-Z0-9.\-\/]+$/; +const logTag = 'CertificateProvider'; export type SecureServerConfig = {| key: Buffer, @@ -124,7 +125,7 @@ export default class CertificateProvider { } generateClientCertificate(csr: string): Promise { - this.logger.warn('Creating new client cert', 'CertificateProvider'); + this.logger.warn('Creating new client cert', logTag); const csrFile = this.writeToTempFile(csr); // Create a certificate for the client, using the details in the CSR. return openssl('x509', { @@ -145,40 +146,66 @@ export default class CertificateProvider { contents: string, csr: string, os: string, - ) { + ): Promise { if (os === 'Android') { - this.extractAppNameFromCSR(csr).then(app => { - const client = adb.createClient(); - client.listDevices().then((devices: Array<{id: string}>) => { - devices.forEach(d => - // To find out which device requested the cert, search them - // all for a matching csr file. - // It's not important to keep these secret from other apps. - // Just need to make sure each app can find it's own one. - this.androidDeviceHasMatchingCSR(destination, d.id, app, csr) - .catch(e => - this.logger.error( - `Unable to check for matching CSR in ${d.id}:${app}`, - 'CertificateProvider', - ), - ) - .then(isMatch => { - if (isMatch) { - this.pushFileToAndroidDevice( - d.id, - app, - destination + filename, - contents, - ); - } - }), - ); - }); - }); + const appNamePromise = this.extractAppNameFromCSR(csr); + const deviceIdPromise = appNamePromise.then(app => + this.getTargetDeviceId(app, destination, csr), + ); + return Promise.all([deviceIdPromise, appNamePromise]).then( + ([deviceId, appName]) => + this.pushFileToAndroidDevice( + deviceId, + appName, + destination + filename, + contents, + ), + ); } if (os === 'iOS') { fs.writeFileSync(destination + filename, contents); + return Promise.resolve(); } + return Promise.reject(new Error(`Unsupported device os: ${os}`)); + } + + getTargetDeviceId( + appName: string, + deviceCsrFilePath: string, + csr: string, + ): Promise { + const client = adb.createClient(); + return client.listDevices().then((devices: Array<{id: string}>) => { + const deviceMatchList = devices.map(device => + // To find out which device requested the cert, search them + // all for a matching csr file. + // It's not important to keep these secret from other apps. + // Just need to make sure each app can find it's own one. + this.androidDeviceHasMatchingCSR( + deviceCsrFilePath, + device.id, + appName, + csr, + ) + .then(isMatch => { + return {id: device.id, isMatch}; + }) + .catch(e => { + this.logger.error( + `Unable to check for matching CSR in ${device.id}:${appName}`, + logTag, + ); + return {id: device.id, isMatch: false}; + }), + ); + return Promise.all(deviceMatchList).then(devices => { + const matchingIds = devices.filter(m => m.isMatch).map(m => m.id); + if (matchingIds.length == 0) { + throw new Error(`No matching device found for app: ${appName}`); + } + return matchingIds[0]; + }); + }); } androidDeviceHasMatchingCSR( @@ -191,14 +218,19 @@ export default class CertificateProvider { deviceId, processName, `cat ${directory + csrFileName}`, - ).then(deviceCsr => { - return ( - deviceCsr - .toString() - .replace(/\r/g, '') - .trim() === csr.replace(/\r/g, '').trim() - ); - }); + ) + .then(deviceCsr => { + return ( + deviceCsr + .toString() + .replace(/\r/g, '') + .trim() === csr.replace(/\r/g, '').trim() + ); + }) + .catch(err => { + this.logger.error(err, logTag); + return false; + }); } pushFileToAndroidDevice( @@ -207,10 +239,7 @@ export default class CertificateProvider { filename: string, contents: string, ): Promise { - this.logger.warn( - `Deploying sonar certificate to ${deviceId}:${app}`, - 'CertificateProvider', - ); + this.logger.warn(`Deploying ${filename} to ${deviceId}:${app}`, logTag); return this.executeCommandOnAndroid( deviceId, app, @@ -245,6 +274,13 @@ export default class CertificateProvider { throw e; } return output; + }) + .catch(err => { + this.logger.error( + `Error executing command on android device ${deviceId}:${user}. Command: ${command}`, + logTag, + ); + this.logger.error(err, logTag); }); } @@ -306,10 +342,7 @@ export default class CertificateProvider { }) .then(output => undefined) .catch(e => { - this.logger.warn( - `Certificate will expire soon: ${filename}`, - 'CertificateProvider', - ); + this.logger.warn(`Certificate will expire soon: ${filename}`, logTag); throw e; }); } @@ -331,7 +364,7 @@ export default class CertificateProvider { if (!fs.existsSync(getFilePath(''))) { fs.mkdirSync(getFilePath('')); } - this.logger.info('Generating new CA', 'CertificateProvider'); + this.logger.info('Generating new CA', logTag); return openssl('genrsa', {out: caKey, '2048': false}) .then(_ => openssl('req', { @@ -364,7 +397,7 @@ export default class CertificateProvider { generateServerCertificate(): Promise { return this.ensureCertificateAuthorityExists() .then(_ => { - this.logger.warn('Creating new server cert', 'CertificateProvider'); + this.logger.warn('Creating new server cert', logTag); }) .then(_ => openssl('genrsa', {out: serverKey, '2048': false})) .then(_ => diff --git a/src/utils/promise.js b/src/utils/promise.js new file mode 100644 index 000000000..e69de29bb diff --git a/xplat/Sonar/SonarWebSocketImpl.cpp b/xplat/Sonar/SonarWebSocketImpl.cpp index cc81efcc4..24c68faa9 100644 --- a/xplat/Sonar/SonarWebSocketImpl.cpp +++ b/xplat/Sonar/SonarWebSocketImpl.cpp @@ -127,7 +127,8 @@ void SonarWebSocketImpl::doCertificateExchange() { folly::SocketAddress address; parameters.payload = rsocket::Payload( - folly::toJson(folly::dynamic::object("os", deviceData_.os))); + folly::toJson(folly::dynamic::object("os", deviceData_.os)( + "device", deviceData_.device)("app", deviceData_.app))); address.setFromHostPort(deviceData_.host, insecurePort); connectionIsTrusted_ = false;