chore: try to setup CI instrumentation tests

.github/workflows/build.yml

@@ -143,9 +143,73 @@ jobs:
         if: ${{ always() }}
         uses: actions/upload-artifact@v3
         with:
-          name: test-reports
+          name: test-results-unit-tests
           path: '**/build/reports/tests/'
 
+  instrumentation_test_debug:
+    runs-on: macos-latest
+    timeout-minutes: 120
+
+    strategy:
+      fail-fast: false
+    env:
+      TERM: dumb
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          java-version: '17'
+          distribution: 'temurin'
+      - name: Change Gradle wrapper permissions
+        run: chmod +x ./gradlew
+      - name: Restore cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.gradle/caches
+            ~/.gradle/wrapper
+          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }}
+          restore-keys: |
+            ${{ runner.os }}-gradle-
+      - name: Run tests
+        uses: reactivecircus/android-emulator-runner@v2
+        with:
+          api-level: 33
+          arch: arm64-v8a
+          channel: stable
+          target: google_apis
+          profile: pixel_6
+          avd-name: macOs-avd-google_apis-arm64-v8a-33
+          script: ./.tools/run_instrumentation_tests.sh --log-file=logcat.txt --run-affected --affected-base-ref=$BASE_REF
+        env:
+          IDE_INSTRUMENTATION_SETUP_BUILDTOOLS: true
+          ANDROIDIDE_TEST: true
+          IDE_SIGNING_ALIAS: ${{ secrets.IDE_SIGNING_ALIAS }}
+          IDE_SIGNING_AUTH_PASS: ${{ secrets.IDE_SIGNING_AUTH_PASS }}
+          IDE_SIGNING_AUTH_USER: ${{ secrets.IDE_SIGNING_AUTH_USER }}
+          IDE_SIGNING_KEY_PASS: ${{ secrets.IDE_SIGNING_KEY_PASS }}
+          IDE_SIGNING_STORE_PASS: ${{ secrets.IDE_SIGNING_STORE_PASS }}
+          IDE_SIGNING_URL: ${{ secrets.IDE_SIGNING_URL }}
+          IDE_SIGNING_KEY_BIN: ${{ secrets.IDE_SIGNING_KEY_BIN }}
+      - name: Upload logs
+        if: always()
+        uses: actions/upload-artifact@v2
+        with:
+          name: emulator-logs
+          path: logcat.txt
+      - name: Upload test results
+        if: always()
+        uses: actions/upload-artifact@v2
+        with:
+          name: test-results-instrumentation-tests
+          path: |
+            **/build/reports/*
+            **/build/outputs/*/connected/*
+
   unit_test_tooling_api:
     name: Build and test Tooling API
     runs-on: ubuntu-latest
@@ -191,7 +255,7 @@ jobs:
         if: ${{ always() }}
         uses: actions/upload-artifact@v3
         with:
-          name: tooling-api-test-reports
+          name: test-results-tooling-api-tests
           path: '**/build/reports/tests/'
 
   publish:

.idea/androidTestResultsUserPreferences.xml

@@ -70,6 +70,19 @@
             </AndroidTestResultsTableState>
           </value>
         </entry>
+        <entry key="1679989688">
+          <value>
+            <AndroidTestResultsTableState>
+              <option name="preferredColumnWidths">
+                <map>
+                  <entry key="Duration" value="90" />
+                  <entry key="Tests" value="360" />
+                  <entry key="realme RMX3312" value="120" />
+                </map>
+              </option>
+            </AndroidTestResultsTableState>
+          </value>
+        </entry>
       </map>
     </option>
   </component>

.tools/run_instrumentation_tests.sh

@@ -0,0 +1,129 @@
+#!/bin/bash
+
+#
+#  This file is part of AndroidIDE.
+#
+#  AndroidIDE is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  AndroidIDE is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#   along with AndroidIDE.  If not, see <https://www.gnu.org/licenses/>.
+#
+
+# Fail on error and print out commands
+set -ex
+
+# By default we don't shard
+SHARD_COUNT=0
+SHARD_INDEX=0
+# By default we don't log
+LOG_FILE=""
+# By default we run tests on device
+DEVICE=true
+
+# The task for installing the tools in the emulator
+INSTALLER_TEST_TASK_NAME=":app:connectedArm64-v8aDebugAndroidTest"
+# Fully qualified name of the class which install installs the build tools
+INSTALLER_TEST_CLASS_NAME="com.itsaky.androidide.BuildToolsInstallerTest"
+
+# Common Gradle invocation arguments
+GRADLE_INVOCATION_ARGS="--scan --continue --no-configuration-cache --stacktrace"
+
+# Parse parameters
+for i in "$@"; do
+  case $i in
+  --shard-count=*)
+    SHARD_COUNT="${i#*=}"
+    shift
+    ;;
+  --unit-tests)
+    DEVICE=false
+    shift
+    ;;
+  --shard-index=*)
+    SHARD_INDEX="${i#*=}"
+    shift
+    ;;
+  --log-file=*)
+    LOG_FILE="${i#*=}"
+    shift
+    ;;
+  --run-affected)
+    RUN_AFFECTED=true
+    shift
+    ;;
+  --run-flaky-tests)
+    RUN_FLAKY=true
+    shift
+    ;;
+  --affected-base-ref=*)
+    BASE_REF="${i#*=}"
+    shift
+    ;;
+  *)
+    echo "Unknown option"
+    exit 1
+    ;;
+  esac
+done
+
+# Start logcat if we have a file to log to
+if [[ -n "$LOG_FILE" ]]; then
+  adb logcat >"$LOG_FILE" &
+fi
+
+FILTER_OPTS=""
+# Filter out flaky tests if we're not set to run them
+if [[ -z "$RUN_FLAKY" ]]; then
+  FILTER_OPTS="$FILTER_OPTS -Pandroid.testInstrumentationRunnerArguments.notAnnotation=androidx.test.filters.FlakyTest"
+fi
+
+# If we're set to only run affected test, update the Gradle task
+if [[ -n "$RUN_AFFECTED" ]]; then
+  if [ "$DEVICE" = true ]; then
+    TASK="runAffectedAndroidTests"
+  else
+    TASK="runAffectedUnitTests"
+  fi
+  TASK="$TASK -Paffected_module_detector.enable"
+
+  # If we have a base branch set, add the Gradle property
+  if [[ -n "$BASE_REF" ]]; then
+    TASK="$TASK -Paffected_base_ref=$BASE_REF"
+  fi
+fi
+
+# If we don't have a task yet, use the defaults
+if [[ -z "$TASK" ]]; then
+  if [ "$DEVICE" = true ]; then
+    TASK="connectedCheck"
+  else
+    TASK="testDebug"
+  fi
+fi
+
+SHARD_OPTS=""
+if [ "$SHARD_COUNT" -gt "0" ]; then
+  # If we have a shard count value, create the necessary Gradle property args.
+  # We assume that SHARD_INDEX has been set too
+  SHARD_OPTS="$SHARD_OPTS -Pandroid.testInstrumentationRunnerArguments.numShards=$SHARD_COUNT"
+  SHARD_OPTS="$SHARD_OPTS -Pandroid.testInstrumentationRunnerArguments.shardIndex=$SHARD_INDEX"
+fi
+
+# Run the build tools installer test
+# shellcheck disable=SC2086
+./gradlew \
+  $GRADLE_INVOCATION_ARGS \
+  $INSTALLER_TEST_TASK_NAME \
+  -Pandroid.testInstrumentationRunnerArguments.class=$INSTALLER_TEST_CLASS_NAME
+
+# Run the actual tests
+# shellcheck disable=SC2086
+./gradlew $GRADLE_INVOCATION_ARGS $TASK $FILTER_OPTS $SHARD_OPTS

app/src/androidTest/java/com/itsaky/androidide/BuildToolsInstallerTest.kt

@@ -0,0 +1,103 @@
+/*
+ *  This file is part of AndroidIDE.
+ *
+ *  AndroidIDE is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  AndroidIDE is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *   along with AndroidIDE.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.itsaky.androidide
+
+import android.util.Log
+import androidx.test.ext.junit.runners.AndroidJUnit4
+import com.google.common.truth.Truth.assertThat
+import com.itsaky.androidide.R.string
+import com.itsaky.androidide.shell.executeProcessAsync
+import com.itsaky.androidide.testing.android.clickAndWaitForWindowUpdate
+import com.itsaky.androidide.testing.android.findObjectWithText
+import com.itsaky.androidide.testing.android.hasObjectWithText
+import com.itsaky.androidide.testing.android.launchAndroidIDE
+import com.itsaky.androidide.utils.Environment
+import org.junit.Test
+import org.junit.runner.RunWith
+
+/**
+ * A special test case which is used to install the build tools on a test device. This is used to
+ * setup the test device before running further tests.
+ *
+ * This class should not define any other tests.
+ *
+ * @author Akash Yadav
+ */
+@RunWith(AndroidJUnit4::class)
+class BuildToolsInstallerTest {
+
+  companion object {
+    private const val TAG = "BuildToolsInstallerTest"
+  }
+
+  @Test
+  fun setupDevice() {
+    if (!System.getenv("IDE_INSTRUMENTATION_SETUP_BUILDTOOLS").toBoolean()) {
+      return
+    }
+
+    val device = launchAndroidIDE()
+
+    // Allow permissions
+    while (true) {
+      val obj = device.findObjectWithText("Allow")
+      obj?.clickAndWaitForWindowUpdate(device) ?: break
+    }
+
+    // Dismiss the 'AndroidIDE statistics' dialog
+    if (device.hasObjectWithText(string.title_androidide_statistics)
+      && device.hasObjectWithText(string.msg_androidide_statistics)
+    ) {
+      device.findObjectWithText(string.btn_opt_in).clickAndWaitForWindowUpdate(device)
+    }
+
+    // The 'Warning' dialog will be shown, no need to dismiss it.
+
+    // Start the installation process
+    execProcAndLogOutput("pkg upd")
+    execProcAndLogOutput("yes | pkg upg") // do not use -y
+    execProcAndLogOutput("yes | idesetup -c")
+
+    // Check if the tools were installed
+    assertThat(Environment.JAVA.exists()).isTrue()
+    assertThat(Environment.ANDROID_HOME.exists()).isTrue()
+
+    Thread.sleep(5000L)
+  }
+
+  private fun execProcAndLogOutput(cmd: String) {
+    val process = executeProcessAsync {
+      redirectErrorStream = true
+      workingDirectory = null
+      environment = Environment.getEnvironment()
+      command = listOf(Environment.SHELL.absolutePath, "-c", cmd).also {
+        Log.d(TAG, "setupDevice: Full command : ${it.joinToString(separator = " ")}")
+      }
+    }
+
+    logOutput(cmd, process)
+
+    process.waitFor()
+  }
+
+  private fun logOutput(cmd: String, process: Process) {
+    process.inputStream.bufferedReader().forEachLine {
+      Log.d(TAG, "setupDevice: '${cmd}': $it")
+    }
+  }
+}

testing/android/src/main/java/androidx/test/uiautomator/UiDeviceAccessor.java

@@ -0,0 +1,40 @@
+/*
+ *  This file is part of AndroidIDE.
+ *
+ *  AndroidIDE is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  AndroidIDE is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *   along with AndroidIDE.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package androidx.test.uiautomator;
+
+import android.app.UiAutomation;
+
+/**
+ * @author Akash Yadav
+ */
+public class UiDeviceAccessor {
+
+  /**
+   * Get the {@link UiAutomation} instance from the given {@link UiDevice}.
+   */
+  public static UiAutomation getUiAutomation(UiDevice device) {
+    return device.getUiAutomation();
+  }
+
+  /**
+   * Get the {@link InteractionController} instance from the given {@link UiDevice}.
+   */
+  public static InteractionController getInteractionController(UiDevice device) {
+    return device.getInteractionController();
+  }
+}

testing/android/src/main/java/com/itsaky/androidide/testing/android/uiAutomatorUtils.kt

@@ -18,6 +18,7 @@
 package com.itsaky.androidide.testing.android
 
 import android.app.Activity
+import android.app.UiAutomation
 import android.content.Intent
 import androidx.test.core.app.ApplicationProvider
 import androidx.test.platform.app.InstrumentationRegistry
@@ -25,6 +26,7 @@ import androidx.test.runner.lifecycle.ActivityLifecycleMonitorRegistry
 import androidx.test.runner.lifecycle.Stage
 import androidx.test.uiautomator.By
 import androidx.test.uiautomator.UiDevice
+import androidx.test.uiautomator.UiDeviceAccessor
 import androidx.test.uiautomator.Until
 import com.google.common.truth.Truth.assertThat
 import com.itsaky.androidide.app.BaseApplication
@@ -91,3 +93,9 @@ fun <T : Activity> getActivityInStage(stage: Stage): T? {
 
   return activity
 }
+
+/**
+ * @see UiDeviceAccessor.getUiAutomation
+ */
+val UiDevice.uiAutomation: UiAutomation
+  get() = UiDeviceAccessor.getUiAutomation(this)

testing/android/src/main/java/com/itsaky/androidide/testing/android/uiDeviceUtils.kt

@@ -17,12 +17,18 @@
 
 package com.itsaky.androidide.testing.android
 
+import android.view.KeyCharacterMap
+import android.view.KeyEvent
 import androidx.annotation.StringRes
 import androidx.test.uiautomator.By
 import androidx.test.uiautomator.UiDevice
 import androidx.test.uiautomator.UiObject2
 import com.itsaky.androidide.buildinfo.BuildInfo
 
+private val keyCharMap by lazy {
+  KeyCharacterMap.load(KeyCharacterMap.VIRTUAL_KEYBOARD)
+}
+
 /**
  * @see UiDevice.waitForWindowUpdate
  */
@@ -52,3 +58,16 @@ fun UiDevice.findObjectWithText(text: String): UiObject2? =
  */
 fun UiDevice.findObjectWithText(@StringRes text: Int): UiObject2? =
   findObject(By.text(stringRes(text)))
+
+/**
+ * Sends key events to this [UiDevice] to type the given [text].
+ */
+fun UiDevice.sendKeyEvents(text: String) {
+  waitForIdle()
+  val keyEvents = keyCharMap.getEvents(text.toCharArray())
+  for (keyEvent in keyEvents) {
+    uiAutomation.injectInputEvent(keyEvent, true)
+//    pressKeyCode(keyEvent.keyCode, keyEvent.metaState)
+    waitForIdle()
+  }
+}