misc/android,cmd/dist: move $GOROOT copying to the exec wrapper

To run the standard library tests on Android, the androidtest.bash
script copies GOROOT to the device. Move that logic to the android
exec wrapper, thereby making androidtest.bash obsolete.

Apart from making Android less special, the sharded builder
infrastructure should now be able to run (emulated) Android builders
and trybots without special treatment.

Updates #23824

Change-Id: I41591fea9a15b38c6dcf84046ea57f1e9165eaa5
Reviewed-on: https://go-review.googlesource.com/c/163619
Run-TryBot: Elias Naur <mail@eliasnaur.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>

misc/android/README

@@ -6,18 +6,18 @@ mobile subrepository:
 
 	https://github.com/golang/mobile
 
-To run the standard library tests, see androidtest.bash. Run it as
+To run the standard library tests, enable Cgo and use an appropriate
+C compiler from the Android NDK. For example,
 
-	CC_FOR_TARGET=$STANDALONE_NDK_PATH/bin/clang GOARCH=arm64 ./androidtest.bash
-
-To create a standalone android NDK tool chain, follow the instructions on
-
-	https://developer.android.com/ndk/guides/standalone_toolchain
+	CGO_ENABLED=1 \
+	GOOS=android \
+	GOARCH=arm64 \
+	CC=$NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android21-clang \
+	./all.bash
 
 To run tests on the Android device, add the bin directory to PATH so the
 go tool can find the go_android_$GOARCH_exec wrapper generated by
-androidtest.bash. Then, use the same GOARCH as when androidtest.bash ran
-and set GOOS to android. For example, to run the go1 benchmarks
+make.bash. For example, to run the go1 benchmarks
 
 	export PATH=$GOROOT/bin:$PATH
 	cd $GOROOT/test/bench/go1/

misc/android/cleaner.go

-// Copyright 2015 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build ignore
-
-// Cleaner removes anything from /data/local/tmp/goroot not on a builtin list.
-// Used by androidtest.bash.
-package main
-
-import (
-	"log"
-	"os"
-	"path/filepath"
-	"strings"
-)
-
-func main() {
-	const goroot = "/data/local/tmp/goroot"
-	expect := make(map[string]bool)
-	for _, f := range strings.Split(files, "\n") {
-		expect[filepath.Join(goroot, f)] = true
-	}
-
-	err := filepath.Walk(goroot, func(path string, info os.FileInfo, err error) error {
-		if expect[path] {
-			return nil
-		}
-		log.Printf("removing %s", path)
-		if err := os.RemoveAll(path); err != nil {
-			return err
-		}
-		if info.IsDir() {
-			return filepath.SkipDir
-		}
-		return nil
-	})
-	if err != nil {
-		log.Fatal(err)
-	}
-}

misc/android/go_android_exec.go

@@ -13,6 +13,7 @@ import (
 	"fmt"
 	"go/build"
 	"io"
+	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
@@ -78,6 +79,8 @@ func main() {
 	deviceCwd := filepath.Join(deviceGoroot, subdir)
 	if !inGoRoot {
 		deviceCwd = filepath.Join(deviceGopath, subdir)
+	} else {
+		adbSyncGoroot()
 	}
 
 	// Binary names can conflict.
@@ -164,3 +167,46 @@ func subdir() (pkgpath string, underGoRoot bool) {
 		cwd, runtime.GOROOT(), build.Default.GOPATH)
 	return "", false
 }
+
+// adbSyncGoroot ensures that files necessary for testing the Go standard
+// packages are present on the attached device.
+func adbSyncGoroot() {
+	// Also known by cmd/dist. The bootstrap command deletes the file.
+	statPath := filepath.Join(os.TempDir(), "go_android_exec-adb-sync-status")
+	stat, err := os.OpenFile(statPath, os.O_CREATE|os.O_RDWR, 0666)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer stat.Close()
+	// Serialize check and syncing.
+	if err := syscall.Flock(int(stat.Fd()), syscall.LOCK_EX); err != nil {
+		log.Fatal(err)
+	}
+	s, err := ioutil.ReadAll(stat)
+	if err != nil {
+		log.Fatal(err)
+	}
+	if string(s) == "done" {
+		return
+	}
+	devRoot := "/data/local/tmp/goroot"
+	run("shell", "rm", "-rf", devRoot)
+	run("shell", "mkdir", "-p", devRoot+"/pkg")
+	goroot := runtime.GOROOT()
+	goCmd := filepath.Join(goroot, "bin", "go")
+	runtimea, err := exec.Command(goCmd, "list", "-f", "{{.Target}}", "runtime").Output()
+	if err != nil {
+		log.Fatal(err)
+	}
+	pkgdir := filepath.Dir(string(runtimea))
+	if pkgdir == "" {
+		log.Fatal("could not find android pkg dir")
+	}
+	for _, dir := range []string{"src", "test", "lib"} {
+		run("push", filepath.Join(goroot, dir), filepath.Join(devRoot))
+	}
+	run("push", filepath.Join(pkgdir), filepath.Join(devRoot, "pkg/"))
+	if _, err := stat.Write([]byte("done")); err != nil {
+		log.Fatal(err)
+	}
+}

src/androidtest.bash

@@ -4,8 +4,6 @@
 # license that can be found in the LICENSE file.
 
 # For testing Android.
-# The compiler runs locally, then a copy of the GOROOT is pushed to a
-# target device using adb, and the tests are run there.
 
 set -e
 ulimit -c 0 # no core files
@@ -31,55 +29,9 @@ fi
 export CGO_ENABLED=1
 unset GOBIN
 
-# Do the build first, so we can build the 'cleaner' binary.
-# Also lets us fail early before the (slow) adb push if the build is broken.
-. ./make.bash --no-banner
 export GOROOT=$(dirname $(pwd))
+# Put the exec wrapper into PATH
 export PATH=$GOROOT/bin:$PATH
 
-export pkgdir=$(dirname $(go list -f '{{.Target}}' runtime))
-if [ "$pkgdir" = "" ]; then
-	echo "could not find android pkg dir" 1>&2
-	exit 1
-fi
-
-export ANDROID_TEST_DIR=/tmp/androidtest-$$
-
-function cleanup() {
-	rm -rf ${ANDROID_TEST_DIR}
-}
-trap cleanup EXIT
-
-# Push GOROOT to target device.
-#
-# The adb sync command will sync either the /system or /data
-# directories of an android device from a similar directory
-# on the host. We copy the files required for running tests under
-# /data/local/tmp/goroot. The adb sync command does not follow
-# symlinks so we have to copy.
-export ANDROID_PRODUCT_OUT="${ANDROID_TEST_DIR}/out"
-FAKE_GOROOT=$ANDROID_PRODUCT_OUT/data/local/tmp/goroot
-mkdir -p $FAKE_GOROOT
-mkdir -p $FAKE_GOROOT/pkg
-cp -a "${GOROOT}/src" "${FAKE_GOROOT}/"
-cp -a "${GOROOT}/test" "${FAKE_GOROOT}/"
-cp -a "${GOROOT}/lib" "${FAKE_GOROOT}/"
-cp -a "${pkgdir}" "${FAKE_GOROOT}/pkg/"
-
-echo '# Syncing test files to android device'
-adb $GOANDROID_ADB_FLAGS shell mkdir -p /data/local/tmp/goroot
-time adb $GOANDROID_ADB_FLAGS sync data &> /dev/null
-
-export CLEANER=${ANDROID_TEST_DIR}/androidcleaner-$$
-cp ../misc/android/cleaner.go $CLEANER.go
-echo 'var files = `' >> $CLEANER.go
-(cd $ANDROID_PRODUCT_OUT/data/local/tmp/goroot; find . >> $CLEANER.go)
-echo '`' >> $CLEANER.go
-go build -o $CLEANER $CLEANER.go
-adb $GOANDROID_ADB_FLAGS push $CLEANER /data/local/tmp/cleaner
-adb $GOANDROID_ADB_FLAGS shell /data/local/tmp/cleaner
-
-echo ''
-
 # Run standard tests.
-bash run.bash --no-rebuild
+bash all.bash

src/cmd/dist/build.go

@@ -1366,7 +1366,11 @@ func cmdbootstrap() {
 	// Remove go_bootstrap now that we're done.
 	xremove(pathf("%s/go_bootstrap", tooldir))
 
-	// Build the exec wrapper if necessary.
+	if goos == "android" {
+		// Make sure the exec wrapper will sync a fresh $GOROOT to the device.
+		xremove(pathf("%s/go_android_exec-adb-sync-status", os.TempDir()))
+	}
+
 	if wrapperPath := wrapperPathFor(goos, goarch); wrapperPath != "" {
 		oldcc := os.Getenv("CC")
 		os.Setenv("GOOS", gohostos)