2015-02-11 22:50:13 +00:00
|
|
|
#!/usr/bin/env python
|
|
|
|
# Copyright 2014 The Chromium Authors. All rights reserved.
|
|
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
|
|
# found in the LICENSE file.
|
|
|
|
|
|
|
|
# Modified from go/bootstrap.py in Chromium infrastructure's repository to patch
|
|
|
|
# out everything but the core toolchain.
|
|
|
|
#
|
|
|
|
# https://chromium.googlesource.com/infra/infra/
|
|
|
|
|
|
|
|
"""Prepares a local hermetic Go installation.
|
|
|
|
|
|
|
|
- Downloads and unpacks the Go toolset in ../golang.
|
|
|
|
"""
|
|
|
|
|
|
|
|
import contextlib
|
|
|
|
import logging
|
|
|
|
import os
|
|
|
|
import platform
|
|
|
|
import shutil
|
|
|
|
import stat
|
|
|
|
import subprocess
|
|
|
|
import sys
|
|
|
|
import tarfile
|
|
|
|
import tempfile
|
|
|
|
import urllib
|
|
|
|
import zipfile
|
|
|
|
|
|
|
|
# TODO(vadimsh): Migrate to new golang.org/x/ paths once Golang moves to
|
|
|
|
# git completely.
|
|
|
|
|
|
|
|
LOGGER = logging.getLogger(__name__)
|
|
|
|
|
|
|
|
|
|
|
|
# /path/to/util/bot
|
|
|
|
ROOT = os.path.dirname(os.path.abspath(__file__))
|
|
|
|
|
|
|
|
# Where to install Go toolset to. GOROOT would be <TOOLSET_ROOT>/go.
|
|
|
|
TOOLSET_ROOT = os.path.join(os.path.dirname(ROOT), 'golang')
|
|
|
|
|
|
|
|
# Default workspace with infra go code.
|
|
|
|
WORKSPACE = os.path.join(ROOT, 'go')
|
|
|
|
|
|
|
|
# Platform depended suffix for executable files.
|
|
|
|
EXE_SFX = '.exe' if sys.platform == 'win32' else ''
|
|
|
|
|
|
|
|
# Pinned version of Go toolset to download.
|
Update tools.
Unfortunately, this requires partially reverting
https://boringssl-review.googlesource.com/31324. This is a mess.
While clang proper includes a fuzzer driver, Chromium doesn't use it.
Chromium builds exclusively with fuzzer-no-link and links to its own
copy of the fuzzer runtime[1]. As of [2], Chromium's clang (which we use
on bots) no longer includes the driver, so we must mimic them.
However, Chromium's setup is somewhat questionable because
fuzzer-no-link pulls in libclang_rt.fuzzer_no_main which still includes
most of libclang_rt.fuzzer, just not the one main function[3]. It
appears Chromium is actually linking two copies of
libclang_rt.fuzzer_no_main. Hopefully this mostly works out as Chromium's
clang and libFuzzer should be relatively aligned, but it's not a good
assumption for our build, which can take other Clangs too.
Thus, if you pass -DFUZZ=1 as-is, we will assume you are using a
"normal" Clang with all its relevant runtimes intact. If, however, you
are using Chromium clang, you must drop the matching libFuzzer where the
bots expected it and build with -DLIBFUZZER_FROM_DEPS=1.
This involves no changes to the bots because we never actually unwound
all the LIBFUZZER_FROM_DEPS bits before.
[1] https://cs.chromium.org/chromium/src/testing/libfuzzer/BUILD.gn?rcl=d21c49585f262e851e2984f96f52905782706325&l=14
[2] https://chromium.googlesource.com/chromium/src/+/c79bf2ea4cf65431dccb57cb2a44528c284645a1
[3] https://github.com/llvm-mirror/compiler-rt/blob/8ebc3668b07fc5cca6010265cd4795443f1c1bea/lib/fuzzer/CMakeLists.txt#L93-L107
https://github.com/llvm-mirror/compiler-rt/blob/8ebc3668b07fc5cca6010265cd4795443f1c1bea/lib/fuzzer/FuzzerMain.cpp
Change-Id: I946b3c821c3d7e6def7e07f1381f58241611ba3d
Reviewed-on: https://boringssl-review.googlesource.com/c/34184
Commit-Queue: David Benjamin <davidben@google.com>
Reviewed-by: Adam Langley <agl@google.com>
2019-01-07 21:13:04 +00:00
|
|
|
TOOLSET_VERSION = 'go1.11.4'
|
2015-02-11 22:50:13 +00:00
|
|
|
|
|
|
|
# Platform dependent portion of a download URL. See http://golang.org/dl/.
|
|
|
|
TOOLSET_VARIANTS = {
|
2015-09-30 19:27:32 +01:00
|
|
|
('darwin', 'x86-64'): 'darwin-amd64.tar.gz',
|
2015-02-11 22:50:13 +00:00
|
|
|
('linux2', 'x86-32'): 'linux-386.tar.gz',
|
|
|
|
('linux2', 'x86-64'): 'linux-amd64.tar.gz',
|
|
|
|
('win32', 'x86-32'): 'windows-386.zip',
|
|
|
|
('win32', 'x86-64'): 'windows-amd64.zip',
|
|
|
|
}
|
|
|
|
|
|
|
|
# Download URL root.
|
|
|
|
DOWNLOAD_URL_PREFIX = 'https://storage.googleapis.com/golang'
|
|
|
|
|
|
|
|
|
|
|
|
class Failure(Exception):
|
|
|
|
"""Bootstrap failed."""
|
|
|
|
|
|
|
|
|
|
|
|
def get_toolset_url():
|
|
|
|
"""URL of a platform specific Go toolset archive."""
|
|
|
|
# TODO(vadimsh): Support toolset for cross-compilation.
|
|
|
|
arch = {
|
|
|
|
'amd64': 'x86-64',
|
|
|
|
'x86_64': 'x86-64',
|
|
|
|
'i386': 'x86-32',
|
|
|
|
'x86': 'x86-32',
|
|
|
|
}.get(platform.machine().lower())
|
|
|
|
variant = TOOLSET_VARIANTS.get((sys.platform, arch))
|
|
|
|
if not variant:
|
|
|
|
# TODO(vadimsh): Compile go lang from source.
|
|
|
|
raise Failure('Unrecognized platform')
|
|
|
|
return '%s/%s.%s' % (DOWNLOAD_URL_PREFIX, TOOLSET_VERSION, variant)
|
|
|
|
|
|
|
|
|
|
|
|
def read_file(path):
|
|
|
|
"""Returns contents of a given file or None if not readable."""
|
|
|
|
assert isinstance(path, (list, tuple))
|
|
|
|
try:
|
|
|
|
with open(os.path.join(*path), 'r') as f:
|
|
|
|
return f.read()
|
|
|
|
except IOError:
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
|
|
|
def write_file(path, data):
|
|
|
|
"""Writes |data| to a file."""
|
|
|
|
assert isinstance(path, (list, tuple))
|
|
|
|
with open(os.path.join(*path), 'w') as f:
|
|
|
|
f.write(data)
|
|
|
|
|
|
|
|
|
|
|
|
def remove_directory(path):
|
|
|
|
"""Recursively removes a directory."""
|
|
|
|
assert isinstance(path, (list, tuple))
|
|
|
|
p = os.path.join(*path)
|
|
|
|
if not os.path.exists(p):
|
|
|
|
return
|
|
|
|
LOGGER.info('Removing %s', p)
|
|
|
|
# Crutch to remove read-only file (.git/* in particular) on Windows.
|
|
|
|
def onerror(func, path, _exc_info):
|
|
|
|
if not os.access(path, os.W_OK):
|
|
|
|
os.chmod(path, stat.S_IWUSR)
|
|
|
|
func(path)
|
|
|
|
else:
|
|
|
|
raise
|
|
|
|
shutil.rmtree(p, onerror=onerror if sys.platform == 'win32' else None)
|
|
|
|
|
|
|
|
|
|
|
|
def install_toolset(toolset_root, url):
|
|
|
|
"""Downloads and installs Go toolset.
|
|
|
|
|
|
|
|
GOROOT would be <toolset_root>/go/.
|
|
|
|
"""
|
|
|
|
if not os.path.exists(toolset_root):
|
|
|
|
os.makedirs(toolset_root)
|
|
|
|
pkg_path = os.path.join(toolset_root, url[url.rfind('/')+1:])
|
|
|
|
|
|
|
|
LOGGER.info('Downloading %s...', url)
|
|
|
|
download_file(url, pkg_path)
|
|
|
|
|
|
|
|
LOGGER.info('Extracting...')
|
|
|
|
if pkg_path.endswith('.zip'):
|
|
|
|
with zipfile.ZipFile(pkg_path, 'r') as f:
|
|
|
|
f.extractall(toolset_root)
|
|
|
|
elif pkg_path.endswith('.tar.gz'):
|
|
|
|
with tarfile.open(pkg_path, 'r:gz') as f:
|
|
|
|
f.extractall(toolset_root)
|
|
|
|
else:
|
|
|
|
raise Failure('Unrecognized archive format')
|
|
|
|
|
|
|
|
LOGGER.info('Validating...')
|
|
|
|
if not check_hello_world(toolset_root):
|
|
|
|
raise Failure('Something is not right, test program doesn\'t work')
|
|
|
|
|
|
|
|
|
|
|
|
def download_file(url, path):
|
|
|
|
"""Fetches |url| to |path|."""
|
|
|
|
last_progress = [0]
|
|
|
|
def report(a, b, c):
|
|
|
|
progress = int(a * b * 100.0 / c)
|
|
|
|
if progress != last_progress[0]:
|
|
|
|
print >> sys.stderr, 'Downloading... %d%%' % progress
|
|
|
|
last_progress[0] = progress
|
|
|
|
# TODO(vadimsh): Use something less crippled, something that validates SSL.
|
|
|
|
urllib.urlretrieve(url, path, reporthook=report)
|
|
|
|
|
|
|
|
|
|
|
|
@contextlib.contextmanager
|
|
|
|
def temp_dir(path):
|
|
|
|
"""Creates a temporary directory, then deletes it."""
|
|
|
|
tmp = tempfile.mkdtemp(dir=path)
|
|
|
|
try:
|
|
|
|
yield tmp
|
|
|
|
finally:
|
|
|
|
remove_directory([tmp])
|
|
|
|
|
|
|
|
|
|
|
|
def check_hello_world(toolset_root):
|
|
|
|
"""Compiles and runs 'hello world' program to verify that toolset works."""
|
|
|
|
with temp_dir(toolset_root) as tmp:
|
|
|
|
path = os.path.join(tmp, 'hello.go')
|
|
|
|
write_file([path], r"""
|
|
|
|
package main
|
|
|
|
func main() { println("hello, world\n") }
|
|
|
|
""")
|
|
|
|
out = subprocess.check_output(
|
|
|
|
[get_go_exe(toolset_root), 'run', path],
|
|
|
|
env=get_go_environ(toolset_root, tmp),
|
|
|
|
stderr=subprocess.STDOUT)
|
|
|
|
if out.strip() != 'hello, world':
|
|
|
|
LOGGER.error('Failed to run sample program:\n%s', out)
|
|
|
|
return False
|
|
|
|
return True
|
|
|
|
|
|
|
|
|
|
|
|
def ensure_toolset_installed(toolset_root):
|
|
|
|
"""Installs or updates Go toolset if necessary.
|
|
|
|
|
|
|
|
Returns True if new toolset was installed.
|
|
|
|
"""
|
|
|
|
installed = read_file([toolset_root, 'INSTALLED_TOOLSET'])
|
|
|
|
available = get_toolset_url()
|
|
|
|
if installed == available:
|
|
|
|
LOGGER.debug('Go toolset is up-to-date: %s', TOOLSET_VERSION)
|
|
|
|
return False
|
|
|
|
|
|
|
|
LOGGER.info('Installing Go toolset.')
|
|
|
|
LOGGER.info(' Old toolset is %s', installed)
|
|
|
|
LOGGER.info(' New toolset is %s', available)
|
|
|
|
remove_directory([toolset_root])
|
|
|
|
install_toolset(toolset_root, available)
|
|
|
|
LOGGER.info('Go toolset installed: %s', TOOLSET_VERSION)
|
|
|
|
write_file([toolset_root, 'INSTALLED_TOOLSET'], available)
|
|
|
|
return True
|
|
|
|
|
|
|
|
|
|
|
|
def get_go_environ(
|
|
|
|
toolset_root,
|
|
|
|
workspace=None):
|
|
|
|
"""Returns a copy of os.environ with added GO* environment variables.
|
|
|
|
|
|
|
|
Overrides GOROOT, GOPATH and GOBIN. Keeps everything else. Idempotent.
|
|
|
|
|
|
|
|
Args:
|
|
|
|
toolset_root: GOROOT would be <toolset_root>/go.
|
|
|
|
workspace: main workspace directory or None if compiling in GOROOT.
|
|
|
|
"""
|
|
|
|
env = os.environ.copy()
|
|
|
|
env['GOROOT'] = os.path.join(toolset_root, 'go')
|
|
|
|
if workspace:
|
|
|
|
env['GOBIN'] = os.path.join(workspace, 'bin')
|
|
|
|
else:
|
|
|
|
env.pop('GOBIN', None)
|
|
|
|
|
|
|
|
all_go_paths = []
|
|
|
|
if workspace:
|
|
|
|
all_go_paths.append(workspace)
|
|
|
|
env['GOPATH'] = os.pathsep.join(all_go_paths)
|
|
|
|
|
|
|
|
# New PATH entries.
|
|
|
|
paths_to_add = [
|
|
|
|
os.path.join(env['GOROOT'], 'bin'),
|
|
|
|
env.get('GOBIN'),
|
|
|
|
]
|
|
|
|
|
|
|
|
# Make sure not to add duplicates entries to PATH over and over again when
|
|
|
|
# get_go_environ is invoked multiple times.
|
|
|
|
path = env['PATH'].split(os.pathsep)
|
|
|
|
paths_to_add = [p for p in paths_to_add if p and p not in path]
|
|
|
|
env['PATH'] = os.pathsep.join(paths_to_add + path)
|
|
|
|
|
|
|
|
return env
|
|
|
|
|
|
|
|
|
|
|
|
def get_go_exe(toolset_root):
|
|
|
|
"""Returns path to go executable."""
|
|
|
|
return os.path.join(toolset_root, 'go', 'bin', 'go' + EXE_SFX)
|
|
|
|
|
|
|
|
|
|
|
|
def bootstrap(logging_level):
|
|
|
|
"""Installs all dependencies in default locations.
|
|
|
|
|
|
|
|
Supposed to be called at the beginning of some script (it modifies logger).
|
|
|
|
|
|
|
|
Args:
|
|
|
|
logging_level: logging level of bootstrap process.
|
|
|
|
"""
|
|
|
|
logging.basicConfig()
|
|
|
|
LOGGER.setLevel(logging_level)
|
|
|
|
ensure_toolset_installed(TOOLSET_ROOT)
|
|
|
|
|
|
|
|
|
|
|
|
def prepare_go_environ():
|
|
|
|
"""Returns dict with environment variables to set to use Go toolset.
|
|
|
|
|
|
|
|
Installs or updates the toolset if necessary.
|
|
|
|
"""
|
|
|
|
bootstrap(logging.INFO)
|
|
|
|
return get_go_environ(TOOLSET_ROOT, WORKSPACE)
|
|
|
|
|
|
|
|
|
|
|
|
def find_executable(name, workspaces):
|
|
|
|
"""Returns full path to an executable in some bin/ (in GOROOT or GOBIN)."""
|
|
|
|
basename = name
|
|
|
|
if EXE_SFX and basename.endswith(EXE_SFX):
|
|
|
|
basename = basename[:-len(EXE_SFX)]
|
|
|
|
roots = [os.path.join(TOOLSET_ROOT, 'go', 'bin')]
|
|
|
|
for path in workspaces:
|
|
|
|
roots.extend([
|
|
|
|
os.path.join(path, 'bin'),
|
|
|
|
])
|
|
|
|
for root in roots:
|
|
|
|
full_path = os.path.join(root, basename + EXE_SFX)
|
|
|
|
if os.path.exists(full_path):
|
|
|
|
return full_path
|
|
|
|
return name
|
|
|
|
|
|
|
|
|
|
|
|
def main(args):
|
|
|
|
if args:
|
|
|
|
print >> sys.stderr, sys.modules[__name__].__doc__,
|
|
|
|
return 2
|
|
|
|
bootstrap(logging.DEBUG)
|
|
|
|
return 0
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
sys.exit(main(sys.argv[1:]))
|