Refresh update_clang.py and download Windows Clang.

This is taken from Chromium and then pared down to remove unnecessary
bits. The Windows setup is somewhat more involved due to needing to copy
some DLL from Visual Studio.

Bug: 201
Change-Id: I0658f7a20ec4fdea007821d5ce331acd3cb494b2
Reviewed-on: https://boringssl-review.googlesource.com/20504
Commit-Queue: Steven Valdez <svaldez@google.com>
Reviewed-by: Steven Valdez <svaldez@google.com>
CQ-Verified: CQ bot account: commit-bot@chromium.org <commit-bot@chromium.org>

util/bot/update_clang.py

@@ -1,71 +1,203 @@
-# Copyright (c) 2015, Google Inc.
+#!/usr/bin/env python
-#
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
-# Permission to use, copy, modify, and/or distribute this software for any
+# Use of this source code is governed by a BSD-style license that can be
-# purpose with or without fee is hereby granted, provided that the above
+# found in the LICENSE file.
-# copyright notice and this permission notice appear in all copies.
-#
-# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
-# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
-# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
-# SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
-# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
-# OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
-# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
-import os.path
+"""This script is used to download prebuilt clang binaries."""
+
+import os
 import shutil
+import subprocess
+import stat
 import sys
 import tarfile
 import tempfile
-import urllib
+import time
+import urllib2
+
 
 # CLANG_REVISION and CLANG_SUB_REVISION determine the build of clang
 # to use. These should be synced with tools/clang/scripts/update.py in
 # Chromium.
-CLANG_REVISION = "305735"
+CLANG_REVISION = '310694'
-CLANG_SUB_REVISION = "2"
+CLANG_SUB_REVISION=2
 
 PACKAGE_VERSION = "%s-%s" % (CLANG_REVISION, CLANG_SUB_REVISION)
-LLVM_BUILD_DIR = os.path.join(os.path.dirname(__file__), "llvm-build")
-STAMP_FILE = os.path.join(LLVM_BUILD_DIR, "cr_build_revision")
 
-CDS_URL = "https://commondatastorage.googleapis.com/chromium-browser-clang"
+# Path constants. (All of these should be absolute paths.)
+THIS_DIR = os.path.abspath(os.path.dirname(__file__))
+LLVM_BUILD_DIR = os.path.join(THIS_DIR, 'llvm-build')
+STAMP_FILE = os.path.join(THIS_DIR, 'cr_build_revision')
 
-def DownloadFile(url, path):
+# URL for pre-built binaries.
-  """DownloadFile fetches |url| to |path|."""
+CDS_URL = os.environ.get('CDS_CLANG_BUCKET_OVERRIDE',
-  last_progress = [0]
+    'https://commondatastorage.googleapis.com/chromium-browser-clang')
-  def report(a, b, c):
-    progress = int(a * b * 100.0 / c)
-    if progress != last_progress[0]:
-      print >> sys.stderr, "Downloading... %d%%" % progress
-      last_progress[0] = progress
-  urllib.urlretrieve(url, path, reporthook=report)
 
-def main(args):
+# Bump after VC updates.
-  # For now, only download clang on Linux.
+DIA_DLL = {
-  if not sys.platform.startswith("linux"):
+  '2013': 'msdia120.dll',
+  '2015': 'msdia140.dll',
+  '2017': 'msdia140.dll',
+}
+
+
+def DownloadUrl(url, output_file):
+  """Download url into output_file."""
+  CHUNK_SIZE = 4096
+  TOTAL_DOTS = 10
+  num_retries = 3
+  retry_wait_s = 5  # Doubled at each retry.
+
+  while True:
+    try:
+      sys.stdout.write('Downloading %s ' % url)
+      sys.stdout.flush()
+      response = urllib2.urlopen(url)
+      total_size = int(response.info().getheader('Content-Length').strip())
+      bytes_done = 0
+      dots_printed = 0
+      while True:
+        chunk = response.read(CHUNK_SIZE)
+        if not chunk:
+          break
+        output_file.write(chunk)
+        bytes_done += len(chunk)
+        num_dots = TOTAL_DOTS * bytes_done / total_size
+        sys.stdout.write('.' * (num_dots - dots_printed))
+        sys.stdout.flush()
+        dots_printed = num_dots
+      if bytes_done != total_size:
+        raise urllib2.URLError("only got %d of %d bytes" %
+                               (bytes_done, total_size))
+      print ' Done.'
+      return
+    except urllib2.URLError as e:
+      sys.stdout.write('\n')
+      print e
+      if num_retries == 0 or isinstance(e, urllib2.HTTPError) and e.code == 404:
+        raise e
+      num_retries -= 1
+      print 'Retrying in %d s ...' % retry_wait_s
+      time.sleep(retry_wait_s)
+      retry_wait_s *= 2
+
+
+def EnsureDirExists(path):
+  if not os.path.exists(path):
+    print "Creating directory %s" % path
+    os.makedirs(path)
+
+
+def DownloadAndUnpack(url, output_dir):
+  with tempfile.TemporaryFile() as f:
+    DownloadUrl(url, f)
+    f.seek(0)
+    EnsureDirExists(output_dir)
+    tarfile.open(mode='r:gz', fileobj=f).extractall(path=output_dir)
+
+
+def ReadStampFile(path=STAMP_FILE):
+  """Return the contents of the stamp file, or '' if it doesn't exist."""
+  try:
+    with open(path, 'r') as f:
+      return f.read().rstrip()
+  except IOError:
+    return ''
+
+
+def WriteStampFile(s, path=STAMP_FILE):
+  """Write s to the stamp file."""
+  EnsureDirExists(os.path.dirname(path))
+  with open(path, 'w') as f:
+    f.write(s)
+    f.write('\n')
+
+
+def RmTree(dir):
+  """Delete dir."""
+  def ChmodAndRetry(func, path, _):
+    # Subversion can leave read-only files around.
+    if not os.access(path, os.W_OK):
+      os.chmod(path, stat.S_IWUSR)
+      return func(path)
+    raise
+
+  shutil.rmtree(dir, onerror=ChmodAndRetry)
+
+
+def CopyFile(src, dst):
+  """Copy a file from src to dst."""
+  print "Copying %s to %s" % (src, dst)
+  shutil.copy(src, dst)
+
+
+vs_version = None
+def GetVSVersion():
+  global vs_version
+  if vs_version:
+    return vs_version
+
+  # Try using the toolchain in depot_tools.
+  # This sets environment variables used by SelectVisualStudioVersion below.
+  sys.path.append(THIS_DIR)
+  import vs_toolchain
+  vs_toolchain.SetEnvironmentAndGetRuntimeDllDirs()
+
+  # Use gyp to find the MSVS installation, either in depot_tools as per above,
+  # or a system-wide installation otherwise.
+  sys.path.append(os.path.join(THIS_DIR, 'gyp', 'pylib'))
+  import gyp.MSVSVersion
+  vs_version = gyp.MSVSVersion.SelectVisualStudioVersion(
+      vs_toolchain.GetVisualStudioVersion())
+  return vs_version
+
+
+def CopyDiaDllTo(target_dir):
+  # This script always wants to use the 64-bit msdia*.dll.
+  dia_path = os.path.join(GetVSVersion().Path(), 'DIA SDK', 'bin', 'amd64')
+  dia_dll = os.path.join(dia_path, DIA_DLL[GetVSVersion().ShortName()])
+  CopyFile(dia_dll, target_dir)
+
+
+def UpdateClang():
+  cds_file = "clang-%s.tgz" %  PACKAGE_VERSION
+  if sys.platform == 'win32' or sys.platform == 'cygwin':
+    cds_full_url = CDS_URL + '/Win/' + cds_file
+  elif sys.platform.startswith('linux'):
+    cds_full_url = CDS_URL + '/Linux_x64/' + cds_file
+  else:
     return 0
 
-  if os.path.exists(STAMP_FILE):
+  print 'Updating Clang to %s...' % PACKAGE_VERSION
-    with open(STAMP_FILE) as f:
-      if f.read().strip() == PACKAGE_VERSION:
-        print >> sys.stderr, "Clang already at %s" % (PACKAGE_VERSION,)
-        return 0
 
+  if ReadStampFile() == PACKAGE_VERSION:
+    print 'Clang is already up to date.'
+    return 0
+
+  # Reset the stamp file in case the build is unsuccessful.
+  WriteStampFile('')
+
+  print 'Downloading prebuilt clang'
   if os.path.exists(LLVM_BUILD_DIR):
-    shutil.rmtree(LLVM_BUILD_DIR)
+    RmTree(LLVM_BUILD_DIR)
+  try:
+    DownloadAndUnpack(cds_full_url, LLVM_BUILD_DIR)
+    print 'clang %s unpacked' % PACKAGE_VERSION
+    if sys.platform == 'win32':
+      CopyDiaDllTo(os.path.join(LLVM_BUILD_DIR, 'bin'))
+    WriteStampFile(PACKAGE_VERSION)
+    return 0
+  except urllib2.URLError:
+    print 'Failed to download prebuilt clang %s' % cds_file
+    print 'Exiting.'
+    return 1
 
-  print >> sys.stderr, "Downloading Clang %s" % (PACKAGE_VERSION,)
-  cds_full_url = "%s/Linux_x64/clang-%s.tgz" % (CDS_URL, PACKAGE_VERSION)
-  with tempfile.NamedTemporaryFile() as temp:
-    DownloadFile(cds_full_url, temp.name)
-    with tarfile.open(temp.name, "r:gz") as tar_file:
-      tar_file.extractall(LLVM_BUILD_DIR)
 
-  with open(STAMP_FILE, "wb") as stamp_file:
+def main():
-    stamp_file.write(PACKAGE_VERSION)
+  # Don't buffer stdout, so that print statements are immediately flushed.
+  sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
+  return UpdateClang()
 
-  return 0
 
-if __name__ == "__main__":
+if __name__ == '__main__':
-  sys.exit(main(sys.argv[1:]))
+  sys.exit(main())

util/bot/vs_toolchain.py

@@ -73,6 +73,11 @@ def FindDepotTools():
   raise Exception("depot_tools not found!")
 
 
+def GetVisualStudioVersion():
+  """Return GYP_MSVS_VERSION of Visual Studio."""
+  return os.environ.get('GYP_MSVS_VERSION', TOOLCHAIN_VERSION)
+
+
 def Update():
   """Requests an update of the toolchain to the specific hashes we have at
   this revision. The update outputs a .json of the various configuration