Add hash of SDE tool for the bots.

1de4bdf147a8a3b93306ecc65cf1f15a8334f508a29ab0b3ee163443ce764d5e sde-external-7.58.0-2017-01-23-lin.tar.bz2

Also teach extract.py to handle symlinks and tar.bz2 files.

BUG=180

Change-Id: Iddce09169f077c8f1bef9bd0dd0e05605b17769a
Reviewed-on: https://boringssl-review.googlesource.com/16244
Reviewed-by: Adam Langley <agl@google.com>
Commit-Queue: David Benjamin <davidben@google.com>
CQ-Verified: CQ bot account: commit-bot@chromium.org <commit-bot@chromium.org>

util/bot/extract.py

@@ -37,30 +37,47 @@ def CheckedJoin(output, path):
   return os.path.join(output, path)
 
 
+class FileEntry(object):
+  def __init__(self, path, mode, fileobj):
+    self.path = path
+    self.mode = mode
+    self.fileobj = fileobj
+
+
+class SymlinkEntry(object):
+  def __init__(self, path, mode, target):
+    self.path = path
+    self.mode = mode
+    self.target = target
+
+
 def IterateZip(path):
   """
-  IterateZip opens the zip file at path and returns a generator of
+  IterateZip opens the zip file at path and returns a generator of entry objects
-  (filename, mode, fileobj) tuples for each file in it.
+  for each file in it.
   """
   with zipfile.ZipFile(path, 'r') as zip_file:
     for info in zip_file.infolist():
       if info.filename.endswith('/'):
         continue
-      yield (info.filename, None, zip_file.open(info))
+      yield FileEntry(info.filename, None, zip_file.open(info))
 
 
-def IterateTar(path):
+def IterateTar(path, compression):
   """
-  IterateTar opens the tar.gz file at path and returns a generator of
+  IterateTar opens the tar.gz or tar.bz2 file at path and returns a generator of
-  (filename, mode, fileobj) tuples for each file in it.
+  entry objects for each file in it.
   """
-  with tarfile.open(path, 'r:gz') as tar_file:
+  with tarfile.open(path, 'r:' + compression) as tar_file:
     for info in tar_file:
       if info.isdir():
-        continue
+        pass
-      if not info.isfile():
+      elif info.issym():
+        yield SymlinkEntry(info.name, None, info.linkname)
+      elif info.isfile():
+        yield FileEntry(info.name, info.mode, tar_file.extractfile(info))
+      else:
         raise ValueError('Unknown entry type "%s"' % (info.name, ))
-      yield (info.name, info.mode, tar_file.extractfile(info))
 
 
 def main(args):
@@ -98,7 +115,9 @@ def main(args):
   if archive.endswith('.zip'):
     entries = IterateZip(archive)
   elif archive.endswith('.tar.gz'):
-    entries = IterateTar(archive)
+    entries = IterateTar(archive, 'gz')
+  elif archive.endswith('.tar.bz2'):
+    entries = IterateTar(archive, 'bz2')
   else:
     raise ValueError(archive)
 
@@ -110,13 +129,13 @@ def main(args):
     print "Extracting %s to %s" % (archive, output)
     prefix = None
     num_extracted = 0
-    for path, mode, inp in entries:
+    for entry in entries:
       # Even on Windows, zip files must always use forward slashes.
-      if '\\' in path or path.startswith('/'):
+      if '\\' in entry.path or entry.path.startswith('/'):
-        raise ValueError(path)
+        raise ValueError(entry.path)
 
       if not options.no_prefix:
-        new_prefix, rest = path.split('/', 1)
+        new_prefix, rest = entry.path.split('/', 1)
 
         # Ensure the archive is consistent.
         if prefix is None:
@@ -124,20 +143,25 @@ def main(args):
         if prefix != new_prefix:
           raise ValueError((prefix, new_prefix))
       else:
-        rest = path
+        rest = entry.path
 
       # Extract the file into the output directory.
       fixed_path = CheckedJoin(output, rest)
       if not os.path.isdir(os.path.dirname(fixed_path)):
         os.makedirs(os.path.dirname(fixed_path))
+      if isinstance(entry, FileEntry):
         with open(fixed_path, 'wb') as out:
-        shutil.copyfileobj(inp, out)
+          shutil.copyfileobj(entry.fileobj, out)
+      elif isinstance(entry, SymlinkEntry):
+        os.symlink(entry.target, fixed_path)
+      else:
+        raise TypeError('unknown entry type')
 
       # Fix up permissions if needbe.
       # TODO(davidben): To be extra tidy, this should only track the execute bit
       # as in git.
-      if mode is not None:
+      if entry.mode is not None:
-        os.chmod(fixed_path, mode)
+        os.chmod(fixed_path, entry.mode)
 
       # Print every 100 files, so bots do not time out on large archives.
       num_extracted += 1

util/bot/sde-linux64.tar.bz2.sha1

@@ -0,0 +1 @@
+3d612f0095a0830656614e5b442f70289e266088