Cockpituous
parent
044b8da55a
commit
d5a822884f
288 changed files with 13040 additions and 1 deletions
305
bots/image-download
Executable file
305
bots/image-download
Executable file
|
|
@ -0,0 +1,305 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
# This file is part of Cockpit.
|
||||
#
|
||||
# Copyright (C) 2013 Red Hat, Inc.
|
||||
#
|
||||
# Cockpit is free software; you can redistribute it and/or modify it
|
||||
# under the terms of the GNU Lesser General Public License as published by
|
||||
# the Free Software Foundation; either version 2.1 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# Cockpit is distributed in the hope that it will be useful, but
|
||||
# WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
# Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with Cockpit; If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
#
|
||||
# Download images or other state
|
||||
#
|
||||
# Images usually have a name specific link committed to git. These
|
||||
# are referred to as 'committed'
|
||||
#
|
||||
# Other state is simply referenced by name without a link in git
|
||||
# This is referred to as 'state'
|
||||
#
|
||||
# The stores are places to look for images or other state
|
||||
#
|
||||
|
||||
import argparse
|
||||
import email
|
||||
import io
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import fcntl
|
||||
import urllib.parse
|
||||
|
||||
from machine import testvm
|
||||
from task import REDHAT_STORE
|
||||
|
||||
CONFIG = "~/.config/image-stores"
|
||||
DEFAULT = [
|
||||
"http://cockpit-images.verify.svc.cluster.local",
|
||||
"https://images-cockpit.apps.ci.centos.org/",
|
||||
"https://209.132.184.41:8493/",
|
||||
REDHAT_STORE
|
||||
]
|
||||
|
||||
BOTS = os.path.dirname(os.path.realpath(__file__))
|
||||
|
||||
DEVNULL = open("/dev/null", "r+")
|
||||
EPOCH = "Thu, 1 Jan 1970 00:00:00 GMT"
|
||||
|
||||
def find(name, stores, latest, quiet):
|
||||
found = [ ]
|
||||
ca = os.path.join(testvm.IMAGES_DIR, "files", "ca.pem")
|
||||
|
||||
for store in stores:
|
||||
url = urllib.parse.urlparse(store)
|
||||
|
||||
defport = url.scheme == 'http' and 80 or 443
|
||||
|
||||
try:
|
||||
ai = socket.getaddrinfo(url.hostname, url.port or defport, socket.AF_INET, 0, socket.IPPROTO_TCP)
|
||||
except socket.gaierror:
|
||||
ai = [ ]
|
||||
message = store
|
||||
|
||||
for (family, socktype, proto, canonname, sockaddr) in ai:
|
||||
message = "{scheme}://{0}:{1}{path}".format(*sockaddr, scheme=url.scheme, path=url.path)
|
||||
|
||||
def curl(*args):
|
||||
try:
|
||||
cmd = ["curl"] + list(args) + ["--head", "--silent", "--fail", "--cacert", ca, source]
|
||||
start = time.time()
|
||||
output = subprocess.check_output(cmd, universal_newlines=True)
|
||||
found.append((cmd, output, message, time.time() - start))
|
||||
if not quiet:
|
||||
sys.stderr.write(" > {0}\n".format(message))
|
||||
return True
|
||||
except subprocess.CalledProcessError:
|
||||
return False
|
||||
|
||||
# first try with stores that accept the "cockpit-tests" host name
|
||||
resolve = "cockpit-tests:{1}:{0}".format(*sockaddr)
|
||||
source = urllib.parse.urljoin("{0}://cockpit-tests:{1}{2}".format(url.scheme, sockaddr[1], url.path), name)
|
||||
if curl("--resolve", resolve):
|
||||
continue
|
||||
|
||||
# fall back for OpenShift proxied stores which send their own SSL cert initially; host name has to match that
|
||||
source = urllib.parse.urljoin(store, name)
|
||||
if curl():
|
||||
continue
|
||||
|
||||
if not quiet:
|
||||
sys.stderr.write(" x {0}\n".format(message))
|
||||
|
||||
# If we couldn't find the file, but it exists, we're good
|
||||
if not found:
|
||||
return None, None
|
||||
|
||||
# Find the most recent version of this file
|
||||
def header_date(args):
|
||||
cmd, output, message, latency = args
|
||||
try:
|
||||
reply_line, headers_alone = output.split('\n', 1)
|
||||
last_modified = email.message_from_file(io.StringIO(headers_alone)).get("Last-Modified", "")
|
||||
return time.mktime(time.strptime(last_modified, '%a, %d %b %Y %H:%M:%S %Z'))
|
||||
except ValueError:
|
||||
return ""
|
||||
|
||||
if latest:
|
||||
found.sort(reverse=True, key=header_date)
|
||||
else:
|
||||
found.sort(reverse=False, key=lambda x: x[3])
|
||||
|
||||
# Return the command and message
|
||||
return found[0][0], found[0][2]
|
||||
|
||||
def download(dest, force, state, quiet, stores):
|
||||
if not stores:
|
||||
config = os.path.expanduser(CONFIG)
|
||||
if os.path.exists(config):
|
||||
with open(config, 'r') as fp:
|
||||
stores = fp.read().strip().split("\n")
|
||||
else:
|
||||
stores = []
|
||||
stores += DEFAULT
|
||||
|
||||
# The time condition for If-Modified-Since
|
||||
exists = not force and os.path.exists(dest)
|
||||
if exists:
|
||||
since = dest
|
||||
else:
|
||||
since = EPOCH
|
||||
|
||||
name = os.path.basename(dest)
|
||||
cmd, message = find(name, stores, latest=state, quiet=quiet)
|
||||
|
||||
# If we couldn't find the file, but it exists, we're good
|
||||
if not cmd:
|
||||
if exists:
|
||||
return
|
||||
raise RuntimeError("image-download: couldn't find file anywhere: {0}".format(name))
|
||||
|
||||
# Choose the first found item after sorting by date
|
||||
if not quiet:
|
||||
sys.stderr.write(" > {0}\n".format(urllib.parse.urljoin(message, name)))
|
||||
|
||||
temp = dest + ".partial"
|
||||
|
||||
# Adjust the command above that worked to make it visible and download real stuff
|
||||
cmd.remove("--head")
|
||||
cmd.append("--show-error")
|
||||
if not quiet and os.isatty(sys.stdout.fileno()):
|
||||
cmd.remove("--silent")
|
||||
cmd.insert(1, "--progress-bar")
|
||||
cmd.append("--remote-time")
|
||||
cmd.append("--time-cond")
|
||||
cmd.append(since)
|
||||
cmd.append("--output")
|
||||
cmd.append(temp)
|
||||
if os.path.exists(temp):
|
||||
if force:
|
||||
os.remove(temp)
|
||||
else:
|
||||
cmd.append("-C")
|
||||
cmd.append("-")
|
||||
|
||||
# Always create the destination file (because --state)
|
||||
else:
|
||||
open(temp, 'a').close()
|
||||
|
||||
curl = subprocess.Popen(cmd)
|
||||
ret = curl.wait()
|
||||
if ret != 0:
|
||||
raise RuntimeError("curl: unable to download %s (returned: %s)" % (message, ret))
|
||||
|
||||
os.chmod(temp, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
|
||||
|
||||
# Due to time-cond the file size may be zero
|
||||
# A new file downloaded, put it in place
|
||||
if not exists or os.path.getsize(temp) > 0:
|
||||
shutil.move(temp, dest)
|
||||
|
||||
# Calculate a place to put images where links are not committed in git
|
||||
def state_target(path):
|
||||
data_dir = testvm.get_images_data_dir()
|
||||
os.makedirs(data_dir, mode=0o775, exist_ok=True)
|
||||
return os.path.join(data_dir, path)
|
||||
|
||||
# Calculate a place to put images where links are committed in git
|
||||
def committed_target(image):
|
||||
link = os.path.join(testvm.IMAGES_DIR, image)
|
||||
if not os.path.islink(link):
|
||||
raise RuntimeError("image link does not exist: " + image)
|
||||
|
||||
dest = os.readlink(link)
|
||||
relative_dir = os.path.dirname(os.path.abspath(link))
|
||||
full_dest = os.path.join(relative_dir, dest)
|
||||
while os.path.islink(full_dest):
|
||||
link = full_dest
|
||||
dest = os.readlink(link)
|
||||
relative_dir = os.path.dirname(os.path.abspath(link))
|
||||
full_dest = os.path.join(relative_dir, dest)
|
||||
|
||||
dest = os.path.join(testvm.get_images_data_dir(), dest)
|
||||
|
||||
# We have the file but there is not valid link
|
||||
if os.path.exists(dest):
|
||||
try:
|
||||
os.symlink(dest, os.path.join(testvm.IMAGES_DIR, os.readlink(link)))
|
||||
except FileExistsError:
|
||||
pass
|
||||
|
||||
# The image file in the images directory, may be same as dest
|
||||
image_file = os.path.join(testvm.IMAGES_DIR, os.readlink(link))
|
||||
|
||||
# Double check that symlink in place but never make a cycle.
|
||||
if os.path.abspath(dest) != os.path.abspath(image_file):
|
||||
try:
|
||||
os.symlink(os.path.abspath(dest), image_file)
|
||||
except FileExistsError:
|
||||
pass
|
||||
|
||||
return dest
|
||||
|
||||
def wait_lock(target):
|
||||
lockfile = os.path.join(tempfile.gettempdir(), ".cockpit-test-resources", os.path.basename(target) + ".lock")
|
||||
os.makedirs(os.path.dirname(lockfile), exist_ok=True)
|
||||
|
||||
# we need to keep the lock fd open throughout the entire runtime, so remember it in a global-scoped variable
|
||||
wait_lock.f = open(lockfile, "w")
|
||||
for retry in range(360):
|
||||
try:
|
||||
fcntl.flock(wait_lock.f, fcntl.LOCK_NB | fcntl.LOCK_EX)
|
||||
return
|
||||
except BlockingIOError:
|
||||
if retry == 0:
|
||||
print("Waiting for concurrent image-download of %s..." % os.path.basename(target))
|
||||
time.sleep(10)
|
||||
else:
|
||||
raise TimeoutError("timed out waiting for concurrent downloads of %s\n" % target)
|
||||
|
||||
def download_images(image_list, force, quiet, state, store):
|
||||
data_dir = testvm.get_images_data_dir()
|
||||
os.makedirs(data_dir, exist_ok=True)
|
||||
|
||||
# A default set of images are all links in git. These links have
|
||||
# no directory part. Other links might exist, such as the
|
||||
# auxiliary links created by committed_target above, and we ignore
|
||||
# them.
|
||||
if not image_list:
|
||||
image_list = []
|
||||
if not state:
|
||||
for filename in os.listdir(testvm.IMAGES_DIR):
|
||||
link = os.path.join(testvm.IMAGES_DIR, filename)
|
||||
if os.path.islink(link) and os.path.dirname(os.readlink(link)) == "":
|
||||
image_list.append(filename)
|
||||
|
||||
success = True
|
||||
|
||||
for image in image_list:
|
||||
image = testvm.get_test_image(image)
|
||||
try:
|
||||
if state:
|
||||
target = state_target(image)
|
||||
else:
|
||||
target = committed_target(image)
|
||||
|
||||
# don't download the same thing multiple times in parallel
|
||||
wait_lock(target)
|
||||
|
||||
if force or state or not os.path.exists(target):
|
||||
download(target, force, state, quiet, store)
|
||||
except Exception as ex:
|
||||
success = False
|
||||
sys.stderr.write("image-download: {0}\n".format(str(ex)))
|
||||
|
||||
return success
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Download a bot state or images')
|
||||
parser.add_argument("--force", action="store_true", help="Force unnecessary downloads")
|
||||
parser.add_argument("--store", action="append", help="Where to find state or images")
|
||||
parser.add_argument("--quiet", action="store_true", help="Make downloading quieter")
|
||||
parser.add_argument("--state", action="store_true", help="Images or state not recorded in git")
|
||||
parser.add_argument('image', nargs='*')
|
||||
args = parser.parse_args()
|
||||
|
||||
if not download_images(args.image, args.force, args.quiet, args.state, args.store):
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Loading…
Add table
Add a link
Reference in a new issue