#!/usr/bin/env python3 # This file is part of Cockpit. # # Copyright (C) 2017 Red Hat, Inc. # # Cockpit is free software; you can redistribute it and/or modify it # under the terms of the GNU Lesser General Public License as published by # the Free Software Foundation; either version 2.1 of the License, or # (at your option) any later version. # # Cockpit is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public License # along with Cockpit; If not, see . # Shared GitHub code. When run as a script, we print out info about # our GitHub interacition. import argparse import os import random import shutil import socket import subprocess import sys import time import traceback import urllib.parse sys.dont_write_bytecode = True from . import github from . import sink __all__ = ( "api", "main", "run", "pull", "comment", "label", "issue", "verbose", "stale", "redhat_network", "REDHAT_STORE", ) # Server which has the private RHEL/Windows images REDHAT_STORE = "https://cockpit-11.e2e.bos.redhat.com:8493" api = github.GitHub() verbose = False BOTS = os.path.normpath(os.path.join(os.path.dirname(__file__), "..")) BASE = os.path.normpath(os.path.join(BOTS, "..")) # # The main function takes a list of tasks, each of wihch has the following # fields, some of which have defaults: # # title: The title for the task # function: The function to call for the task # options=[]: A list of string options to pass to the function argument # # The function for the task will be called with all the context for the task. # In addition it will be called with named arguments for all other task fields # and additional fields such as |verbose|. It should return a zero or None value # if successful, and a string or non-zero value if unsuccessful. # # def run(context, verbose=False, **kwargs): # if verbose: # sys.stderr.write(image + "\n") # return 0 # # Call the task.main() as the entry point of your script in one of these ways: # # # As a single task # task.main(title="My title", function=run) # def main(**kwargs): global verbose task = kwargs.copy() # Figure out a descriptoin for the --help task["name"] = named(task) parser = argparse.ArgumentParser(description=task.get("title", task["name"])) parser.add_argument("-v", "--verbose", action="store_true", help="Verbose output") parser.add_argument("--issue", dest="issue", action="store", help="Act on an already created task issue") parser.add_argument("--publish", dest="publish", default=os.environ.get("TEST_PUBLISH", ""), action="store", help="Publish results centrally to a sink") parser.add_argument("--dry", dest="dry", action="store_true", help="Dry run to validate this task if supported") parser.add_argument("context", nargs="?") opts = parser.parse_args() verbose = opts.verbose ret = 0 if "verbose" not in task: task["verbose"] = opts.verbose task["issue"] = opts.issue task["publish"] = opts.publish task["dry"] = opts.dry ret = run(opts.context, **task) if ret: sys.stderr.write("{0}: {1}\n".format(task["name"], ret)) sys.exit(ret and 1 or 0) def named(task): if "name" in task: return task["name"] else: return os.path.basename(os.path.realpath(sys.argv[0])) def begin(publish, name, context, issue): if not publish: return None hostname = socket.gethostname().split(".")[0] current = time.strftime('%Y%m%d-%H%M%M') # Update the body for an existing issue if issue: number = issue["number"] identifier = "{0}-{1}-{2}".format(name, number, current) title = issue["title"] wip = "WIP: {0}: [no-test] {1}".format(hostname, title) requests = [ { "method": "POST", "resource": api.qualify("issues/{0}".format(number)), "data": { "title": wip } }, { "method": "POST", "resource": api.qualify("issues/{0}/comments".format(number)), "data": { "body": "{0} in progress on {1}.\nLog: :link".format(name, hostname) } } ] watches = [ { "resource": api.qualify("issues/{0}".format(number)), "result": { "title": wip } } ] aborted = [ { "method": "POST", "resource": api.qualify("issues/{0}".format(number)), "data": { "title": title } }, { "method": "POST", "resource": api.qualify("issues/{0}/comments".format(number)), "data": { "body": "Task aborted." } } ] else: identifier = "{0}-{1}".format(name, current) requests = [ ] watches = [ ] aborted = [ ] status = { "github": { "token": api.token, "requests": requests, "watches": watches }, "onaborted": { "github": { "token": api.token, "requests": aborted } } } publishing = sink.Sink(publish, identifier, status) sys.stderr.write("# Task: {0} {1}\n# Host: {2}\n\n".format(name, context or "", hostname)) # For statistics publishing.start = time.time() return publishing def finish(publishing, ret, name, context, issue): if not publishing: return if not ret: comment = None result = "Completed" elif isinstance(ret, str): comment = "{0}: :link".format(ret) result = ret else: comment = "Task failed: :link" result = "Failed" duration = int(time.time() - publishing.start) sys.stderr.write("\n# Result: {0}\n# Duration: {1}s\n".format(result, duration)) if issue: # Note that we check whether pass or fail ... this is because # the task is considered "done" until a human comes through and # triggers it again by unchecking the box. item = "{0} {1}".format(name, context or "").strip() checklist = github.Checklist(issue["body"]) checklist.check(item, ret and "FAIL" or True) number = issue["number"] # The sink wants us to escape colons :S body = checklist.body.replace(':', '::') requests = [ { "method": "POST", "resource": api.qualify("issues/{0}".format(number)), "data": { "title": "{0}".format(issue["title"]), "body": body } } ] # Close the issue if it's not a pull request, successful, and all tasks done if "pull_request" not in issue and not ret and len(checklist.items) == len(checklist.checked()): requests[0]["data"]["state"] = "closed" # Comment if there was a failure if comment: requests.insert(0, { "method": "POST", "resource": api.qualify("issues/{0}/comments".format(number)), "data": { "body": comment } }) else: requests = [ ] publishing.status['github']['requests'] = requests publishing.status['github']['watches'] = None publishing.status['github']['onaborted'] = None publishing.flush() def run(context, function, **kwargs): number = kwargs.get("issue", None) publish = kwargs.get("publish", "") name = kwargs["name"] issue = None if number: issue = api.get("issues/{0}".format(number)) if not issue: return "No such issue: {0}".format(number) elif issue["title"].startswith("WIP:"): return "Issue is work in progress: {0}: {1}\n".format(number, issue["title"]) kwargs["issue"] = issue kwargs["title"] = issue["title"] publishing = begin(publish, name, context, issue=issue) ret = "Task threw an exception" try: if issue and "pull_request" in issue: kwargs["pull"] = api.get(issue["pull_request"]["url"]) ret = function(context, **kwargs) except (RuntimeError, subprocess.CalledProcessError) as ex: ret = str(ex) except (AssertionError, KeyboardInterrupt): raise except: traceback.print_exc() finally: finish(publishing, ret, name, context, issue) return ret or 0 # Check if the given files that match @pathspec are stale # and haven't been updated in @days. def stale(days, pathspec, ref="HEAD"): global verbose def execute(*args): if verbose: sys.stderr.write("+ " + " ".join(args) + "\n") output = subprocess.check_output(args, cwd=BASE, universal_newlines=True) if verbose: sys.stderr.write("> " + output + "\n") return output timestamp = execute("git", "log", "--max-count=1", "--pretty=format:%ct", ref, "--", pathspec) try: timestamp = int(timestamp) except ValueError: timestamp = 0 # We randomize when we think this should happen over a day offset = days * 86400 due = time.time() - random.randint(offset - 43200, offset + 43200) return timestamp < due def issue(title, body, item, context=None, items=[], state="open", since=None): if context: item = "{0} {1}".format(item, context).strip() if since: # don't let all bots pass the deadline in the same second, to avoid many duplicates since += random.randint(-3600, 3600) for issue in api.issues(state=state, since=since): checklist = github.Checklist(issue["body"]) if item in checklist.items: return issue if not items: items = [ item ] checklist = github.Checklist(body) for x in items: checklist.add(x) data = { "title": title, "body": checklist.body, "labels": [ "bot" ] } return api.post("issues", data) def execute(*args): global verbose if verbose: sys.stderr.write("+ " + " ".join(args) + "\n") # Make double sure that the token does not appear anywhere in the output def censored(text): return text.replace(api.token, "CENSORED") env = os.environ.copy() # No prompting for passwords if "GIT_ASKPASS" not in env: env["GIT_ASKPASS"] = "/bin/true" output = subprocess.check_output(args, cwd=BASE, stderr=subprocess.STDOUT, env=env, universal_newlines=True) sys.stderr.write(censored(output)) return output def find_our_fork(user): repos = api.get("/users/{0}/repos".format(user)) for r in repos: if r["full_name"] == api.repo: # We actually own the origin repo, so use that. return api.repo if r["fork"]: full = api.get("/repos/{0}/{1}".format(user, r["name"])) if full["parent"]["full_name"] == api.repo: return full["full_name"] raise RuntimeError("%s doesn't have a fork of %s" % (user, api.repo)) def push_branch(user, branch, force=False): fork_repo = find_our_fork(user) url = "https://github.com/{0}".format(fork_repo) cmd = ["git", "push", url, "+HEAD:refs/heads/{0}".format(branch)] if force: cmd.insert(2, "-f") execute(*cmd) def branch(context, message, pathspec=".", issue=None, branch=None, push=True, **kwargs): name = named(kwargs) if not branch: execute("git", "checkout", "--detach") current = time.strftime('%Y%m%d-%H%M%M') branch = "{0} {1} {2}".format(name, context or "", current).strip() branch = branch.replace(" ", "-").replace("--", "-") # Tell git about our github token as a user name try: subprocess.check_call(["git", "config", "credential.https://github.com.username", api.token]) except subprocess.CalledProcessError: raise RuntimeError("Couldn't configure git config with our API token") user = api.get("/user")['login'] fork_repo = find_our_fork(user) clean = "https://github.com/{0}".format(fork_repo) if pathspec is not None: execute("git", "add", "--", pathspec) # If there's nothing to add at that pathspec return None try: execute("git", "commit", "-m", message) except subprocess.CalledProcessError: return None # No need to push if we want to add another commits into the same branch if push: push_branch(user, branch) # Comment on the issue if present and we pushed the branch if issue and push: comment_done(issue, name, clean, branch, context) return "{0}:{1}".format(user, branch) def pull(branch, body=None, issue=None, base="master", labels=['bot'], run_tests=True, **kwargs): if "pull" in kwargs: return kwargs["pull"] data = { "head": branch, "base": base, "maintainer_can_modify": True } if issue: try: data["issue"] = issue["number"] except TypeError: data["issue"] = int(issue) else: data["title"] = "[no-test] " + kwargs["title"] if body: data["body"] = body pull = api.post("pulls", data, accept=[ 422 ]) # If we were refused to grant maintainer_can_modify, then try without if "errors" in pull: if pull["errors"][0]["field"] == "fork_collab": data["maintainer_can_modify"] = False pull = api.post("pulls", data) # Update the pull request label(pull, labels) # Update the issue if it is a dict if issue: try: issue["title"] = kwargs["title"] issue["pull_request"] = { "url": pull["url"] } except TypeError: pass if pull["number"]: # If we want to run tests automatically, drop [no-test] from title before force push if run_tests: pull = api.post("pulls/" + str(pull["number"]), {"title": kwargs["title"]}, accept=[ 422 ]) # Force push last_commit_m = execute("git", "show", "--no-patch", "--format=%B") last_commit_m += "Closes #" + str(pull["number"]) execute("git", "commit", "--amend", "-m", last_commit_m) (user, branch) = branch.split(":") push_branch(user, branch, True) # Make sure we return the updated pull data for retry in range(20): new_data = api.get("pulls/{}".format(pull["number"])) if pull["head"]["sha"] != new_data["head"]["sha"]: pull = new_data break time.sleep(6) else: raise RuntimeError("Failed to retrieve updated pull data after force pushing") return pull def label(issue, labels=['bot']): try: resource = "issues/{0}/labels".format(issue["number"]) except TypeError: resource = "issues/{0}/labels".format(issue) return api.post(resource, labels) def labels_of_pull(pull): if "labels" not in pull: pull["labels"] = api.get("issues/{0}/labels".format(pull["number"])) return list(map(lambda label: label["name"], pull["labels"])) def comment(issue, comment): try: number = issue["number"] except TypeError: number = issue return api.post("issues/{0}/comments".format(number), { "body": comment }) def comment_done(issue, name, clean, branch, context=None): message = "{0} {1} done: {2}/commits/{3}".format(name, context or "", clean, branch) comment(issue, message) def attach(filename): if "TEST_ATTACHMENTS" in os.environ: shutil.copy(filename, os.environ["TEST_ATTACHMENTS"]) def redhat_network(): '''Check if we can access the Red Hat network This checks if the image server can be accessed. The result gets cached, so this can be called several times. ''' if redhat_network.result is None: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) store = urllib.parse.urlparse(REDHAT_STORE) try: s.connect((store.hostname, store.port)) redhat_network.result = True except OSError: redhat_network.result = False return redhat_network.result redhat_network.result = None