From fd8240d10cd946e88d48a7e8bdadde2112bc1d62 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Fri, 14 Jun 2019 20:36:45 -0700 Subject: [PATCH] [SPARK-28051][INFRA] Exposing JIRA issue component types at GitHub PRs ## What changes were proposed in this pull request? This PR aims to expose JIRA issue component types at GitHub PRs. ## How was this patch tested? Manual. ``` $ export GITHUB_OAUTH_KEY=... $ export JIRA_PASSWORD=... $ export GITHUB_API_BASE='https://api.github.com/repos/your-id/spark' $ dev/github_jira_sync.py ``` Please note that the existing script will raise the following exceptions if your repo has less than 100 PRs. This will be handled at #24874 . ``` Traceback (most recent call last): File "dev/github_jira_sync.py", line 139, in jira_prs = get_jira_prs() File "dev/github_jira_sync.py", line 83, in get_jira_prs link_header = filter(lambda k: k.startswith("Link"), page.info().headers)[0] IndexError: list index out of range ``` That is beyond the scope of this PR. Closes #24871 from dongjoon-hyun/SPARK-28051. Authored-by: Dongjoon Hyun Signed-off-by: Dongjoon Hyun --- dev/github_jira_sync.py | 46 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/dev/github_jira_sync.py b/dev/github_jira_sync.py index 4f0f7fa71b..7b693cb64f 100755 --- a/dev/github_jira_sync.py +++ b/dev/github_jira_sync.py @@ -33,6 +33,7 @@ except ImportError: # User facing configs GITHUB_API_BASE = os.environ.get("GITHUB_API_BASE", "https://api.github.com/repos/apache/spark") +GITHUB_OAUTH_KEY = os.environ.get("GITHUB_OAUTH_KEY") JIRA_PROJECT_NAME = os.environ.get("JIRA_PROJECT_NAME", "SPARK") JIRA_API_BASE = os.environ.get("JIRA_API_BASE", "https://issues.apache.org/jira") JIRA_USERNAME = os.environ.get("JIRA_USERNAME", "apachespark") @@ -51,7 +52,9 @@ MAX_FILE = ".github-jira-max" def get_url(url): try: - return urllib2.urlopen(url) + request = urllib2.Request(url) + request.add_header('Authorization', 'token %s' % GITHUB_OAUTH_KEY) + return urllib2.urlopen(request) except urllib2.HTTPError: print("Unable to fetch URL, exiting: %s" % url) sys.exit(-1) @@ -101,6 +104,35 @@ def get_max_pr(): return 0 +def build_pr_component_dic(jira_prs): + print("Build PR dictionary") + dic = {} + for issue, pr in jira_prs: + print(issue) + jira_components = [c.name.upper() for c in jira_client.issue(issue).fields.components] + if pr['number'] in dic: + dic[pr['number']][1].update(jira_components) + else: + pr_components = set(label['name'].upper() for label in pr['labels']) + dic[pr['number']] = (pr_components, set(jira_components)) + return dic + + +def reset_pr_labels(pr_num, jira_components): + url = '%s/issues/%s/labels' % (GITHUB_API_BASE, pr_num) + labels = ', '.join(('"%s"' % c) for c in jira_components) + try: + request = urllib2.Request(url, data='{"labels":[%s]}' % labels) + request.add_header('Content-Type', 'application/json') + request.add_header('Authorization', 'token %s' % GITHUB_OAUTH_KEY) + request.get_method = lambda: 'PUT' + urllib2.urlopen(request) + print("Set %s with labels %s" % (pr_num, labels)) + except urllib2.HTTPError: + print("Unable to update PR labels, exiting: %s" % url) + sys.exit(-1) + + jira_client = jira.client.JIRA({'server': JIRA_API_BASE}, basic_auth=(JIRA_USERNAME, JIRA_PASSWORD)) @@ -151,3 +183,15 @@ for issue, pr in sorted(jira_prs, key=lambda kv: int(kv[1]['number'])): if len(considered) > 0: set_max_pr(max(considered)) + + +# Additionally, expose the JIRA labels to the PR +num_updates = 0 +for pr_num, (pr_components, jira_components) in build_pr_component_dic(jira_prs).items(): + print(pr_num) + if pr_components == jira_components: + continue + if num_updates >= MAX_UPDATES: + break + reset_pr_labels(pr_num, jira_components) + num_updates += 1