ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-28051][INFRA] Exposing JIRA issue component types at GitHub PRs

Browse source 
## What changes were proposed in this pull request?

This PR aims to expose JIRA issue component types at GitHub PRs.

## How was this patch tested?

Manual.
```
$ export GITHUB_OAUTH_KEY=...
$ export JIRA_PASSWORD=...
$ export GITHUB_API_BASE='https://api.github.com/repos/your-id/spark'
$ dev/github_jira_sync.py
```

Please note that the existing script will raise the following exceptions if your repo has less than 100 PRs. This will be handled at #24874 .
```
Traceback (most recent call last):
  File "dev/github_jira_sync.py", line 139, in <module>
    jira_prs = get_jira_prs()
  File "dev/github_jira_sync.py", line 83, in get_jira_prs
    link_header = filter(lambda k: k.startswith("Link"), page.info().headers)[0]
IndexError: list index out of range
```
That is beyond the scope of this PR.

Closes #24871 from dongjoon-hyun/SPARK-28051.

Authored-by: Dongjoon Hyun <dhyun@apple.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
This commit is contained in:
Dongjoon Hyun 2019-06-14 20:36:45 -07:00
parent bd0a04baab
commit fd8240d10c
1 changed files with 45 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
dev/github_jira_sync.py
View file

@ -33,6 +33,7 @@ except ImportError:
# User facing configs
GITHUB_API_BASE = os.environ.get("GITHUB_API_BASE", "https://api.github.com/repos/apache/spark")
GITHUB_OAUTH_KEY = os.environ.get("GITHUB_OAUTH_KEY")
JIRA_PROJECT_NAME = os.environ.get("JIRA_PROJECT_NAME", "SPARK")
JIRA_API_BASE = os.environ.get("JIRA_API_BASE", "https://issues.apache.org/jira")
JIRA_USERNAME = os.environ.get("JIRA_USERNAME", "apachespark")
@ -51,7 +52,9 @@ MAX_FILE = ".github-jira-max"
def get_url(url):
try:
return urllib2.urlopen(url)
request = urllib2.Request(url)
request.add_header('Authorization', 'token %s' % GITHUB_OAUTH_KEY)
return urllib2.urlopen(request)
except urllib2.HTTPError:
print("Unable to fetch URL, exiting: %s" % url)
sys.exit(-1)
@ -101,6 +104,35 @@ def get_max_pr():
return 0
def build_pr_component_dic(jira_prs):
print("Build PR dictionary")
dic = {}
for issue, pr in jira_prs:
print(issue)
jira_components = [c.name.upper() for c in jira_client.issue(issue).fields.components]
if pr['number'] in dic:
dic[pr['number']][1].update(jira_components)
else:
pr_components = set(label['name'].upper() for label in pr['labels'])
dic[pr['number']] = (pr_components, set(jira_components))
return dic
def reset_pr_labels(pr_num, jira_components):
url = '%s/issues/%s/labels' % (GITHUB_API_BASE, pr_num)
labels = ', '.join(('"%s"' % c) for c in jira_components)
try:
request = urllib2.Request(url, data='{"labels":[%s]}' % labels)
request.add_header('Content-Type', 'application/json')
request.add_header('Authorization', 'token %s' % GITHUB_OAUTH_KEY)
request.get_method = lambda: 'PUT'
urllib2.urlopen(request)
print("Set %s with labels %s" % (pr_num, labels))
except urllib2.HTTPError:
print("Unable to update PR labels, exiting: %s" % url)
sys.exit(-1)
jira_client = jira.client.JIRA({'server': JIRA_API_BASE},
basic_auth=(JIRA_USERNAME, JIRA_PASSWORD))
@ -151,3 +183,15 @@ for issue, pr in sorted(jira_prs, key=lambda kv: int(kv[1]['number'])):
if len(considered) > 0:
set_max_pr(max(considered))
# Additionally, expose the JIRA labels to the PR
num_updates = 0
for pr_num, (pr_components, jira_components) in build_pr_component_dic(jira_prs).items():
print(pr_num)
if pr_components == jira_components:
continue
if num_updates >= MAX_UPDATES:
break
reset_pr_labels(pr_num, jira_components)
num_updates += 1