[SPARK-27889][INFRA] Make development scripts under dev/ support Python 3
## What changes were proposed in this pull request? I made an audit and update all dev scripts to support python3. (except `merge_spark_pr.py` which already updated) ## How was this patch tested? Manual. Closes #25289 from WeichenXu123/dev_py3. Authored-by: WeichenXu <weichen.xu@databricks.com> Signed-off-by: HyukjinKwon <gurwls223@apache.org>
This commit is contained in:
parent
5bb69945e4
commit
f21bc1874a
|
@ -22,7 +22,14 @@ import json
|
|||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib2
|
||||
if sys.version < '3':
|
||||
from urllib2 import urlopen
|
||||
from urllib2 import Request
|
||||
from urllib2 import HTTPError
|
||||
else:
|
||||
from urllib.request import urlopen
|
||||
from urllib.request import Request
|
||||
from urllib.error import HTTPError
|
||||
|
||||
try:
|
||||
import jira.client
|
||||
|
@ -52,16 +59,16 @@ MAX_FILE = ".github-jira-max"
|
|||
|
||||
def get_url(url):
|
||||
try:
|
||||
request = urllib2.Request(url)
|
||||
request = Request(url)
|
||||
request.add_header('Authorization', 'token %s' % GITHUB_OAUTH_KEY)
|
||||
return urllib2.urlopen(request)
|
||||
except urllib2.HTTPError:
|
||||
return urlopen(request)
|
||||
except HTTPError:
|
||||
print("Unable to fetch URL, exiting: %s" % url)
|
||||
sys.exit(-1)
|
||||
|
||||
|
||||
def get_json(urllib_response):
|
||||
return json.load(urllib_response)
|
||||
return json.loads(urllib_response.read().decode("utf-8"))
|
||||
|
||||
|
||||
# Return a list of (JIRA id, JSON dict) tuples:
|
||||
|
@ -80,7 +87,7 @@ def get_jira_prs():
|
|||
result = result + [(jira, pull)]
|
||||
|
||||
# Check if there is another page
|
||||
link_headers = filter(lambda k: k.startswith("Link"), page.info().headers)
|
||||
link_headers = list(filter(lambda k: k.startswith("Link"), page.headers))
|
||||
if not link_headers or "next" not in link_headers[0]:
|
||||
has_next_page = False
|
||||
else:
|
||||
|
@ -122,13 +129,13 @@ def reset_pr_labels(pr_num, jira_components):
|
|||
url = '%s/issues/%s/labels' % (GITHUB_API_BASE, pr_num)
|
||||
labels = ', '.join(('"%s"' % c) for c in jira_components)
|
||||
try:
|
||||
request = urllib2.Request(url, data='{"labels":[%s]}' % labels)
|
||||
request = Request(url, data=('{"labels":[%s]}' % labels).encode('utf-8'))
|
||||
request.add_header('Content-Type', 'application/json')
|
||||
request.add_header('Authorization', 'token %s' % GITHUB_OAUTH_KEY)
|
||||
request.get_method = lambda: 'PUT'
|
||||
urllib2.urlopen(request)
|
||||
urlopen(request)
|
||||
print("Set %s with labels %s" % (pr_num, labels))
|
||||
except urllib2.HTTPError:
|
||||
except HTTPError:
|
||||
print("Unable to update PR labels, exiting: %s" % url)
|
||||
sys.exit(-1)
|
||||
|
||||
|
|
|
@ -97,9 +97,9 @@ def fail(msg):
|
|||
def run_cmd(cmd):
|
||||
print(cmd)
|
||||
if isinstance(cmd, list):
|
||||
return subprocess.check_output(cmd).decode(sys.stdout.encoding)
|
||||
return subprocess.check_output(cmd).decode(sys.getdefaultencoding())
|
||||
else:
|
||||
return subprocess.check_output(cmd.split(" ")).decode(sys.stdout.encoding)
|
||||
return subprocess.check_output(cmd.split(" ")).decode(sys.getdefaultencoding())
|
||||
|
||||
|
||||
def continue_maybe(prompt):
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python2
|
||||
#!/usr/bin/env python
|
||||
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
|
@ -21,9 +21,17 @@ from __future__ import print_function
|
|||
import os
|
||||
import sys
|
||||
import json
|
||||
import urllib2
|
||||
import functools
|
||||
import subprocess
|
||||
if sys.version < '3':
|
||||
from urllib2 import urlopen
|
||||
from urllib2 import Request
|
||||
from urllib2 import HTTPError, URLError
|
||||
else:
|
||||
from urllib.request import urlopen
|
||||
from urllib.request import Request
|
||||
from urllib.error import HTTPError, URLError
|
||||
|
||||
|
||||
from sparktestsupport import SPARK_HOME, ERROR_CODES
|
||||
from sparktestsupport.shellutils import run_cmd
|
||||
|
@ -44,25 +52,25 @@ def post_message_to_github(msg, ghprb_pull_id):
|
|||
github_oauth_key = os.environ["GITHUB_OAUTH_KEY"]
|
||||
|
||||
posted_message = json.dumps({"body": msg})
|
||||
request = urllib2.Request(url,
|
||||
headers={
|
||||
"Authorization": "token %s" % github_oauth_key,
|
||||
"Content-Type": "application/json"
|
||||
},
|
||||
data=posted_message)
|
||||
request = Request(url,
|
||||
headers={
|
||||
"Authorization": "token %s" % github_oauth_key,
|
||||
"Content-Type": "application/json"
|
||||
},
|
||||
data=posted_message.encode('utf-8'))
|
||||
try:
|
||||
response = urllib2.urlopen(request)
|
||||
response = urlopen(request)
|
||||
|
||||
if response.getcode() == 201:
|
||||
print(" > Post successful.")
|
||||
except urllib2.HTTPError as http_e:
|
||||
except HTTPError as http_e:
|
||||
print_err("Failed to post message to Github.")
|
||||
print_err(" > http_code: %s" % http_e.code)
|
||||
print_err(" > api_response: %s" % http_e.read())
|
||||
print_err(" > data: %s" % posted_message)
|
||||
except urllib2.URLError as url_e:
|
||||
except URLError as url_e:
|
||||
print_err("Failed to post message to Github.")
|
||||
print_err(" > urllib2_status: %s" % url_e.reason[1])
|
||||
print_err(" > urllib_status: %s" % url_e.reason[1])
|
||||
print_err(" > data: %s" % posted_message)
|
||||
|
||||
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python2
|
||||
#!/usr/bin/env python
|
||||
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
|
|
|
@ -55,7 +55,7 @@ def run_cmd(cmd, return_output=False):
|
|||
cmd = cmd.split()
|
||||
try:
|
||||
if return_output:
|
||||
return subprocess_check_output(cmd)
|
||||
return subprocess_check_output(cmd).decode(sys.getdefaultencoding())
|
||||
else:
|
||||
return subprocess_check_call(cmd)
|
||||
except subprocess.CalledProcessError as e:
|
||||
|
|
Loading…
Reference in a new issue