[SPARK-27889][INFRA] Make development scripts under dev/ support Python 3
## What changes were proposed in this pull request? I made an audit and update all dev scripts to support python3. (except `merge_spark_pr.py` which already updated) ## How was this patch tested? Manual. Closes #25289 from WeichenXu123/dev_py3. Authored-by: WeichenXu <weichen.xu@databricks.com> Signed-off-by: HyukjinKwon <gurwls223@apache.org>
This commit is contained in:
parent
5bb69945e4
commit
f21bc1874a
|
@ -22,7 +22,14 @@ import json
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import sys
|
import sys
|
||||||
import urllib2
|
if sys.version < '3':
|
||||||
|
from urllib2 import urlopen
|
||||||
|
from urllib2 import Request
|
||||||
|
from urllib2 import HTTPError
|
||||||
|
else:
|
||||||
|
from urllib.request import urlopen
|
||||||
|
from urllib.request import Request
|
||||||
|
from urllib.error import HTTPError
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import jira.client
|
import jira.client
|
||||||
|
@ -52,16 +59,16 @@ MAX_FILE = ".github-jira-max"
|
||||||
|
|
||||||
def get_url(url):
|
def get_url(url):
|
||||||
try:
|
try:
|
||||||
request = urllib2.Request(url)
|
request = Request(url)
|
||||||
request.add_header('Authorization', 'token %s' % GITHUB_OAUTH_KEY)
|
request.add_header('Authorization', 'token %s' % GITHUB_OAUTH_KEY)
|
||||||
return urllib2.urlopen(request)
|
return urlopen(request)
|
||||||
except urllib2.HTTPError:
|
except HTTPError:
|
||||||
print("Unable to fetch URL, exiting: %s" % url)
|
print("Unable to fetch URL, exiting: %s" % url)
|
||||||
sys.exit(-1)
|
sys.exit(-1)
|
||||||
|
|
||||||
|
|
||||||
def get_json(urllib_response):
|
def get_json(urllib_response):
|
||||||
return json.load(urllib_response)
|
return json.loads(urllib_response.read().decode("utf-8"))
|
||||||
|
|
||||||
|
|
||||||
# Return a list of (JIRA id, JSON dict) tuples:
|
# Return a list of (JIRA id, JSON dict) tuples:
|
||||||
|
@ -80,7 +87,7 @@ def get_jira_prs():
|
||||||
result = result + [(jira, pull)]
|
result = result + [(jira, pull)]
|
||||||
|
|
||||||
# Check if there is another page
|
# Check if there is another page
|
||||||
link_headers = filter(lambda k: k.startswith("Link"), page.info().headers)
|
link_headers = list(filter(lambda k: k.startswith("Link"), page.headers))
|
||||||
if not link_headers or "next" not in link_headers[0]:
|
if not link_headers or "next" not in link_headers[0]:
|
||||||
has_next_page = False
|
has_next_page = False
|
||||||
else:
|
else:
|
||||||
|
@ -122,13 +129,13 @@ def reset_pr_labels(pr_num, jira_components):
|
||||||
url = '%s/issues/%s/labels' % (GITHUB_API_BASE, pr_num)
|
url = '%s/issues/%s/labels' % (GITHUB_API_BASE, pr_num)
|
||||||
labels = ', '.join(('"%s"' % c) for c in jira_components)
|
labels = ', '.join(('"%s"' % c) for c in jira_components)
|
||||||
try:
|
try:
|
||||||
request = urllib2.Request(url, data='{"labels":[%s]}' % labels)
|
request = Request(url, data=('{"labels":[%s]}' % labels).encode('utf-8'))
|
||||||
request.add_header('Content-Type', 'application/json')
|
request.add_header('Content-Type', 'application/json')
|
||||||
request.add_header('Authorization', 'token %s' % GITHUB_OAUTH_KEY)
|
request.add_header('Authorization', 'token %s' % GITHUB_OAUTH_KEY)
|
||||||
request.get_method = lambda: 'PUT'
|
request.get_method = lambda: 'PUT'
|
||||||
urllib2.urlopen(request)
|
urlopen(request)
|
||||||
print("Set %s with labels %s" % (pr_num, labels))
|
print("Set %s with labels %s" % (pr_num, labels))
|
||||||
except urllib2.HTTPError:
|
except HTTPError:
|
||||||
print("Unable to update PR labels, exiting: %s" % url)
|
print("Unable to update PR labels, exiting: %s" % url)
|
||||||
sys.exit(-1)
|
sys.exit(-1)
|
||||||
|
|
||||||
|
|
|
@ -97,9 +97,9 @@ def fail(msg):
|
||||||
def run_cmd(cmd):
|
def run_cmd(cmd):
|
||||||
print(cmd)
|
print(cmd)
|
||||||
if isinstance(cmd, list):
|
if isinstance(cmd, list):
|
||||||
return subprocess.check_output(cmd).decode(sys.stdout.encoding)
|
return subprocess.check_output(cmd).decode(sys.getdefaultencoding())
|
||||||
else:
|
else:
|
||||||
return subprocess.check_output(cmd.split(" ")).decode(sys.stdout.encoding)
|
return subprocess.check_output(cmd.split(" ")).decode(sys.getdefaultencoding())
|
||||||
|
|
||||||
|
|
||||||
def continue_maybe(prompt):
|
def continue_maybe(prompt):
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
#!/usr/bin/env python2
|
#!/usr/bin/env python
|
||||||
|
|
||||||
#
|
#
|
||||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
@ -21,9 +21,17 @@ from __future__ import print_function
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
import json
|
import json
|
||||||
import urllib2
|
|
||||||
import functools
|
import functools
|
||||||
import subprocess
|
import subprocess
|
||||||
|
if sys.version < '3':
|
||||||
|
from urllib2 import urlopen
|
||||||
|
from urllib2 import Request
|
||||||
|
from urllib2 import HTTPError, URLError
|
||||||
|
else:
|
||||||
|
from urllib.request import urlopen
|
||||||
|
from urllib.request import Request
|
||||||
|
from urllib.error import HTTPError, URLError
|
||||||
|
|
||||||
|
|
||||||
from sparktestsupport import SPARK_HOME, ERROR_CODES
|
from sparktestsupport import SPARK_HOME, ERROR_CODES
|
||||||
from sparktestsupport.shellutils import run_cmd
|
from sparktestsupport.shellutils import run_cmd
|
||||||
|
@ -44,25 +52,25 @@ def post_message_to_github(msg, ghprb_pull_id):
|
||||||
github_oauth_key = os.environ["GITHUB_OAUTH_KEY"]
|
github_oauth_key = os.environ["GITHUB_OAUTH_KEY"]
|
||||||
|
|
||||||
posted_message = json.dumps({"body": msg})
|
posted_message = json.dumps({"body": msg})
|
||||||
request = urllib2.Request(url,
|
request = Request(url,
|
||||||
headers={
|
headers={
|
||||||
"Authorization": "token %s" % github_oauth_key,
|
"Authorization": "token %s" % github_oauth_key,
|
||||||
"Content-Type": "application/json"
|
"Content-Type": "application/json"
|
||||||
},
|
},
|
||||||
data=posted_message)
|
data=posted_message.encode('utf-8'))
|
||||||
try:
|
try:
|
||||||
response = urllib2.urlopen(request)
|
response = urlopen(request)
|
||||||
|
|
||||||
if response.getcode() == 201:
|
if response.getcode() == 201:
|
||||||
print(" > Post successful.")
|
print(" > Post successful.")
|
||||||
except urllib2.HTTPError as http_e:
|
except HTTPError as http_e:
|
||||||
print_err("Failed to post message to Github.")
|
print_err("Failed to post message to Github.")
|
||||||
print_err(" > http_code: %s" % http_e.code)
|
print_err(" > http_code: %s" % http_e.code)
|
||||||
print_err(" > api_response: %s" % http_e.read())
|
print_err(" > api_response: %s" % http_e.read())
|
||||||
print_err(" > data: %s" % posted_message)
|
print_err(" > data: %s" % posted_message)
|
||||||
except urllib2.URLError as url_e:
|
except URLError as url_e:
|
||||||
print_err("Failed to post message to Github.")
|
print_err("Failed to post message to Github.")
|
||||||
print_err(" > urllib2_status: %s" % url_e.reason[1])
|
print_err(" > urllib_status: %s" % url_e.reason[1])
|
||||||
print_err(" > data: %s" % posted_message)
|
print_err(" > data: %s" % posted_message)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
#!/usr/bin/env python2
|
#!/usr/bin/env python
|
||||||
|
|
||||||
#
|
#
|
||||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
|
|
@ -55,7 +55,7 @@ def run_cmd(cmd, return_output=False):
|
||||||
cmd = cmd.split()
|
cmd = cmd.split()
|
||||||
try:
|
try:
|
||||||
if return_output:
|
if return_output:
|
||||||
return subprocess_check_output(cmd)
|
return subprocess_check_output(cmd).decode(sys.getdefaultencoding())
|
||||||
else:
|
else:
|
||||||
return subprocess_check_call(cmd)
|
return subprocess_check_call(cmd)
|
||||||
except subprocess.CalledProcessError as e:
|
except subprocess.CalledProcessError as e:
|
||||||
|
|
Loading…
Reference in a new issue