[SPARK-27889][INFRA] Make development scripts under dev/ support Python 3

## What changes were proposed in this pull request?

I made an audit and update all dev scripts to support python3. (except `merge_spark_pr.py` which already updated)

## How was this patch tested?

Manual.

Closes #25289 from WeichenXu123/dev_py3.

Authored-by: WeichenXu <weichen.xu@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>

dev/github_jira_sync.py

@@ -22,7 +22,14 @@ import json
 import os
 import re
 import sys
-import urllib2
+if sys.version < '3':
+    from urllib2 import urlopen
+    from urllib2 import Request
+    from urllib2 import HTTPError
+else:
+    from urllib.request import urlopen
+    from urllib.request import Request
+    from urllib.error import HTTPError
 
 try:
     import jira.client
@@ -52,16 +59,16 @@ MAX_FILE = ".github-jira-max"
 
 def get_url(url):
     try:
-        request = urllib2.Request(url)
+        request = Request(url)
         request.add_header('Authorization', 'token %s' % GITHUB_OAUTH_KEY)
-        return urllib2.urlopen(request)
-    except urllib2.HTTPError:
+        return urlopen(request)
+    except HTTPError:
         print("Unable to fetch URL, exiting: %s" % url)
         sys.exit(-1)
 
 
 def get_json(urllib_response):
-    return json.load(urllib_response)
+    return json.loads(urllib_response.read().decode("utf-8"))
 
 
 # Return a list of (JIRA id, JSON dict) tuples:
@@ -80,7 +87,7 @@ def get_jira_prs():
                 result = result + [(jira, pull)]
 
         # Check if there is another page
-        link_headers = filter(lambda k: k.startswith("Link"), page.info().headers)
+        link_headers = list(filter(lambda k: k.startswith("Link"), page.headers))
         if not link_headers or "next" not in link_headers[0]:
             has_next_page = False
         else:
@@ -122,13 +129,13 @@ def reset_pr_labels(pr_num, jira_components):
     url = '%s/issues/%s/labels' % (GITHUB_API_BASE, pr_num)
     labels = ', '.join(('"%s"' % c) for c in jira_components)
     try:
-        request = urllib2.Request(url, data='{"labels":[%s]}' % labels)
+        request = Request(url, data=('{"labels":[%s]}' % labels).encode('utf-8'))
         request.add_header('Content-Type', 'application/json')
         request.add_header('Authorization', 'token %s' % GITHUB_OAUTH_KEY)
         request.get_method = lambda: 'PUT'
-        urllib2.urlopen(request)
+        urlopen(request)
         print("Set %s with labels %s" % (pr_num, labels))
-    except urllib2.HTTPError:
+    except HTTPError:
         print("Unable to update PR labels, exiting: %s" % url)
         sys.exit(-1)
 

dev/merge_spark_pr.py

@@ -97,9 +97,9 @@ def fail(msg):
 def run_cmd(cmd):
     print(cmd)
     if isinstance(cmd, list):
-        return subprocess.check_output(cmd).decode(sys.stdout.encoding)
+        return subprocess.check_output(cmd).decode(sys.getdefaultencoding())
     else:
-        return subprocess.check_output(cmd.split(" ")).decode(sys.stdout.encoding)
+        return subprocess.check_output(cmd.split(" ")).decode(sys.getdefaultencoding())
 
 
 def continue_maybe(prompt):

dev/run-tests-jenkins.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 
 #
 # Licensed to the Apache Software Foundation (ASF) under one or more
@@ -21,9 +21,17 @@ from __future__ import print_function
 import os
 import sys
 import json
-import urllib2
 import functools
 import subprocess
+if sys.version < '3':
+    from urllib2 import urlopen
+    from urllib2 import Request
+    from urllib2 import HTTPError, URLError
+else:
+    from urllib.request import urlopen
+    from urllib.request import Request
+    from urllib.error import HTTPError, URLError
+
 
 from sparktestsupport import SPARK_HOME, ERROR_CODES
 from sparktestsupport.shellutils import run_cmd
@@ -44,25 +52,25 @@ def post_message_to_github(msg, ghprb_pull_id):
     github_oauth_key = os.environ["GITHUB_OAUTH_KEY"]
 
     posted_message = json.dumps({"body": msg})
-    request = urllib2.Request(url,
-                              headers={
-                                  "Authorization": "token %s" % github_oauth_key,
-                                  "Content-Type": "application/json"
-                              },
-                              data=posted_message)
+    request = Request(url,
+                      headers={
+                          "Authorization": "token %s" % github_oauth_key,
+                          "Content-Type": "application/json"
+                      },
+                      data=posted_message.encode('utf-8'))
     try:
-        response = urllib2.urlopen(request)
+        response = urlopen(request)
 
         if response.getcode() == 201:
             print(" > Post successful.")
-    except urllib2.HTTPError as http_e:
+    except HTTPError as http_e:
         print_err("Failed to post message to Github.")
         print_err(" > http_code: %s" % http_e.code)
         print_err(" > api_response: %s" % http_e.read())
         print_err(" > data: %s" % posted_message)
-    except urllib2.URLError as url_e:
+    except URLError as url_e:
         print_err("Failed to post message to Github.")
-        print_err(" > urllib2_status: %s" % url_e.reason[1])
+        print_err(" > urllib_status: %s" % url_e.reason[1])
         print_err(" > data: %s" % posted_message)
 
 

dev/run-tests.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 
 #
 # Licensed to the Apache Software Foundation (ASF) under one or more

dev/sparktestsupport/shellutils.py

@@ -55,7 +55,7 @@ def run_cmd(cmd, return_output=False):
         cmd = cmd.split()
     try:
         if return_output:
-            return subprocess_check_output(cmd)
+            return subprocess_check_output(cmd).decode(sys.getdefaultencoding())
         else:
             return subprocess_check_call(cmd)
     except subprocess.CalledProcessError as e: