From 013f2b7439360950c71341edeb3b26fa51613f01 Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Thu, 19 Aug 2021 13:27:58 +0800
Subject: [PATCH] [SPARK-36512][UI][TESTS] Fix UISeleniumSuite in
 sql/hive-thriftserver

### What changes were proposed in this pull request?

This PR fixes an issue that `UISeleniumSuite` in `sql/hive-thriftserver` doesn't work even though the ignored test is enabled due to the following two reasons.

(1) The suite waits for thriftserver starting up by reading a startup message from stdin but the expected message is never read.
(2) The URL and CSS selector for test are wrong.

To resolve (1), this PR adopt the way that `HiveThriftServer2Suite` does.
https://github.com/apache/spark/blob/3f8ec0dae4ddfd7ee55370dad5d44d03a9f10387/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala#L1222-L1248

This PR also enables the ignored test. Let's disable it again in the following PR if it's still flaky.

### Why are the changes needed?

Bug fix.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

CIs.

Closes #33741 from sarutak/fix-thrift-uiseleniumsuite.

Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com>
Signed-off-by: Gengliang Wang <gengliang@apache.org>
---
 .../hive/thriftserver/UISeleniumSuite.scala   | 31 ++++++++++++++++---
 1 file changed, 26 insertions(+), 5 deletions(-)

diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala
index 2d0edb8eb8..0747a11225 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala
@@ -17,8 +17,12 @@
 
 package org.apache.spark.sql.hive.thriftserver
 
+import java.io.File
+import java.nio.charset.StandardCharsets
+
 import scala.util.Random
 
+import com.google.common.io.Files
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars
 import org.openqa.selenium.WebDriver
 import org.openqa.selenium.htmlunit.HtmlUnitDriver
@@ -64,21 +68,38 @@ class UISeleniumSuite
       ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT
     }
 
+    val driverClassPath = {
+      // Writes a temporary log4j.properties and prepend it to driver classpath, so that it
+      // overrides all other potential log4j configurations contained in other dependency jar files.
+      val tempLog4jConf = org.apache.spark.util.Utils.createTempDir().getCanonicalPath
+
+      Files.write(
+        """log4j.rootCategory=INFO, console
+          |log4j.appender.console=org.apache.log4j.ConsoleAppender
+          |log4j.appender.console.target=System.err
+          |log4j.appender.console.layout=org.apache.log4j.PatternLayout
+          |log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
+        """.stripMargin,
+        new File(s"$tempLog4jConf/log4j.properties"),
+        StandardCharsets.UTF_8)
+
+      tempLog4jConf
+    }
+
     s"""$startScript
         |  --master local
-        |  --hiveconf hive.root.logger=INFO,console
         |  --hiveconf ${ConfVars.METASTORECONNECTURLKEY}=$metastoreJdbcUri
         |  --hiveconf ${ConfVars.METASTOREWAREHOUSE}=$warehousePath
         |  --hiveconf ${ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST}=localhost
         |  --hiveconf ${ConfVars.HIVE_SERVER2_TRANSPORT_MODE}=$mode
         |  --hiveconf $portConf=0
-        |  --driver-class-path ${sys.props("java.class.path")}
+        |  --driver-class-path $driverClassPath
         |  --conf spark.ui.enabled=true
         |  --conf spark.ui.port=$uiPort
      """.stripMargin.split("\\s+").toSeq
   }
 
-  ignore("thrift server ui test") {
+  test("thrift server ui test") {
     withJdbcStatement("test_map") { statement =>
       val baseURL = s"http://localhost:$uiPort"
 
@@ -94,13 +115,13 @@ class UISeleniumSuite
       }
 
       eventually(timeout(10.seconds), interval(50.milliseconds)) {
-        go to (baseURL + "/sql")
+        go to (baseURL + "/sqlserver")
         find(id("sessionstat")) should not be None
         find(id("sqlstat")) should not be None
 
         // check whether statements exists
         queries.foreach { line =>
-          findAll(cssSelector("""ul table tbody tr td""")).map(_.text).toList should contain (line)
+          findAll(cssSelector("span.description-input")).map(_.text).toList should contain (line)
         }
       }
     }