ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-2817] [SQL] add "show create table" support

Browse source 
In spark sql component, the "show create table" syntax had been disabled.
We thought it is a useful funciton to describe a hive table.

Author: tianyi <tianyi@asiainfo-linkage.com>
Author: tianyi <tianyi@asiainfo.com>
Author: tianyi <tianyi.asiainfo@gmail.com>

Closes #1760 from tianyi/spark-2817 and squashes the following commits:

7d28b15 [tianyi] [SPARK-2817] fix too short prefix problem
cbffe8b [tianyi] [SPARK-2817] fix the case problem
565ec14 [tianyi] [SPARK-2817] fix the case problem
60d48a9 [tianyi] [SPARK-2817] use system temporary folder instead of temporary files in the source tree, and also clean some empty line
dbe1031 [tianyi] [SPARK-2817] move some code out of function rewritePaths, as it may be called multiple times
9b2ba11 [tianyi] [SPARK-2817] fix the line length problem
9f97586 [tianyi] [SPARK-2817] remove test.tmp.dir from pom.xml
bfc2999 [tianyi] [SPARK-2817] add "File.separator" support, create a "testTmpDir" outside the rewritePaths
bde800a [tianyi] [SPARK-2817] add "${system:test.tmp.dir}" support add "last_modified_by" to nonDeterministicLineIndicators in HiveComparisonTest
bb82726 [tianyi] [SPARK-2817] remove test which requires a system from the whitelist.
bbf6b42 [tianyi] [SPARK-2817] add a systemProperties named "test.tmp.dir" to pass the test which contains "${system:test.tmp.dir}"
a337bd6 [tianyi] [SPARK-2817] add "show create table" support
a03db77 [tianyi] [SPARK-2817] add "show create table" support
This commit is contained in:
tianyi 2014-08-13 16:50:02 -07:00 committed by Michael Armbrust
parent bdc7a1a474
commit 13f54e2b97
37 changed files with 199 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
View file

@ -635,6 +635,14 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
"serde_regex",
"serde_reported_schema",
"set_variable_sub",
"show_create_table_partitioned",
"show_create_table_delimited",
"show_create_table_alter",
"show_create_table_view",
"show_create_table_serde",
"show_create_table_db_table",
"show_create_table_does_not_exist",
"show_create_table_index",
"show_describe_func_quotes",
"show_functions",
"show_partitions",

1
sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala
View file

@ -53,6 +53,7 @@ private[hive] object HiveQl {
protected val nativeCommands = Seq(
"TOK_DESCFUNCTION",
"TOK_DESCDATABASE",
"TOK_SHOW_CREATETABLE",
"TOK_SHOW_TABLESTATUS",
"TOK_SHOWDATABASES",
"TOK_SHOWFUNCTIONS",

8
sql/hive/src/main/scala/org/apache/spark/sql/hive/TestHive.scala
View file

@ -70,6 +70,13 @@ class TestHiveContext(sc: SparkContext) extends HiveContext(sc) {
setConf("hive.metastore.warehouse.dir", warehousePath)
}
val testTempDir = File.createTempFile("testTempFiles", "spark.hive.tmp")
testTempDir.delete()
testTempDir.mkdir()
// For some hive test case which contain ${system:test.tmp.dir}
System.setProperty("test.tmp.dir", testTempDir.getCanonicalPath)
configure() // Must be called before initializing the catalog below.
/** The location of the compiled hive distribution */
@ -109,6 +116,7 @@ class TestHiveContext(sc: SparkContext) extends HiveContext(sc) {
hiveFilesTemp.mkdir()
hiveFilesTemp.deleteOnExit()
val inRepoTests = if (System.getProperty("user.dir").endsWith("sql" + File.separator + "hive")) {
new File("src" + File.separator + "test" + File.separator + "resources" + File.separator)
} else {

0
sql/hive/src/test/resources/golden/show_create_table_alter-0-813886d6cf0875c62e89cd1d06b8b0b4 Normal file
View file

18
sql/hive/src/test/resources/golden/show_create_table_alter-1-2a91d52719cf4552ebeb867204552a26 Normal file
View file

@ -0,0 +1,18 @@
CREATE EXTERNAL TABLE tmp_showcrt1(
key smallint,
value float)
CLUSTERED BY (
key)
SORTED BY (
value DESC)
INTO 5 BUCKETS
ROW FORMAT SERDE
'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
STORED AS INPUTFORMAT
'org.apache.hadoop.mapred.TextInputFormat'
OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
LOCATION
'file:/tmp/sparkHiveWarehouse1280221975983654134/tmp_showcrt1'
TBLPROPERTIES (
'transient_lastDdlTime'='1407132100')

0
sql/hive/src/test/resources/golden/show_create_table_alter-10-259d978ed9543204c8b9c25b6e25b0de Normal file
View file

0
sql/hive/src/test/resources/golden/show_create_table_alter-2-928cc85c025440b731e5ee33e437e404 Normal file
View file

22
sql/hive/src/test/resources/golden/show_create_table_alter-3-2a91d52719cf4552ebeb867204552a26 Normal file
View file

@ -0,0 +1,22 @@
CREATE TABLE tmp_showcrt1(
key smallint,
value float)
COMMENT 'temporary table'
CLUSTERED BY (
key)
SORTED BY (
value DESC)
INTO 5 BUCKETS
ROW FORMAT SERDE
'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
STORED AS INPUTFORMAT
'org.apache.hadoop.mapred.TextInputFormat'
OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
LOCATION
'file:/tmp/sparkHiveWarehouse1280221975983654134/tmp_showcrt1'
TBLPROPERTIES (
'EXTERNAL'='FALSE',
'last_modified_by'='tianyi',
'last_modified_time'='1407132100',
'transient_lastDdlTime'='1407132100')

0
sql/hive/src/test/resources/golden/show_create_table_alter-4-c2cb6a7d942d4dddd1aababccb1239f9 Normal file
View file

21
sql/hive/src/test/resources/golden/show_create_table_alter-5-2a91d52719cf4552ebeb867204552a26 Normal file
View file

@ -0,0 +1,21 @@
CREATE EXTERNAL TABLE tmp_showcrt1(
key smallint,
value float)
COMMENT 'changed comment'
CLUSTERED BY (
key)
SORTED BY (
value DESC)
INTO 5 BUCKETS
ROW FORMAT SERDE
'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
STORED AS INPUTFORMAT
'org.apache.hadoop.mapred.TextInputFormat'
OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
LOCATION
'file:/tmp/sparkHiveWarehouse1280221975983654134/tmp_showcrt1'
TBLPROPERTIES (
'last_modified_by'='tianyi',
'last_modified_time'='1407132100',
'transient_lastDdlTime'='1407132100')

0
sql/hive/src/test/resources/golden/show_create_table_alter-6-fdd1bd7f9acf0b2c8c9b7503d4046cb Normal file
View file

21
sql/hive/src/test/resources/golden/show_create_table_alter-7-2a91d52719cf4552ebeb867204552a26 Normal file
View file

@ -0,0 +1,21 @@
CREATE EXTERNAL TABLE tmp_showcrt1(
key smallint,
value float)
COMMENT 'changed comment'
CLUSTERED BY (
key)
SORTED BY (
value DESC)
INTO 5 BUCKETS
ROW FORMAT SERDE
'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
STORED AS INPUTFORMAT
'org.apache.hadoop.mapred.TextInputFormat'
OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
LOCATION
'file:/tmp/sparkHiveWarehouse1280221975983654134/tmp_showcrt1'
TBLPROPERTIES (
'last_modified_by'='tianyi',
'last_modified_time'='1407132101',
'transient_lastDdlTime'='1407132101')

0
sql/hive/src/test/resources/golden/show_create_table_alter-8-22ab6ed5b15a018756f454dd2294847e Normal file
View file

21
sql/hive/src/test/resources/golden/show_create_table_alter-9-2a91d52719cf4552ebeb867204552a26 Normal file
View file

@ -0,0 +1,21 @@
CREATE EXTERNAL TABLE tmp_showcrt1(
key smallint,
value float)
COMMENT 'changed comment'
CLUSTERED BY (
key)
SORTED BY (
value DESC)
INTO 5 BUCKETS
ROW FORMAT SERDE
'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
STORED BY
'org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler'
WITH SERDEPROPERTIES (
'serialization.format'='1')
LOCATION
'file:/tmp/sparkHiveWarehouse1280221975983654134/tmp_showcrt1'
TBLPROPERTIES (
'last_modified_by'='tianyi',
'last_modified_time'='1407132101',
'transient_lastDdlTime'='1407132101')

0
sql/hive/src/test/resources/golden/show_create_table_db_table-0-67509558a4b2d39b25787cca33f52635 Normal file
View file

2
sql/hive/src/test/resources/golden/show_create_table_db_table-1-549981e00a3d95f03dd5a9ef6044aa20 Normal file
View file

@ -0,0 +1,2 @@
default
tmp_feng

0
sql/hive/src/test/resources/golden/show_create_table_db_table-2-34ae7e611d0aedbc62b6e420347abee Normal file
View file

0
sql/hive/src/test/resources/golden/show_create_table_db_table-3-7a9e67189d3d4151f23b12c22bde06b5 Normal file
View file

13
sql/hive/src/test/resources/golden/show_create_table_db_table-4-b585371b624cbab2616a49f553a870a0 Normal file
View file

@ -0,0 +1,13 @@
CREATE TABLE tmp_feng.tmp_showcrt(
key string,
value int)
ROW FORMAT SERDE
'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
STORED AS INPUTFORMAT
'org.apache.hadoop.mapred.TextInputFormat'
OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
LOCATION
'file:/tmp/sparkHiveWarehouse1280221975983654134/tmp_feng.db/tmp_showcrt'
TBLPROPERTIES (
'transient_lastDdlTime'='1407132107')

0
sql/hive/src/test/resources/golden/show_create_table_db_table-5-964757b7e7f2a69fe36132c1a5712199 Normal file
View file

0
sql/hive/src/test/resources/golden/show_create_table_db_table-6-ac09cf81e7e734cf10406f30b9fa566e Normal file
View file

0
sql/hive/src/test/resources/golden/show_create_table_delimited-0-97228478b9925f06726ceebb6571bf34 Normal file
View file

17
sql/hive/src/test/resources/golden/show_create_table_delimited-1-2a91d52719cf4552ebeb867204552a26 Normal file
View file

@ -0,0 +1,17 @@
CREATE TABLE tmp_showcrt1(
key int,
value string,
newvalue bigint)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY ','
COLLECTION ITEMS TERMINATED BY '|'
MAP KEYS TERMINATED BY '%'
LINES TERMINATED BY '\n'
STORED AS INPUTFORMAT
'org.apache.hadoop.mapred.TextInputFormat'
OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
LOCATION
'file:/tmp/tmp_showcrt1'
TBLPROPERTIES (
'transient_lastDdlTime'='1407132730')

0
sql/hive/src/test/resources/golden/show_create_table_delimited-2-259d978ed9543204c8b9c25b6e25b0de Normal file
View file

0
sql/hive/src/test/resources/golden/show_create_table_partitioned-0-4be9a3b1ff0840786a1f001cba170a0c Normal file
View file

16
sql/hive/src/test/resources/golden/show_create_table_partitioned-1-2a91d52719cf4552ebeb867204552a26 Normal file
View file

@ -0,0 +1,16 @@
CREATE EXTERNAL TABLE tmp_showcrt1(
key string,
newvalue boolean COMMENT 'a new value')
COMMENT 'temporary table'
PARTITIONED BY (
value bigint COMMENT 'some value')
ROW FORMAT SERDE
'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
STORED AS INPUTFORMAT
'org.apache.hadoop.mapred.TextInputFormat'
OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
LOCATION
'file:/tmp/sparkHiveWarehouse1280221975983654134/tmp_showcrt1'
TBLPROPERTIES (
'transient_lastDdlTime'='1407132112')

0
sql/hive/src/test/resources/golden/show_create_table_partitioned-2-259d978ed9543204c8b9c25b6e25b0de Normal file
View file

0
sql/hive/src/test/resources/golden/show_create_table_serde-0-33f15d91810b75ee05c7b9dea0abb01c Normal file
View file

15
sql/hive/src/test/resources/golden/show_create_table_serde-1-2a91d52719cf4552ebeb867204552a26 Normal file
View file

@ -0,0 +1,15 @@
CREATE TABLE tmp_showcrt1(
key int,
value string,
newvalue bigint)
COMMENT 'temporary table'
ROW FORMAT SERDE
'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe'
STORED AS INPUTFORMAT
'org.apache.hadoop.hive.ql.io.RCFileInputFormat'
OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.RCFileOutputFormat'
LOCATION
'file:/tmp/sparkHiveWarehouse1280221975983654134/tmp_showcrt1'
TBLPROPERTIES (
'transient_lastDdlTime'='1407132115')

0
sql/hive/src/test/resources/golden/show_create_table_serde-2-259d978ed9543204c8b9c25b6e25b0de Normal file
View file

0
sql/hive/src/test/resources/golden/show_create_table_serde-3-fd12b3e0fe30f5d71c67676791b4a33b Normal file
View file

14
sql/hive/src/test/resources/golden/show_create_table_serde-4-2a91d52719cf4552ebeb867204552a26 Normal file
View file

@ -0,0 +1,14 @@
CREATE EXTERNAL TABLE tmp_showcrt1(
key string,
value boolean)
ROW FORMAT SERDE
'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe'
STORED BY
'org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler'
WITH SERDEPROPERTIES (
'serialization.format'='$',
'field.delim'=',')
LOCATION
'file:/tmp/sparkHiveWarehouse1280221975983654134/tmp_showcrt1'
TBLPROPERTIES (
'transient_lastDdlTime'='1407132115')

0
sql/hive/src/test/resources/golden/show_create_table_serde-5-259d978ed9543204c8b9c25b6e25b0de Normal file
View file

0
sql/hive/src/test/resources/golden/show_create_table_view-0-ecef6821e4e9212e553ca38142fd0250 Normal file
View file

1
sql/hive/src/test/resources/golden/show_create_table_view-1-1e931ea3fa6065107859ffbb29bb0ed7 Normal file
View file

@ -0,0 +1 @@
CREATE VIEW tmp_copy_src AS SELECT `src`.`key`, `src`.`value` FROM `default`.`src`

0
sql/hive/src/test/resources/golden/show_create_table_view-2-ed97e9e56d95c5b3db57485cba5ad17f Normal file
View file

1
sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
View file

@ -161,6 +161,7 @@ abstract class HiveComparisonTest
"transient_lastDdlTime",
"grantTime",
"lastUpdateTime",
"last_modified_by",
"last_modified_time",
"Owner:",
// The following are hive specific schema parameters which we do not need to match exactly.