ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-7915] [SQL] Support specifying the column list for target table in CTAS

Browse source 
```
create table t1 (a int, b string) as select key, value from src;

desc t1;
key	int	NULL
value	string	NULL
```

Thus Hive doesn't support specifying the column list for target table in CTAS, however, we should either throwing exception explicity, or supporting the this feature, we just pick up the later one, which seems useful and straightforward.

Author: Cheng Hao <hao.cheng@intel.com>

Closes #6458 from chenghao-intel/ctas_column and squashes the following commits:

d1fa9b6 [Cheng Hao] bug in unittest
4e701aa [Cheng Hao] update as feedback
f305ec1 [Cheng Hao] support specifying the column list for target table in CTAS
This commit is contained in:
Cheng Hao 2015-06-11 14:03:08 -07:00 committed by Michael Armbrust
parent c8d551d546
commit 040f223c5b
2 changed files with 30 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/CreateTableAsSelect.scala
View file

@ -50,17 +50,25 @@ case class CreateTableAsSelect(
import org.apache.hadoop.io.Text
import org.apache.hadoop.mapred.TextInputFormat
val withSchema =
val withFormat =
tableDesc.copy(
schema =
query.output.map(c =>
HiveColumn(c.name, HiveMetastoreTypes.toMetastoreType(c.dataType), null)),
inputFormat =
tableDesc.inputFormat.orElse(Some(classOf[TextInputFormat].getName)),
outputFormat =
tableDesc.outputFormat
.orElse(Some(classOf[HiveIgnoreKeyTextOutputFormat[Text, Text]].getName)),
serde = tableDesc.serde.orElse(Some(classOf[LazySimpleSerDe].getName())))
val withSchema = if (withFormat.schema.isEmpty) {
// Hive doesn't support specifying the column list for target table in CTAS
// However we don't think SparkSQL should follow that.
tableDesc.copy(schema =
query.output.map(c =>
HiveColumn(c.name, HiveMetastoreTypes.toMetastoreType(c.dataType), null)))
} else {
withFormat
}
hiveContext.catalog.client.createTable(withSchema)
// Get the Metastore Relation

18
sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
View file

@ -360,6 +360,24 @@ class SQLQuerySuite extends QueryTest {
}
}
test("specifying the column list for CTAS") {
Seq((1, "111111"), (2, "222222")).toDF("key", "value").registerTempTable("mytable1")
sql("create table gen__tmp(a int, b string) as select key, value from mytable1")
checkAnswer(
sql("SELECT a, b from gen__tmp"),
sql("select key, value from mytable1").collect())
sql("DROP TABLE gen__tmp")
sql("create table gen__tmp(a double, b double) as select key, value from mytable1")
checkAnswer(
sql("SELECT a, b from gen__tmp"),
sql("select cast(key as double), cast(value as double) from mytable1").collect())
sql("DROP TABLE gen__tmp")
sql("drop table mytable1")
}
test("command substitution") {
sql("set tbl=src")
checkAnswer(