[SPARK-7915] [SQL] Support specifying the column list for target table in CTAS
``` create table t1 (a int, b string) as select key, value from src; desc t1; key int NULL value string NULL ``` Thus Hive doesn't support specifying the column list for target table in CTAS, however, we should either throwing exception explicity, or supporting the this feature, we just pick up the later one, which seems useful and straightforward. Author: Cheng Hao <hao.cheng@intel.com> Closes #6458 from chenghao-intel/ctas_column and squashes the following commits: d1fa9b6 [Cheng Hao] bug in unittest 4e701aa [Cheng Hao] update as feedback f305ec1 [Cheng Hao] support specifying the column list for target table in CTAS
This commit is contained in:
parent
c8d551d546
commit
040f223c5b
|
@ -50,17 +50,25 @@ case class CreateTableAsSelect(
|
|||
import org.apache.hadoop.io.Text
|
||||
import org.apache.hadoop.mapred.TextInputFormat
|
||||
|
||||
val withSchema =
|
||||
val withFormat =
|
||||
tableDesc.copy(
|
||||
schema =
|
||||
query.output.map(c =>
|
||||
HiveColumn(c.name, HiveMetastoreTypes.toMetastoreType(c.dataType), null)),
|
||||
inputFormat =
|
||||
tableDesc.inputFormat.orElse(Some(classOf[TextInputFormat].getName)),
|
||||
outputFormat =
|
||||
tableDesc.outputFormat
|
||||
.orElse(Some(classOf[HiveIgnoreKeyTextOutputFormat[Text, Text]].getName)),
|
||||
serde = tableDesc.serde.orElse(Some(classOf[LazySimpleSerDe].getName())))
|
||||
|
||||
val withSchema = if (withFormat.schema.isEmpty) {
|
||||
// Hive doesn't support specifying the column list for target table in CTAS
|
||||
// However we don't think SparkSQL should follow that.
|
||||
tableDesc.copy(schema =
|
||||
query.output.map(c =>
|
||||
HiveColumn(c.name, HiveMetastoreTypes.toMetastoreType(c.dataType), null)))
|
||||
} else {
|
||||
withFormat
|
||||
}
|
||||
|
||||
hiveContext.catalog.client.createTable(withSchema)
|
||||
|
||||
// Get the Metastore Relation
|
||||
|
|
|
@ -360,6 +360,24 @@ class SQLQuerySuite extends QueryTest {
|
|||
}
|
||||
}
|
||||
|
||||
test("specifying the column list for CTAS") {
|
||||
Seq((1, "111111"), (2, "222222")).toDF("key", "value").registerTempTable("mytable1")
|
||||
|
||||
sql("create table gen__tmp(a int, b string) as select key, value from mytable1")
|
||||
checkAnswer(
|
||||
sql("SELECT a, b from gen__tmp"),
|
||||
sql("select key, value from mytable1").collect())
|
||||
sql("DROP TABLE gen__tmp")
|
||||
|
||||
sql("create table gen__tmp(a double, b double) as select key, value from mytable1")
|
||||
checkAnswer(
|
||||
sql("SELECT a, b from gen__tmp"),
|
||||
sql("select cast(key as double), cast(value as double) from mytable1").collect())
|
||||
sql("DROP TABLE gen__tmp")
|
||||
|
||||
sql("drop table mytable1")
|
||||
}
|
||||
|
||||
test("command substitution") {
|
||||
sql("set tbl=src")
|
||||
checkAnswer(
|
||||
|
|
Loading…
Reference in a new issue