[SPARK-11474][SQL] change fetchSize to fetchsize
In DefaultDataSource.scala, it has override def createRelation( sqlContext: SQLContext, parameters: Map[String, String]): BaseRelation The parameters is CaseInsensitiveMap. After this line parameters.foreach(kv => properties.setProperty(kv._1, kv._2)) properties is set to all lower case key/value pairs and fetchSize becomes fetchsize. However, in compute method in JDBCRDD, it has val fetchSize = properties.getProperty("fetchSize", "0").toInt so fetchSize value is always 0 and never gets set correctly. Author: Huaxin Gao <huaxing@oc0558782468.ibm.com> Closes #9473 from huaxingao/spark-11474.
This commit is contained in:
parent
a4b5cefcf1
commit
b072ff4d1d
|
@ -347,6 +347,7 @@ private[sql] class JDBCRDD(
|
|||
|
||||
/**
|
||||
* Runs the SQL query against the JDBC driver.
|
||||
*
|
||||
*/
|
||||
override def compute(thePart: Partition, context: TaskContext): Iterator[InternalRow] =
|
||||
new Iterator[InternalRow] {
|
||||
|
@ -368,7 +369,7 @@ private[sql] class JDBCRDD(
|
|||
val sqlText = s"SELECT $columnList FROM $fqTable $myWhereClause"
|
||||
val stmt = conn.prepareStatement(sqlText,
|
||||
ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY)
|
||||
val fetchSize = properties.getProperty("fetchSize", "0").toInt
|
||||
val fetchSize = properties.getProperty("fetchsize", "0").toInt
|
||||
stmt.setFetchSize(fetchSize)
|
||||
val rs = stmt.executeQuery()
|
||||
|
||||
|
|
Loading…
Reference in a new issue