[MINOR][SQL][DOCS] Update sql/README.md and remove some unused imports in sql module.
## What changes were proposed in this pull request? This PR updates `sql/README.md` according to the latest console output and removes some unused imports in `sql` module. This is done by manually, so there is no guarantee to remove all unused imports. ## How was this patch tested? Manual. Author: Dongjoon Hyun <dongjoon@apache.org> Closes #11907 from dongjoon-hyun/update_sql_module.
This commit is contained in:
Dongjoon Hyun
Reynold Xin
parent
75dc29620e
commit
1a22cf1e9b
|
|
@ -47,7 +47,7 @@ An interactive scala console can be invoked by running `build/sbt hive/console`.
|
|||
From here you can execute queries with HiveQl and manipulate DataFrame by using DSL.
|
||||
|
||||
```scala
|
||||
catalyst$ build/sbt hive/console
|
||||
$ build/sbt hive/console
|
||||
|
||||
[info] Starting scala interpreter...
|
||||
import org.apache.spark.sql.catalyst.analysis._
|
||||
|
|
@ -61,22 +61,23 @@ import org.apache.spark.sql.execution
|
|||
import org.apache.spark.sql.functions._
|
||||
import org.apache.spark.sql.hive._
|
||||
import org.apache.spark.sql.hive.test.TestHive._
|
||||
import org.apache.spark.sql.hive.test.TestHive.implicits._
|
||||
import org.apache.spark.sql.types._
|
||||
Type in expressions to have them evaluated.
|
||||
Type :help for more information.
|
||||
|
||||
scala> val query = sql("SELECT * FROM (SELECT * FROM src) a")
|
||||
query: org.apache.spark.sql.DataFrame = org.apache.spark.sql.DataFrame@74448eed
|
||||
query: org.apache.spark.sql.DataFrame = [key: int, value: string]
|
||||
```
|
||||
|
||||
Query results are `DataFrames` and can be operated as such.
|
||||
```
|
||||
scala> query.collect()
|
||||
res2: Array[org.apache.spark.sql.Row] = Array([238,val_238], [86,val_86], [311,val_311], [27,val_27]...
|
||||
res0: Array[org.apache.spark.sql.Row] = Array([238,val_238], [86,val_86], [311,val_311], [27,val_27]...
|
||||
```
|
||||
|
||||
You can also build further queries on top of these `DataFrames` using the query DSL.
|
||||
```
|
||||
scala> query.where(query("key") > 30).select(avg(query("key"))).collect()
|
||||
res3: Array[org.apache.spark.sql.Row] = Array([274.79025423728814])
|
||||
res1: Array[org.apache.spark.sql.Row] = Array([274.79025423728814])
|
||||
```
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ import org.apache.spark.sql.AnalysisException
|
|||
import org.apache.spark.sql.catalyst.analysis._
|
||||
import org.apache.spark.sql.catalyst.expressions._
|
||||
import org.apache.spark.sql.catalyst.plans.QueryPlan
|
||||
import org.apache.spark.sql.catalyst.trees.{CurrentOrigin, TreeNode}
|
||||
import org.apache.spark.sql.catalyst.trees.CurrentOrigin
|
||||
import org.apache.spark.sql.types.StructType
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -32,13 +32,9 @@ import org.apache.spark.internal.Logging
|
|||
import org.apache.spark.rdd.RDD
|
||||
import org.apache.spark.scheduler.{SparkListener, SparkListenerApplicationEnd}
|
||||
import org.apache.spark.sql.catalyst._
|
||||
import org.apache.spark.sql.catalyst.analysis._
|
||||
import org.apache.spark.sql.catalyst.encoders.encoderFor
|
||||
import org.apache.spark.sql.catalyst.expressions._
|
||||
import org.apache.spark.sql.catalyst.optimizer.Optimizer
|
||||
import org.apache.spark.sql.catalyst.parser.ParserInterface
|
||||
import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan, Range}
|
||||
import org.apache.spark.sql.catalyst.rules.RuleExecutor
|
||||
import org.apache.spark.sql.execution._
|
||||
import org.apache.spark.sql.execution.command.ShowTablesCommand
|
||||
import org.apache.spark.sql.execution.datasources._
|
||||
|
|
|
|||
|
|
@ -21,11 +21,7 @@ import scala.language.implicitConversions
|
|||
import scala.reflect.runtime.universe.TypeTag
|
||||
|
||||
import org.apache.spark.rdd.RDD
|
||||
import org.apache.spark.sql.catalyst.InternalRow
|
||||
import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
|
||||
import org.apache.spark.sql.catalyst.expressions.SpecificMutableRow
|
||||
import org.apache.spark.sql.types._
|
||||
import org.apache.spark.unsafe.types.UTF8String
|
||||
|
||||
/**
|
||||
* A collection of implicit methods for converting common Scala objects into [[DataFrame]]s.
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@ package org.apache.spark.sql.execution
|
|||
|
||||
import org.apache.spark.sql.{AnalysisException, SaveMode}
|
||||
import org.apache.spark.sql.catalyst.TableIdentifier
|
||||
import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
|
||||
import org.apache.spark.sql.catalyst.parser._
|
||||
import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, OneRowRelation}
|
||||
import org.apache.spark.sql.execution.command._
|
||||
|
|
|
|||
|
|
@ -22,7 +22,6 @@ import org.apache.spark.rdd.RDD
|
|||
import org.apache.spark.sql.catalyst.InternalRow
|
||||
import org.apache.spark.sql.catalyst.expressions._
|
||||
import org.apache.spark.sql.catalyst.expressions.codegen._
|
||||
import org.apache.spark.sql.catalyst.plans.Inner
|
||||
import org.apache.spark.sql.catalyst.plans.physical.Partitioning
|
||||
import org.apache.spark.sql.catalyst.rules.Rule
|
||||
import org.apache.spark.sql.catalyst.util.toCommentSafeString
|
||||
|
|
|
|||
|
|
@ -31,7 +31,6 @@ import org.apache.spark.sql.catalyst.util._
|
|||
import org.apache.spark.sql.execution.{LogicalRDD, Queryable}
|
||||
import org.apache.spark.sql.execution.columnar.InMemoryRelation
|
||||
import org.apache.spark.sql.execution.datasources.LogicalRelation
|
||||
import org.apache.spark.sql.internal.SQLConf
|
||||
|
||||
abstract class QueryTest extends PlanTest {
|
||||
|
||||
|
|
|
|||
|
|
@ -28,7 +28,6 @@ import scala.util.control.NonFatal
|
|||
|
||||
import org.scalatest.Assertions
|
||||
import org.scalatest.concurrent.{Eventually, Timeouts}
|
||||
import org.scalatest.concurrent.Eventually.timeout
|
||||
import org.scalatest.concurrent.PatienceConfiguration.Timeout
|
||||
import org.scalatest.exceptions.TestFailedDueToTimeoutException
|
||||
import org.scalatest.time.Span
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Repartition}
|
|||
import org.apache.spark.sql.catalyst.plans.physical._
|
||||
import org.apache.spark.sql.execution.columnar.InMemoryRelation
|
||||
import org.apache.spark.sql.execution.exchange.{EnsureRequirements, ReusedExchange, ReuseExchange, ShuffleExchange}
|
||||
import org.apache.spark.sql.execution.joins.{BroadcastHashJoin, ShuffledHashJoin, SortMergeJoin}
|
||||
import org.apache.spark.sql.execution.joins.{BroadcastHashJoin, SortMergeJoin}
|
||||
import org.apache.spark.sql.functions._
|
||||
import org.apache.spark.sql.internal.SQLConf
|
||||
import org.apache.spark.sql.test.SharedSQLContext
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@
|
|||
package org.apache.spark.sql.execution
|
||||
|
||||
import org.apache.spark.sql.Row
|
||||
import org.apache.spark.sql.catalyst.dsl.expressions._
|
||||
import org.apache.spark.sql.execution.aggregate.TungstenAggregate
|
||||
import org.apache.spark.sql.execution.joins.BroadcastHashJoin
|
||||
import org.apache.spark.sql.functions.{avg, broadcast, col, max}
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@
|
|||
|
||||
package org.apache.spark.sql.execution.datasources
|
||||
|
||||
import java.io.{File, FilenameFilter}
|
||||
import java.io.File
|
||||
|
||||
import org.apache.hadoop.fs.FileStatus
|
||||
import org.apache.hadoop.mapreduce.Job
|
||||
|
|
@ -28,7 +28,7 @@ import org.apache.spark.sql._
|
|||
import org.apache.spark.sql.catalyst.InternalRow
|
||||
import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionSet, PredicateHelper}
|
||||
import org.apache.spark.sql.catalyst.util
|
||||
import org.apache.spark.sql.execution.{DataSourceScan, PhysicalRDD}
|
||||
import org.apache.spark.sql.execution.DataSourceScan
|
||||
import org.apache.spark.sql.functions._
|
||||
import org.apache.spark.sql.internal.SQLConf
|
||||
import org.apache.spark.sql.sources._
|
||||
|
|
|
|||
|
|
@ -22,7 +22,6 @@ import scala.collection.JavaConverters._
|
|||
import scala.util.Try
|
||||
|
||||
import org.apache.spark.{SparkConf, SparkContext}
|
||||
import org.apache.spark.sql.catalyst.InternalRow
|
||||
import org.apache.spark.sql.internal.SQLConf
|
||||
import org.apache.spark.sql.SQLContext
|
||||
import org.apache.spark.util.{Benchmark, Utils}
|
||||
|
|
|
|||
|
|
@ -22,7 +22,6 @@ import java.io.{ByteArrayInputStream, ByteArrayOutputStream, ObjectInputStream,
|
|||
import org.apache.spark.SparkFunSuite
|
||||
import org.apache.spark.sql.catalyst.InternalRow
|
||||
import org.apache.spark.sql.catalyst.expressions._
|
||||
import org.apache.spark.sql.execution.metric.SQLMetrics
|
||||
import org.apache.spark.sql.test.SharedSQLContext
|
||||
import org.apache.spark.sql.types.{IntegerType, StructField, StructType}
|
||||
import org.apache.spark.util.collection.CompactBuffer
|
||||
|
|
|
|||
Loading…
Reference in a new issue