[MINOR][MLLIB][SQL] Clean up unused variables and unused import
## What changes were proposed in this pull request? Clean up unused variables and unused import statements, unnecessary `return` and `toArray`, and some more style improvement, when I walk through the code examples. ## How was this patch tested? Testet manually on local laptop. Author: Xin Ren <iamshrek@126.com> Closes #14836 from keypointt/codeWalkThroughML.
This commit is contained in:
parent
d4eee9932e
commit
27209252f0
|
@ -100,7 +100,9 @@ class AccumulatorSuite extends SparkFunSuite with Matchers with LocalSparkContex
|
|||
val acc: Accumulator[Int] = sc.accumulator(0)
|
||||
|
||||
val d = sc.parallelize(1 to 20)
|
||||
an [Exception] should be thrownBy {d.foreach{x => acc.value = x}}
|
||||
intercept[SparkException] {
|
||||
d.foreach(x => acc.value = x)
|
||||
}
|
||||
}
|
||||
|
||||
test ("add value to collection accumulators") {
|
||||
|
@ -171,7 +173,7 @@ class AccumulatorSuite extends SparkFunSuite with Matchers with LocalSparkContex
|
|||
d.foreach {
|
||||
x => acc.localValue ++= x
|
||||
}
|
||||
acc.value should be ( (0 to maxI).toSet)
|
||||
acc.value should be ((0 to maxI).toSet)
|
||||
resetSparkContext()
|
||||
}
|
||||
}
|
||||
|
|
|
@ -136,7 +136,7 @@ class Interaction @Since("1.6.0") (@Since("1.6.0") override val uid: String) ext
|
|||
case _: VectorUDT =>
|
||||
val attrs = AttributeGroup.fromStructField(f).attributes.getOrElse(
|
||||
throw new SparkException("Vector attributes must be defined for interaction."))
|
||||
attrs.map(getNumFeatures).toArray
|
||||
attrs.map(getNumFeatures)
|
||||
}
|
||||
new FeatureEncoder(numFeatures)
|
||||
}.toArray
|
||||
|
|
|
@ -23,7 +23,7 @@ import org.json4s.JsonDSL._
|
|||
import org.json4s.jackson.JsonMethods._
|
||||
|
||||
import org.apache.spark.ml.{Pipeline, PipelineModel}
|
||||
import org.apache.spark.ml.attribute.{AttributeGroup}
|
||||
import org.apache.spark.ml.attribute.AttributeGroup
|
||||
import org.apache.spark.ml.feature.RFormula
|
||||
import org.apache.spark.ml.regression.{IsotonicRegression, IsotonicRegressionModel}
|
||||
import org.apache.spark.ml.util._
|
||||
|
|
|
@ -20,7 +20,7 @@ package org.apache.spark.ml.util
|
|||
import scala.collection.mutable
|
||||
|
||||
import org.apache.spark.SparkContext
|
||||
import org.apache.spark.util.LongAccumulator;
|
||||
import org.apache.spark.util.LongAccumulator
|
||||
|
||||
/**
|
||||
* Abstract class for stopwatches.
|
||||
|
|
|
@ -164,7 +164,7 @@ object ChiSqSelectorModel extends Loader[ChiSqSelectorModel] {
|
|||
case Row(feature: Int) => (feature)
|
||||
}.collect()
|
||||
|
||||
return new ChiSqSelectorModel(features)
|
||||
new ChiSqSelectorModel(features)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -438,10 +438,10 @@ object RandomRDDs {
|
|||
@DeveloperApi
|
||||
@Since("1.6.0")
|
||||
def randomJavaRDD[T](
|
||||
jsc: JavaSparkContext,
|
||||
generator: RandomDataGenerator[T],
|
||||
size: Long): JavaRDD[T] = {
|
||||
randomJavaRDD(jsc, generator, size, 0);
|
||||
jsc: JavaSparkContext,
|
||||
generator: RandomDataGenerator[T],
|
||||
size: Long): JavaRDD[T] = {
|
||||
randomJavaRDD(jsc, generator, size, 0)
|
||||
}
|
||||
|
||||
// TODO Generate RDD[Vector] from multivariate distributions.
|
||||
|
|
|
@ -26,7 +26,7 @@ import org.apache.spark.api.java.JavaRDD
|
|||
import org.apache.spark.internal.Logging
|
||||
import org.apache.spark.rdd.RDD
|
||||
import org.apache.spark.sql.execution.LogicalRDD
|
||||
import org.apache.spark.sql.execution.datasources.{DataSource, LogicalRelation}
|
||||
import org.apache.spark.sql.execution.datasources.DataSource
|
||||
import org.apache.spark.sql.execution.datasources.jdbc.{JDBCPartition, JDBCPartitioningInfo, JDBCRelation}
|
||||
import org.apache.spark.sql.execution.datasources.json.{InferSchema, JacksonParser, JSONOptions}
|
||||
import org.apache.spark.sql.types.StructType
|
||||
|
|
|
@ -43,7 +43,7 @@ import org.apache.spark.sql.catalyst.plans.logical._
|
|||
import org.apache.spark.sql.catalyst.util.usePrettyExpression
|
||||
import org.apache.spark.sql.execution.{FileRelation, LogicalRDD, QueryExecution, SQLExecution}
|
||||
import org.apache.spark.sql.execution.command.{CreateViewCommand, ExplainCommand}
|
||||
import org.apache.spark.sql.execution.datasources.{CreateTable, LogicalRelation}
|
||||
import org.apache.spark.sql.execution.datasources.LogicalRelation
|
||||
import org.apache.spark.sql.execution.datasources.json.JacksonGenerator
|
||||
import org.apache.spark.sql.execution.python.EvaluatePython
|
||||
import org.apache.spark.sql.streaming.{DataStreamWriter, StreamingQuery}
|
||||
|
|
|
@ -1093,7 +1093,7 @@ object SQLContext {
|
|||
}
|
||||
data.map{ element =>
|
||||
new GenericInternalRow(
|
||||
methodsToConverts.map { case (e, convert) => convert(e.invoke(element)) }.toArray[Any]
|
||||
methodsToConverts.map { case (e, convert) => convert(e.invoke(element)) }
|
||||
): InternalRow
|
||||
}
|
||||
}
|
||||
|
|
|
@ -61,7 +61,7 @@ import org.apache.spark.util.Utils
|
|||
* qualified. This option only works when reading from a [[FileFormat]].
|
||||
* @param userSpecifiedSchema An optional specification of the schema of the data. When present
|
||||
* we skip attempting to infer the schema.
|
||||
* @param partitionColumns A list of column names that the relation is partitioned by. When this
|
||||
* @param partitionColumns A list of column names that the relation is partitioned by. When this
|
||||
* list is empty, the relation is unpartitioned.
|
||||
* @param bucketSpec An optional specification for bucketing (hash-partitioning) of the data.
|
||||
*/
|
||||
|
|
Loading…
Reference in a new issue