[SPARK-21731][BUILD] Upgrade scalastyle to 0.9.
This version fixes a few issues in the import order checker; it provides better error messages, and detects more improper ordering (thus the need to change a lot of files in this patch). The main fix is that it correctly complains about the order of packages vs. classes. As part of the above, I moved some "SparkSession" import in ML examples inside the "$example on$" blocks; that didn't seem consistent across different source files to start with, and avoids having to add more on/off blocks around specific imports. The new scalastyle also seems to have a better header detector, so a few license headers had to be updated to match the expected indentation. Author: Marcelo Vanzin <vanzin@cloudera.com> Closes #18943 from vanzin/SPARK-21731.
This commit is contained in:
Marcelo Vanzin
parent
cba826d001
commit
3f958a9992
|
|
@ -22,8 +22,8 @@ import java.lang.{Double => JDouble}
|
|||
import scala.language.implicitConversions
|
||||
import scala.reflect.ClassTag
|
||||
|
||||
import org.apache.spark.annotation.Since
|
||||
import org.apache.spark.Partitioner
|
||||
import org.apache.spark.annotation.Since
|
||||
import org.apache.spark.api.java.function.{Function => JFunction}
|
||||
import org.apache.spark.partial.{BoundedDouble, PartialResult}
|
||||
import org.apache.spark.rdd.RDD
|
||||
|
|
|
|||
|
|
@ -22,8 +22,8 @@ import java.net.{InetAddress, ServerSocket, Socket, SocketException}
|
|||
import java.nio.charset.StandardCharsets
|
||||
import java.util.Arrays
|
||||
|
||||
import scala.collection.mutable
|
||||
import scala.collection.JavaConverters._
|
||||
import scala.collection.mutable
|
||||
|
||||
import org.apache.spark._
|
||||
import org.apache.spark.internal.Logging
|
||||
|
|
|
|||
|
|
@ -17,8 +17,8 @@
|
|||
|
||||
package org.apache.spark.api.r
|
||||
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
/** JVM object ID wrapper */
|
||||
private[r] case class JVMObjectId(id: String) {
|
||||
|
|
|
|||
|
|
@ -26,9 +26,9 @@ import io.netty.channel.{ChannelHandlerContext, SimpleChannelInboundHandler}
|
|||
import io.netty.channel.ChannelHandler.Sharable
|
||||
import io.netty.handler.timeout.ReadTimeoutException
|
||||
|
||||
import org.apache.spark.SparkConf
|
||||
import org.apache.spark.api.r.SerDe._
|
||||
import org.apache.spark.internal.Logging
|
||||
import org.apache.spark.SparkConf
|
||||
import org.apache.spark.util.{ThreadUtils, Utils}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -20,8 +20,8 @@ package org.apache.spark.deploy
|
|||
import java.io.File
|
||||
import java.net.URI
|
||||
|
||||
import scala.collection.mutable.ArrayBuffer
|
||||
import scala.collection.JavaConverters._
|
||||
import scala.collection.mutable.ArrayBuffer
|
||||
import scala.util.Try
|
||||
|
||||
import org.apache.spark.{SparkConf, SparkUserAppException}
|
||||
|
|
|
|||
|
|
@ -22,8 +22,8 @@ import java.security.PrivilegedExceptionAction
|
|||
import java.text.DateFormat
|
||||
import java.util.{Arrays, Comparator, Date, Locale}
|
||||
|
||||
import scala.collection.immutable.Map
|
||||
import scala.collection.JavaConverters._
|
||||
import scala.collection.immutable.Map
|
||||
import scala.collection.mutable
|
||||
import scala.collection.mutable.HashMap
|
||||
import scala.util.control.NonFatal
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@
|
|||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@
|
|||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
|
|
|
|||
|
|
@ -21,8 +21,8 @@ import java.lang.management.ManagementFactory
|
|||
|
||||
import scala.annotation.tailrec
|
||||
|
||||
import org.apache.spark.util.{IntParam, MemoryParam, Utils}
|
||||
import org.apache.spark.SparkConf
|
||||
import org.apache.spark.util.{IntParam, MemoryParam, Utils}
|
||||
|
||||
/**
|
||||
* Command-line parser for the worker.
|
||||
|
|
|
|||
|
|
@ -27,9 +27,9 @@ import org.apache.hadoop.fs.Path
|
|||
import org.apache.hadoop.mapreduce.{InputSplit, JobContext, RecordReader, TaskAttemptContext}
|
||||
import org.apache.hadoop.mapreduce.lib.input.{CombineFileInputFormat, CombineFileRecordReader, CombineFileSplit}
|
||||
|
||||
import org.apache.spark.internal.config
|
||||
import org.apache.spark.SparkContext
|
||||
import org.apache.spark.annotation.Since
|
||||
import org.apache.spark.internal.config
|
||||
|
||||
/**
|
||||
* A general format for reading whole files in as streams, byte arrays,
|
||||
|
|
|
|||
|
|
@ -21,11 +21,8 @@ import scala.collection.JavaConverters._
|
|||
|
||||
import org.apache.hadoop.fs.Path
|
||||
import org.apache.hadoop.io.Text
|
||||
import org.apache.hadoop.mapreduce.InputSplit
|
||||
import org.apache.hadoop.mapreduce.JobContext
|
||||
import org.apache.hadoop.mapreduce.{InputSplit, JobContext, RecordReader, TaskAttemptContext}
|
||||
import org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat
|
||||
import org.apache.hadoop.mapreduce.RecordReader
|
||||
import org.apache.hadoop.mapreduce.TaskAttemptContext
|
||||
|
||||
/**
|
||||
* A [[org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat CombineFileInputFormat]] for
|
||||
|
|
|
|||
|
|
@ -26,8 +26,8 @@ import com.codahale.metrics.{Metric, MetricFilter, MetricRegistry}
|
|||
import org.eclipse.jetty.servlet.ServletContextHandler
|
||||
|
||||
import org.apache.spark.{SecurityManager, SparkConf}
|
||||
import org.apache.spark.internal.config._
|
||||
import org.apache.spark.internal.Logging
|
||||
import org.apache.spark.internal.config._
|
||||
import org.apache.spark.metrics.sink.{MetricsServlet, Sink}
|
||||
import org.apache.spark.metrics.source.{Source, StaticSources}
|
||||
import org.apache.spark.util.Utils
|
||||
|
|
|
|||
|
|
@ -26,8 +26,8 @@ import scala.reflect.ClassTag
|
|||
import org.apache.spark._
|
||||
import org.apache.spark.annotation.DeveloperApi
|
||||
import org.apache.spark.serializer.Serializer
|
||||
import org.apache.spark.util.collection.{CompactBuffer, ExternalAppendOnlyMap}
|
||||
import org.apache.spark.util.Utils
|
||||
import org.apache.spark.util.collection.{CompactBuffer, ExternalAppendOnlyMap}
|
||||
|
||||
/**
|
||||
* The references to rdd and splitIndex are transient because redundant information is stored
|
||||
|
|
|
|||
|
|
@ -17,8 +17,8 @@
|
|||
|
||||
package org.apache.spark.rdd
|
||||
|
||||
import org.apache.spark.annotation.Since
|
||||
import org.apache.spark.TaskContext
|
||||
import org.apache.spark.annotation.Since
|
||||
import org.apache.spark.internal.Logging
|
||||
import org.apache.spark.partial.BoundedDouble
|
||||
import org.apache.spark.partial.MeanEvaluator
|
||||
|
|
|
|||
|
|
@ -35,8 +35,8 @@ import org.apache.hadoop.mapreduce.{Job => NewAPIHadoopJob, OutputFormat => NewO
|
|||
import org.apache.spark._
|
||||
import org.apache.spark.Partitioner.defaultPartitioner
|
||||
import org.apache.spark.annotation.Experimental
|
||||
import org.apache.spark.internal.io._
|
||||
import org.apache.spark.internal.Logging
|
||||
import org.apache.spark.internal.io._
|
||||
import org.apache.spark.partial.{BoundedDouble, PartialResult}
|
||||
import org.apache.spark.serializer.Serializer
|
||||
import org.apache.spark.util.{SerializableConfiguration, SerializableJobConf, Utils}
|
||||
|
|
|
|||
|
|
@ -22,8 +22,8 @@ import java.util.Random
|
|||
import scala.reflect.ClassTag
|
||||
|
||||
import org.apache.spark.{Partition, TaskContext}
|
||||
import org.apache.spark.util.random.RandomSampler
|
||||
import org.apache.spark.util.Utils
|
||||
import org.apache.spark.util.random.RandomSampler
|
||||
|
||||
private[spark]
|
||||
class PartitionwiseSampledRDDPartition(val prev: Partition, val seed: Long)
|
||||
|
|
|
|||
|
|
@ -19,8 +19,8 @@ package org.apache.spark.rdd
|
|||
|
||||
import scala.collection.mutable
|
||||
|
||||
import org.apache.spark.annotation.DeveloperApi
|
||||
import org.apache.spark.Partition
|
||||
import org.apache.spark.annotation.DeveloperApi
|
||||
|
||||
/**
|
||||
* ::DeveloperApi::
|
||||
|
|
|
|||
|
|
@ -35,8 +35,8 @@ import org.apache.commons.lang3.SerializationUtils
|
|||
import org.apache.spark._
|
||||
import org.apache.spark.broadcast.Broadcast
|
||||
import org.apache.spark.executor.TaskMetrics
|
||||
import org.apache.spark.internal.config
|
||||
import org.apache.spark.internal.Logging
|
||||
import org.apache.spark.internal.config
|
||||
import org.apache.spark.network.util.JavaUtils
|
||||
import org.apache.spark.partial.{ApproximateActionListener, ApproximateEvaluator, PartialResult}
|
||||
import org.apache.spark.rdd.RDD
|
||||
|
|
|
|||
|
|
@ -26,9 +26,9 @@ import scala.math.max
|
|||
import scala.util.control.NonFatal
|
||||
|
||||
import org.apache.spark._
|
||||
import org.apache.spark.TaskState.TaskState
|
||||
import org.apache.spark.internal.Logging
|
||||
import org.apache.spark.scheduler.SchedulingMode._
|
||||
import org.apache.spark.TaskState.TaskState
|
||||
import org.apache.spark.util.{AccumulatorV2, Clock, SystemClock, Utils}
|
||||
import org.apache.spark.util.collection.MedianHeap
|
||||
|
||||
|
|
|
|||
|
|
@ -1,19 +1,19 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.spark.status.api.v1
|
||||
|
||||
import javax.ws.rs.{GET, Produces}
|
||||
|
|
|
|||
|
|
@ -1,19 +1,19 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.spark.status.api.v1
|
||||
|
||||
import javax.ws.rs.{GET, Produces}
|
||||
|
|
|
|||
|
|
@ -19,8 +19,8 @@ package org.apache.spark.storage
|
|||
|
||||
import java.util.{HashMap => JHashMap}
|
||||
|
||||
import scala.collection.mutable
|
||||
import scala.collection.JavaConverters._
|
||||
import scala.collection.mutable
|
||||
import scala.concurrent.{ExecutionContext, Future}
|
||||
import scala.util.Random
|
||||
|
||||
|
|
|
|||
|
|
@ -25,8 +25,8 @@ import scala.collection.mutable.ArrayBuffer
|
|||
import scala.language.implicitConversions
|
||||
import scala.xml.Node
|
||||
|
||||
import org.eclipse.jetty.client.api.Response
|
||||
import org.eclipse.jetty.client.HttpClient
|
||||
import org.eclipse.jetty.client.api.Response
|
||||
import org.eclipse.jetty.client.http.HttpClientTransportOverHTTP
|
||||
import org.eclipse.jetty.proxy.ProxyServlet
|
||||
import org.eclipse.jetty.server._
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@
|
|||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
|
|
|
|||
|
|
@ -17,8 +17,8 @@
|
|||
|
||||
package org.apache.spark
|
||||
|
||||
import org.scalatest.concurrent.Timeouts._
|
||||
import org.scalatest.Matchers
|
||||
import org.scalatest.concurrent.Timeouts._
|
||||
import org.scalatest.time.{Millis, Span}
|
||||
|
||||
import org.apache.spark.security.EncryptionFunSuite
|
||||
|
|
|
|||
|
|
@ -20,8 +20,8 @@ package org.apache.spark
|
|||
import java.util.concurrent.Semaphore
|
||||
|
||||
import scala.concurrent.ExecutionContext.Implicits.global
|
||||
import scala.concurrent.duration._
|
||||
import scala.concurrent.Future
|
||||
import scala.concurrent.duration._
|
||||
|
||||
import org.scalatest.BeforeAndAfter
|
||||
import org.scalatest.Matchers
|
||||
|
|
|
|||
|
|
@ -22,8 +22,8 @@ import scala.collection.mutable.ArrayBuffer
|
|||
import org.mockito.Matchers.any
|
||||
import org.mockito.Mockito._
|
||||
|
||||
import org.apache.spark.broadcast.BroadcastManager
|
||||
import org.apache.spark.LocalSparkContext._
|
||||
import org.apache.spark.broadcast.BroadcastManager
|
||||
import org.apache.spark.rpc.{RpcAddress, RpcCallContext, RpcEnv}
|
||||
import org.apache.spark.scheduler.{CompressedMapStatus, MapStatus}
|
||||
import org.apache.spark.shuffle.FetchFailedException
|
||||
|
|
|
|||
|
|
@ -30,8 +30,8 @@ import org.apache.hadoop.fs.{FileSystem, Path}
|
|||
import org.apache.hadoop.io.{BytesWritable, LongWritable, Text}
|
||||
import org.apache.hadoop.mapred.TextInputFormat
|
||||
import org.apache.hadoop.mapreduce.lib.input.{TextInputFormat => NewTextInputFormat}
|
||||
import org.scalatest.concurrent.Eventually
|
||||
import org.scalatest.Matchers._
|
||||
import org.scalatest.concurrent.Eventually
|
||||
|
||||
import org.apache.spark.scheduler.{SparkListener, SparkListenerJobStart, SparkListenerTaskEnd, SparkListenerTaskStart}
|
||||
import org.apache.spark.util.{ThreadUtils, Utils}
|
||||
|
|
|
|||
|
|
@ -35,12 +35,12 @@ import org.scalatest.concurrent.Timeouts
|
|||
import org.scalatest.time.SpanSugar._
|
||||
|
||||
import org.apache.spark._
|
||||
import org.apache.spark.TestUtils.JavaSourceFromString
|
||||
import org.apache.spark.api.r.RUtils
|
||||
import org.apache.spark.deploy.SparkSubmit._
|
||||
import org.apache.spark.deploy.SparkSubmitUtils.MavenCoordinate
|
||||
import org.apache.spark.internal.config._
|
||||
import org.apache.spark.internal.Logging
|
||||
import org.apache.spark.TestUtils.JavaSourceFromString
|
||||
import org.apache.spark.internal.config._
|
||||
import org.apache.spark.scheduler.EventLoggingListener
|
||||
import org.apache.spark.util.{CommandLineUtils, ResetSystemProperties, Utils}
|
||||
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@
|
|||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
|
|
|
|||
|
|
@ -28,8 +28,8 @@ import scala.util.{Failure, Success, Try}
|
|||
|
||||
import com.google.common.io.CharStreams
|
||||
import org.mockito.Mockito._
|
||||
import org.scalatest.mock.MockitoSugar
|
||||
import org.scalatest.ShouldMatchers
|
||||
import org.scalatest.mock.MockitoSugar
|
||||
|
||||
import org.apache.spark.{SecurityManager, SparkConf, SparkFunSuite}
|
||||
import org.apache.spark.network.{BlockDataManager, BlockTransferService}
|
||||
|
|
|
|||
|
|
@ -20,8 +20,8 @@ package org.apache.spark.rdd
|
|||
import java.util.concurrent.Semaphore
|
||||
|
||||
import scala.concurrent._
|
||||
import scala.concurrent.duration.Duration
|
||||
import scala.concurrent.ExecutionContext.Implicits.global
|
||||
import scala.concurrent.duration.Duration
|
||||
|
||||
import org.scalatest.BeforeAndAfterAll
|
||||
import org.scalatest.concurrent.Timeouts
|
||||
|
|
|
|||
|
|
@ -22,8 +22,8 @@ import java.nio.charset.StandardCharsets.UTF_8
|
|||
import java.util.UUID
|
||||
import java.util.concurrent.{ConcurrentLinkedQueue, CountDownLatch, TimeUnit}
|
||||
|
||||
import scala.collection.mutable
|
||||
import scala.collection.JavaConverters._
|
||||
import scala.collection.mutable
|
||||
import scala.concurrent.Await
|
||||
import scala.concurrent.duration._
|
||||
import scala.language.postfixOps
|
||||
|
|
|
|||
|
|
@ -17,9 +17,9 @@
|
|||
|
||||
package org.apache.spark.scheduler
|
||||
|
||||
import org.mockito.invocation.InvocationOnMock
|
||||
import org.mockito.Matchers.any
|
||||
import org.mockito.Mockito.{never, verify, when}
|
||||
import org.mockito.invocation.InvocationOnMock
|
||||
import org.mockito.stubbing.Answer
|
||||
import org.scalatest.BeforeAndAfterEach
|
||||
import org.scalatest.mock.MockitoSugar
|
||||
|
|
|
|||
|
|
@ -25,8 +25,8 @@ import org.mockito.Mockito._
|
|||
import org.roaringbitmap.RoaringBitmap
|
||||
|
||||
import org.apache.spark.{SparkConf, SparkContext, SparkEnv, SparkFunSuite}
|
||||
import org.apache.spark.internal.config
|
||||
import org.apache.spark.LocalSparkContext._
|
||||
import org.apache.spark.internal.config
|
||||
import org.apache.spark.serializer.{JavaSerializer, KryoSerializer}
|
||||
import org.apache.spark.storage.BlockManagerId
|
||||
|
||||
|
|
|
|||
|
|
@ -19,8 +19,8 @@ package org.apache.spark.scheduler
|
|||
|
||||
import java.util.concurrent.Semaphore
|
||||
|
||||
import scala.collection.mutable
|
||||
import scala.collection.JavaConverters._
|
||||
import scala.collection.mutable
|
||||
|
||||
import org.mockito.Mockito
|
||||
import org.scalatest.Matchers
|
||||
|
|
|
|||
|
|
@ -34,8 +34,8 @@ import org.scalatest.BeforeAndAfter
|
|||
import org.scalatest.concurrent.Eventually._
|
||||
|
||||
import org.apache.spark._
|
||||
import org.apache.spark.storage.TaskResultBlockId
|
||||
import org.apache.spark.TestUtils.JavaSourceFromString
|
||||
import org.apache.spark.storage.TaskResultBlockId
|
||||
import org.apache.spark.util.{MutableURLClassLoader, RpcUtils, Utils}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -27,8 +27,8 @@ import org.scalatest.BeforeAndAfterEach
|
|||
import org.scalatest.mock.MockitoSugar
|
||||
|
||||
import org.apache.spark._
|
||||
import org.apache.spark.internal.config
|
||||
import org.apache.spark.internal.Logging
|
||||
import org.apache.spark.internal.config
|
||||
import org.apache.spark.util.ManualClock
|
||||
|
||||
class FakeSchedulerBackend extends SchedulerBackend {
|
||||
|
|
|
|||
|
|
@ -28,8 +28,8 @@ import org.mockito.invocation.InvocationOnMock
|
|||
import org.mockito.stubbing.Answer
|
||||
|
||||
import org.apache.spark._
|
||||
import org.apache.spark.internal.config
|
||||
import org.apache.spark.internal.Logging
|
||||
import org.apache.spark.internal.config
|
||||
import org.apache.spark.serializer.SerializerInstance
|
||||
import org.apache.spark.storage.BlockManagerId
|
||||
import org.apache.spark.util.{AccumulatorV2, ManualClock, Utils}
|
||||
|
|
|
|||
|
|
@ -23,8 +23,8 @@ import java.nio.ByteBuffer
|
|||
import scala.collection.JavaConverters._
|
||||
import scala.collection.mutable
|
||||
import scala.collection.mutable.ArrayBuffer
|
||||
import scala.concurrent.duration._
|
||||
import scala.concurrent.Future
|
||||
import scala.concurrent.duration._
|
||||
import scala.language.{implicitConversions, postfixOps}
|
||||
import scala.reflect.ClassTag
|
||||
|
||||
|
|
|
|||
|
|
@ -26,8 +26,8 @@ import io.netty.channel.FileRegion
|
|||
import org.apache.spark.{SecurityManager, SparkConf, SparkFunSuite}
|
||||
import org.apache.spark.network.util.{ByteArrayWritableChannel, JavaUtils}
|
||||
import org.apache.spark.security.CryptoStreamUtils
|
||||
import org.apache.spark.util.io.ChunkedByteBuffer
|
||||
import org.apache.spark.util.Utils
|
||||
import org.apache.spark.util.io.ChunkedByteBuffer
|
||||
|
||||
class DiskStoreSuite extends SparkFunSuite {
|
||||
|
||||
|
|
|
|||
|
|
@ -22,9 +22,9 @@ import java.util.Properties
|
|||
import scala.collection.JavaConverters._
|
||||
import scala.collection.Map
|
||||
|
||||
import org.json4s.jackson.JsonMethods._
|
||||
import org.json4s.JsonAST.{JArray, JInt, JString, JValue}
|
||||
import org.json4s.JsonDSL._
|
||||
import org.json4s.jackson.JsonMethods._
|
||||
import org.scalatest.Assertions
|
||||
import org.scalatest.exceptions.TestFailedException
|
||||
|
||||
|
|
|
|||
|
|
@ -21,9 +21,9 @@ package org.apache.spark.examples.ml
|
|||
// $example on$
|
||||
import org.apache.spark.ml.feature.BucketedRandomProjectionLSH
|
||||
import org.apache.spark.ml.linalg.Vectors
|
||||
import org.apache.spark.sql.SparkSession
|
||||
import org.apache.spark.sql.functions.col
|
||||
// $example off$
|
||||
import org.apache.spark.sql.SparkSession
|
||||
|
||||
/**
|
||||
* An example demonstrating BucketedRandomProjectionLSH.
|
||||
|
|
|
|||
|
|
@ -21,9 +21,9 @@ package org.apache.spark.examples.ml
|
|||
// $example on$
|
||||
import org.apache.spark.ml.feature.MinHashLSH
|
||||
import org.apache.spark.ml.linalg.Vectors
|
||||
import org.apache.spark.sql.SparkSession
|
||||
import org.apache.spark.sql.functions.col
|
||||
// $example off$
|
||||
import org.apache.spark.sql.SparkSession
|
||||
|
||||
/**
|
||||
* An example demonstrating MinHashLSH.
|
||||
|
|
|
|||
|
|
@ -20,9 +20,9 @@ package org.apache.spark.examples.ml
|
|||
|
||||
// $example on$
|
||||
import org.apache.spark.ml.feature.{RegexTokenizer, Tokenizer}
|
||||
import org.apache.spark.sql.SparkSession
|
||||
import org.apache.spark.sql.functions._
|
||||
// $example off$
|
||||
import org.apache.spark.sql.SparkSession
|
||||
|
||||
object TokenizerExample {
|
||||
def main(args: Array[String]): Unit = {
|
||||
|
|
|
|||
|
|
@ -22,10 +22,8 @@ package org.apache.spark.examples.ml
|
|||
import org.apache.spark.ml.UnaryTransformer
|
||||
import org.apache.spark.ml.param.DoubleParam
|
||||
import org.apache.spark.ml.util.{DefaultParamsReadable, DefaultParamsWritable, Identifiable}
|
||||
import org.apache.spark.sql.functions.col
|
||||
// $example off$
|
||||
import org.apache.spark.sql.SparkSession
|
||||
// $example on$
|
||||
import org.apache.spark.sql.functions.col
|
||||
import org.apache.spark.sql.types.{DataType, DataTypes}
|
||||
import org.apache.spark.util.Utils
|
||||
// $example off$
|
||||
|
|
|
|||
|
|
@ -24,10 +24,9 @@ import java.util.Arrays
|
|||
import org.apache.spark.ml.attribute.{Attribute, AttributeGroup, NumericAttribute}
|
||||
import org.apache.spark.ml.feature.VectorSlicer
|
||||
import org.apache.spark.ml.linalg.Vectors
|
||||
import org.apache.spark.sql.Row
|
||||
import org.apache.spark.sql.{Row, SparkSession}
|
||||
import org.apache.spark.sql.types.StructType
|
||||
// $example off$
|
||||
import org.apache.spark.sql.SparkSession
|
||||
|
||||
object VectorSlicerExample {
|
||||
def main(args: Array[String]): Unit = {
|
||||
|
|
|
|||
|
|
@ -19,8 +19,8 @@
|
|||
package org.apache.spark.examples.mllib
|
||||
|
||||
import org.apache.spark.{SparkConf, SparkContext}
|
||||
import org.apache.spark.mllib.linalg.distributed.RowMatrix
|
||||
import org.apache.spark.mllib.linalg.Vectors
|
||||
import org.apache.spark.mllib.linalg.distributed.RowMatrix
|
||||
|
||||
/**
|
||||
* Compute the principal components of a tall-and-skinny matrix, whose rows are observations.
|
||||
|
|
|
|||
|
|
@ -19,8 +19,8 @@
|
|||
package org.apache.spark.examples.mllib
|
||||
|
||||
import org.apache.spark.{SparkConf, SparkContext}
|
||||
import org.apache.spark.mllib.linalg.distributed.RowMatrix
|
||||
import org.apache.spark.mllib.linalg.Vectors
|
||||
import org.apache.spark.mllib.linalg.distributed.RowMatrix
|
||||
|
||||
/**
|
||||
* Compute the singular value decomposition (SVD) of a tall-and-skinny matrix.
|
||||
|
|
|
|||
|
|
@ -21,13 +21,13 @@ import java.util.{Collection => JCollection, Map => JMap}
|
|||
|
||||
import scala.collection.JavaConverters._
|
||||
|
||||
import org.apache.avro.generic.{GenericFixed, IndexedRecord}
|
||||
import org.apache.avro.mapred.AvroWrapper
|
||||
import org.apache.avro.Schema
|
||||
import org.apache.avro.Schema.Type._
|
||||
import org.apache.avro.generic.{GenericFixed, IndexedRecord}
|
||||
import org.apache.avro.mapred.AvroWrapper
|
||||
|
||||
import org.apache.spark.api.python.Converter
|
||||
import org.apache.spark.SparkException
|
||||
import org.apache.spark.api.python.Converter
|
||||
|
||||
|
||||
object AvroConversionUtil extends Serializable {
|
||||
|
|
|
|||
|
|
@ -17,10 +17,8 @@
|
|||
package org.apache.spark.examples.sql
|
||||
|
||||
// $example on:typed_custom_aggregation$
|
||||
import org.apache.spark.sql.{Encoder, Encoders, SparkSession}
|
||||
import org.apache.spark.sql.expressions.Aggregator
|
||||
import org.apache.spark.sql.Encoder
|
||||
import org.apache.spark.sql.Encoders
|
||||
import org.apache.spark.sql.SparkSession
|
||||
// $example off:typed_custom_aggregation$
|
||||
|
||||
object UserDefinedTypedAggregation {
|
||||
|
|
|
|||
|
|
@ -17,11 +17,10 @@
|
|||
package org.apache.spark.examples.sql
|
||||
|
||||
// $example on:untyped_custom_aggregation$
|
||||
import org.apache.spark.sql.{Row, SparkSession}
|
||||
import org.apache.spark.sql.expressions.MutableAggregationBuffer
|
||||
import org.apache.spark.sql.expressions.UserDefinedAggregateFunction
|
||||
import org.apache.spark.sql.types._
|
||||
import org.apache.spark.sql.Row
|
||||
import org.apache.spark.sql.SparkSession
|
||||
// $example off:untyped_custom_aggregation$
|
||||
|
||||
object UserDefinedUntypedAggregation {
|
||||
|
|
|
|||
|
|
@ -20,9 +20,9 @@ package org.apache.spark.examples.streaming
|
|||
|
||||
import kafka.serializer.StringDecoder
|
||||
|
||||
import org.apache.spark.SparkConf
|
||||
import org.apache.spark.streaming._
|
||||
import org.apache.spark.streaming.kafka._
|
||||
import org.apache.spark.SparkConf
|
||||
|
||||
/**
|
||||
* Consumes messages from one or more topics in Kafka and does wordcount.
|
||||
|
|
|
|||
|
|
@ -35,8 +35,8 @@ import org.jboss.netty.channel.socket.SocketChannel
|
|||
import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory
|
||||
import org.jboss.netty.handler.codec.compression.{ZlibDecoder, ZlibEncoder}
|
||||
|
||||
import org.apache.spark.util.Utils
|
||||
import org.apache.spark.SparkConf
|
||||
import org.apache.spark.util.Utils
|
||||
|
||||
/**
|
||||
* Share codes for Scala and Python unit tests
|
||||
|
|
|
|||
|
|
@ -23,10 +23,10 @@ import java.util.concurrent._
|
|||
|
||||
import scala.collection.mutable.ArrayBuffer
|
||||
|
||||
import org.apache.flume.event.EventBuilder
|
||||
import org.apache.flume.Context
|
||||
import org.apache.flume.channel.MemoryChannel
|
||||
import org.apache.flume.conf.Configurables
|
||||
import org.apache.flume.event.EventBuilder
|
||||
|
||||
import org.apache.spark.streaming.flume.sink.{SparkSink, SparkSinkConfig}
|
||||
|
||||
|
|
|
|||
|
|
@ -40,9 +40,9 @@ import org.apache.zookeeper.server.{NIOServerCnxnFactory, ZooKeeperServer}
|
|||
import org.scalatest.concurrent.Eventually._
|
||||
import org.scalatest.time.SpanSugar._
|
||||
|
||||
import org.apache.spark.SparkConf
|
||||
import org.apache.spark.internal.Logging
|
||||
import org.apache.spark.util.Utils
|
||||
import org.apache.spark.SparkConf
|
||||
|
||||
/**
|
||||
* This is a helper class for Kafka test suites. This has the functionality to set up
|
||||
|
|
|
|||
|
|
@ -20,8 +20,8 @@ package org.apache.spark.streaming.kafka010
|
|||
import java.io.File
|
||||
import java.lang.{ Long => JLong }
|
||||
import java.util.{ Arrays, HashMap => JHashMap, Map => JMap }
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.concurrent.ConcurrentLinkedQueue
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
import scala.collection.JavaConverters._
|
||||
import scala.concurrent.duration._
|
||||
|
|
|
|||
|
|
@ -19,8 +19,8 @@ package org.apache.spark.streaming.kafka
|
|||
|
||||
import java.io.File
|
||||
import java.util.Arrays
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.concurrent.ConcurrentLinkedQueue
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
import scala.collection.JavaConverters._
|
||||
import scala.concurrent.duration._
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ import scala.reflect.ClassTag
|
|||
|
||||
import com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionInStream
|
||||
import com.amazonaws.services.kinesis.model.Record
|
||||
import KinesisReadConfigurations._
|
||||
|
||||
import org.apache.spark.annotation.InterfaceStability
|
||||
import org.apache.spark.rdd.RDD
|
||||
|
|
@ -47,6 +46,8 @@ private[kinesis] class KinesisInputDStream[T: ClassTag](
|
|||
val cloudWatchCreds: Option[SparkAWSCredentials]
|
||||
) extends ReceiverInputDStream[T](_ssc) {
|
||||
|
||||
import KinesisReadConfigurations._
|
||||
|
||||
private[streaming]
|
||||
override def createBlockRDD(time: Time, blockInfos: Seq[ReceivedBlockInfo]): RDD[T] = {
|
||||
|
||||
|
|
|
|||
|
|
@ -20,10 +20,10 @@ package org.apache.spark.graphx
|
|||
import scala.reflect.ClassTag
|
||||
import scala.util.Random
|
||||
|
||||
import org.apache.spark.SparkException
|
||||
import org.apache.spark.graphx.lib._
|
||||
import org.apache.spark.ml.linalg.Vector
|
||||
import org.apache.spark.rdd.RDD
|
||||
import org.apache.spark.SparkException
|
||||
|
||||
/**
|
||||
* Contains additional functionality for [[Graph]]. All operations are expressed in terms of the
|
||||
|
|
|
|||
|
|
@ -17,8 +17,8 @@
|
|||
|
||||
package org.apache.spark.ml.impl
|
||||
|
||||
import org.apache.spark.ml.impl.Utils.EPSILON
|
||||
import org.apache.spark.ml.SparkMLFunSuite
|
||||
import org.apache.spark.ml.impl.Utils.EPSILON
|
||||
|
||||
|
||||
class UtilsSuite extends SparkMLFunSuite {
|
||||
|
|
|
|||
|
|
@ -21,12 +21,12 @@ import scala.collection.mutable.ArrayBuilder
|
|||
|
||||
import org.apache.spark.SparkException
|
||||
import org.apache.spark.annotation.Since
|
||||
import org.apache.spark.ml.Transformer
|
||||
import org.apache.spark.ml.attribute._
|
||||
import org.apache.spark.ml.linalg.{Vector, Vectors, VectorUDT}
|
||||
import org.apache.spark.ml.param._
|
||||
import org.apache.spark.ml.param.shared._
|
||||
import org.apache.spark.ml.util._
|
||||
import org.apache.spark.ml.Transformer
|
||||
import org.apache.spark.ml.linalg.{Vector, Vectors, VectorUDT}
|
||||
import org.apache.spark.sql.{DataFrame, Dataset, Row}
|
||||
import org.apache.spark.sql.functions._
|
||||
import org.apache.spark.sql.types._
|
||||
|
|
|
|||
|
|
@ -18,8 +18,8 @@
|
|||
package org.apache.spark.ml.feature
|
||||
|
||||
import org.apache.spark.annotation.Since
|
||||
import org.apache.spark.ml.param.{Param, ParamMap}
|
||||
import org.apache.spark.ml.Transformer
|
||||
import org.apache.spark.ml.param.{Param, ParamMap}
|
||||
import org.apache.spark.ml.util._
|
||||
import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
|
||||
import org.apache.spark.sql.types.StructType
|
||||
|
|
|
|||
|
|
@ -22,8 +22,8 @@ import java.util.{List => JList}
|
|||
import java.util.NoSuchElementException
|
||||
|
||||
import scala.annotation.varargs
|
||||
import scala.collection.mutable
|
||||
import scala.collection.JavaConverters._
|
||||
import scala.collection.mutable
|
||||
|
||||
import org.json4s._
|
||||
import org.json4s.jackson.JsonMethods._
|
||||
|
|
|
|||
|
|
@ -27,11 +27,11 @@ import org.apache.hadoop.fs.Path
|
|||
import org.apache.spark.SparkException
|
||||
import org.apache.spark.annotation.{Experimental, Since}
|
||||
import org.apache.spark.internal.Logging
|
||||
import org.apache.spark.ml.PredictorParams
|
||||
import org.apache.spark.ml.feature.Instance
|
||||
import org.apache.spark.ml.linalg.{Vector, Vectors}
|
||||
import org.apache.spark.ml.linalg.BLAS._
|
||||
import org.apache.spark.ml.optim.WeightedLeastSquares
|
||||
import org.apache.spark.ml.PredictorParams
|
||||
import org.apache.spark.ml.optim.aggregator.LeastSquaresAggregator
|
||||
import org.apache.spark.ml.optim.loss.{L2Regularization, RDDLossFunction}
|
||||
import org.apache.spark.ml.param.{Param, ParamMap, ParamValidators}
|
||||
|
|
|
|||
|
|
@ -23,8 +23,8 @@ import org.apache.hadoop.conf.Configuration
|
|||
import org.apache.hadoop.fs.{FileStatus, Path}
|
||||
import org.apache.hadoop.mapreduce.{Job, TaskAttemptContext}
|
||||
|
||||
import org.apache.spark.internal.Logging
|
||||
import org.apache.spark.TaskContext
|
||||
import org.apache.spark.internal.Logging
|
||||
import org.apache.spark.ml.feature.LabeledPoint
|
||||
import org.apache.spark.ml.linalg.{Vectors, VectorUDT}
|
||||
import org.apache.spark.mllib.util.MLUtils
|
||||
|
|
|
|||
|
|
@ -19,8 +19,8 @@ package org.apache.spark.mllib.clustering
|
|||
|
||||
import org.json4s._
|
||||
import org.json4s.DefaultFormats
|
||||
import org.json4s.jackson.JsonMethods._
|
||||
import org.json4s.JsonDSL._
|
||||
import org.json4s.jackson.JsonMethods._
|
||||
|
||||
import org.apache.spark.SparkContext
|
||||
import org.apache.spark.annotation.Since
|
||||
|
|
|
|||
|
|
@ -23,13 +23,13 @@ import org.json4s._
|
|||
import org.json4s.JsonDSL._
|
||||
import org.json4s.jackson.JsonMethods._
|
||||
|
||||
import org.apache.spark.SparkContext
|
||||
import org.apache.spark.annotation.Since
|
||||
import org.apache.spark.mllib.linalg.{DenseVector, SparseVector, Vector, Vectors}
|
||||
import org.apache.spark.mllib.regression.LabeledPoint
|
||||
import org.apache.spark.mllib.stat.Statistics
|
||||
import org.apache.spark.mllib.util.{Loader, Saveable}
|
||||
import org.apache.spark.rdd.RDD
|
||||
import org.apache.spark.SparkContext
|
||||
import org.apache.spark.sql.{Row, SparkSession}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -28,14 +28,13 @@ import org.json4s._
|
|||
import org.json4s.JsonDSL._
|
||||
import org.json4s.jackson.JsonMethods._
|
||||
|
||||
import org.apache.spark.SparkContext
|
||||
import org.apache.spark.{RangePartitioner, SparkContext}
|
||||
import org.apache.spark.annotation.Since
|
||||
import org.apache.spark.api.java.{JavaDoubleRDD, JavaRDD}
|
||||
import org.apache.spark.mllib.linalg.{Vector, Vectors}
|
||||
import org.apache.spark.mllib.util.{Loader, Saveable}
|
||||
import org.apache.spark.rdd.RDD
|
||||
import org.apache.spark.sql.SparkSession
|
||||
import org.apache.spark.RangePartitioner
|
||||
|
||||
/**
|
||||
* Regression model for isotonic regression.
|
||||
|
|
|
|||
|
|
@ -19,11 +19,11 @@ package org.apache.spark.mllib.regression
|
|||
|
||||
import scala.beans.BeanInfo
|
||||
|
||||
import org.apache.spark.SparkException
|
||||
import org.apache.spark.annotation.Since
|
||||
import org.apache.spark.ml.feature.{LabeledPoint => NewLabeledPoint}
|
||||
import org.apache.spark.mllib.linalg.{Vector, Vectors}
|
||||
import org.apache.spark.mllib.util.NumericParser
|
||||
import org.apache.spark.SparkException
|
||||
|
||||
/**
|
||||
* Class that represents the features and labels of a data point.
|
||||
|
|
|
|||
|
|
@ -20,10 +20,10 @@ package org.apache.spark.ml.recommendation
|
|||
import java.io.File
|
||||
import java.util.Random
|
||||
|
||||
import scala.collection.JavaConverters._
|
||||
import scala.collection.mutable
|
||||
import scala.collection.mutable.ArrayBuffer
|
||||
import scala.collection.mutable.WrappedArray
|
||||
import scala.collection.JavaConverters._
|
||||
import scala.language.existentials
|
||||
|
||||
import com.github.fommil.netlib.BLAS.{getInstance => blas}
|
||||
|
|
|
|||
2
pom.xml
2
pom.xml
|
|
@ -2424,7 +2424,7 @@
|
|||
<plugin>
|
||||
<groupId>org.scalastyle</groupId>
|
||||
<artifactId>scalastyle-maven-plugin</artifactId>
|
||||
<version>0.8.0</version>
|
||||
<version>0.9.0</version>
|
||||
<configuration>
|
||||
<verbose>false</verbose>
|
||||
<failOnViolation>true</failOnViolation>
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ import sbtunidoc.Plugin.UnidocKeys.unidocGenjavadocVersion
|
|||
import com.simplytyped.Antlr4Plugin._
|
||||
import com.typesafe.sbt.pom.{PomBuild, SbtPomKeys}
|
||||
import com.typesafe.tools.mima.plugin.MimaKeys
|
||||
import org.scalastyle.sbt.ScalastylePlugin._
|
||||
import org.scalastyle.sbt.ScalastylePlugin.autoImport._
|
||||
import org.scalastyle.sbt.Tasks
|
||||
|
||||
import spray.revolver.RevolverPlugin._
|
||||
|
|
@ -116,7 +116,7 @@ object SparkBuild extends PomBuild {
|
|||
|
||||
lazy val scalaStyleRules = Project("scalaStyleRules", file("scalastyle"))
|
||||
.settings(
|
||||
libraryDependencies += "org.scalastyle" %% "scalastyle" % "0.8.0"
|
||||
libraryDependencies += "org.scalastyle" %% "scalastyle" % "0.9.0"
|
||||
)
|
||||
|
||||
lazy val scalaStyleOnCompile = taskKey[Unit]("scalaStyleOnCompile")
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ addSbtPlugin("com.typesafe.sbteclipse" % "sbteclipse-plugin" % "5.1.0")
|
|||
addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.8.2")
|
||||
|
||||
// need to make changes to uptake sbt 1.0 support in "org.scalastyle" %% "scalastyle-sbt-plugin" % "0.9.0"
|
||||
addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "0.8.0")
|
||||
addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "0.9.0")
|
||||
|
||||
addSbtPlugin("com.typesafe" % "sbt-mima-plugin" % "0.1.17")
|
||||
|
||||
|
|
|
|||
|
|
@ -20,8 +20,8 @@ package org.apache.spark.deploy.mesos
|
|||
import scala.annotation.tailrec
|
||||
import scala.collection.mutable
|
||||
|
||||
import org.apache.spark.util.{IntParam, Utils}
|
||||
import org.apache.spark.SparkConf
|
||||
import org.apache.spark.util.{IntParam, Utils}
|
||||
|
||||
private[mesos] class MesosClusterDispatcherArguments(args: Array[String], conf: SparkConf) {
|
||||
var host: String = Utils.localHostName()
|
||||
|
|
|
|||
|
|
@ -23,8 +23,8 @@ import scala.xml.Node
|
|||
|
||||
import org.apache.mesos.Protos.TaskStatus
|
||||
|
||||
import org.apache.spark.deploy.mesos.config._
|
||||
import org.apache.spark.deploy.mesos.MesosDriverDescription
|
||||
import org.apache.spark.deploy.mesos.config._
|
||||
import org.apache.spark.scheduler.cluster.mesos.MesosClusterSubmissionState
|
||||
import org.apache.spark.ui.{UIUtils, WebUIPage}
|
||||
|
||||
|
|
|
|||
|
|
@ -30,8 +30,8 @@ import org.apache.mesos.Protos.Environment.Variable
|
|||
import org.apache.mesos.Protos.TaskStatus.Reason
|
||||
|
||||
import org.apache.spark.{SecurityManager, SparkConf, SparkException, TaskState}
|
||||
import org.apache.spark.deploy.mesos.config
|
||||
import org.apache.spark.deploy.mesos.MesosDriverDescription
|
||||
import org.apache.spark.deploy.mesos.config
|
||||
import org.apache.spark.deploy.rest.{CreateSubmissionResponse, KillSubmissionResponse, SubmissionStatusResponse}
|
||||
import org.apache.spark.metrics.MetricsSystem
|
||||
import org.apache.spark.util.Utils
|
||||
|
|
|
|||
|
|
@ -28,17 +28,17 @@ import org.apache.mesos.Protos._
|
|||
import org.mockito.Matchers
|
||||
import org.mockito.Matchers._
|
||||
import org.mockito.Mockito._
|
||||
import org.scalatest.BeforeAndAfter
|
||||
import org.scalatest.concurrent.ScalaFutures
|
||||
import org.scalatest.mock.MockitoSugar
|
||||
import org.scalatest.BeforeAndAfter
|
||||
|
||||
import org.apache.spark.{LocalSparkContext, SecurityManager, SparkConf, SparkContext, SparkFunSuite}
|
||||
import org.apache.spark.deploy.mesos.config._
|
||||
import org.apache.spark.internal.config._
|
||||
import org.apache.spark.network.shuffle.mesos.MesosExternalShuffleClient
|
||||
import org.apache.spark.rpc.{RpcAddress, RpcEndpointRef}
|
||||
import org.apache.spark.scheduler.cluster.CoarseGrainedClusterMessages.{RegisterExecutor, RemoveExecutor}
|
||||
import org.apache.spark.scheduler.TaskSchedulerImpl
|
||||
import org.apache.spark.scheduler.cluster.CoarseGrainedClusterMessages.{RegisterExecutor, RemoveExecutor}
|
||||
import org.apache.spark.scheduler.cluster.mesos.Utils._
|
||||
|
||||
class MesosCoarseGrainedSchedulerBackendSuite extends SparkFunSuite
|
||||
|
|
|
|||
|
|
@ -17,8 +17,8 @@
|
|||
|
||||
package org.apache.spark.deploy.yarn
|
||||
|
||||
import scala.collection.mutable.{ArrayBuffer, HashMap, Set}
|
||||
import scala.collection.JavaConverters._
|
||||
import scala.collection.mutable.{ArrayBuffer, HashMap, Set}
|
||||
|
||||
import org.apache.hadoop.conf.Configuration
|
||||
import org.apache.hadoop.yarn.api.records.{ContainerId, Resource}
|
||||
|
|
|
|||
|
|
@ -22,9 +22,9 @@ import java.util.concurrent._
|
|||
import java.util.concurrent.atomic.AtomicInteger
|
||||
import java.util.regex.Pattern
|
||||
|
||||
import scala.collection.JavaConverters._
|
||||
import scala.collection.mutable
|
||||
import scala.collection.mutable.{ArrayBuffer, HashMap, HashSet, Queue}
|
||||
import scala.collection.JavaConverters._
|
||||
import scala.util.control.NonFatal
|
||||
|
||||
import org.apache.hadoop.yarn.api.records._
|
||||
|
|
|
|||
|
|
@ -28,9 +28,9 @@ import org.mockito.Mockito._
|
|||
import org.scalatest.{BeforeAndAfterEach, Matchers}
|
||||
|
||||
import org.apache.spark.{SecurityManager, SparkConf, SparkFunSuite}
|
||||
import org.apache.spark.deploy.yarn.config._
|
||||
import org.apache.spark.deploy.yarn.YarnAllocator._
|
||||
import org.apache.spark.deploy.yarn.YarnSparkHadoopUtil._
|
||||
import org.apache.spark.deploy.yarn.config._
|
||||
import org.apache.spark.rpc.RpcEndpointRef
|
||||
import org.apache.spark.scheduler.SplitInfo
|
||||
import org.apache.spark.util.ManualClock
|
||||
|
|
|
|||
|
|
@ -1,19 +1,19 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.spark.deploy.yarn
|
||||
|
||||
|
|
|
|||
|
|
@ -25,9 +25,9 @@ import org.apache.spark.sql.catalyst._
|
|||
import org.apache.spark.sql.catalyst.catalog._
|
||||
import org.apache.spark.sql.catalyst.encoders.OuterScopes
|
||||
import org.apache.spark.sql.catalyst.expressions._
|
||||
import org.apache.spark.sql.catalyst.expressions.SubExprUtils._
|
||||
import org.apache.spark.sql.catalyst.expressions.aggregate._
|
||||
import org.apache.spark.sql.catalyst.expressions.objects.{LambdaVariable, MapObjects, NewInstance, UnresolvedMapObjects}
|
||||
import org.apache.spark.sql.catalyst.expressions.SubExprUtils._
|
||||
import org.apache.spark.sql.catalyst.plans._
|
||||
import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, _}
|
||||
import org.apache.spark.sql.catalyst.rules._
|
||||
|
|
|
|||
|
|
@ -19,8 +19,8 @@ package org.apache.spark.sql.catalyst.analysis
|
|||
|
||||
import org.apache.spark.sql.AnalysisException
|
||||
import org.apache.spark.sql.catalyst.expressions._
|
||||
import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression
|
||||
import org.apache.spark.sql.catalyst.expressions.SubExprUtils._
|
||||
import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression
|
||||
import org.apache.spark.sql.catalyst.optimizer.BooleanSimplification
|
||||
import org.apache.spark.sql.catalyst.plans._
|
||||
import org.apache.spark.sql.catalyst.plans.logical._
|
||||
|
|
|
|||
|
|
@ -22,11 +22,11 @@ import scala.reflect.ClassTag
|
|||
|
||||
import org.apache.spark.SparkException
|
||||
import org.apache.spark.sql.Row
|
||||
import org.apache.spark.sql.catalyst.expressions._
|
||||
import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, ArrayData, DateTimeUtils, GenericArrayData}
|
||||
import org.apache.spark.sql.catalyst.ScalaReflection
|
||||
import org.apache.spark.sql.catalyst.analysis.GetColumnByOrdinal
|
||||
import org.apache.spark.sql.catalyst.expressions._
|
||||
import org.apache.spark.sql.catalyst.expressions.objects._
|
||||
import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, ArrayData, DateTimeUtils, GenericArrayData}
|
||||
import org.apache.spark.sql.types._
|
||||
import org.apache.spark.unsafe.types.UTF8String
|
||||
|
||||
|
|
|
|||
|
|
@ -19,8 +19,8 @@ package org.apache.spark.sql.catalyst
|
|||
|
||||
import scala.util.control.NonFatal
|
||||
|
||||
import org.apache.spark.sql.catalyst.trees.TreeNode
|
||||
import org.apache.spark.SparkException
|
||||
import org.apache.spark.sql.catalyst.trees.TreeNode
|
||||
|
||||
/**
|
||||
* Functions for attaching and retrieving trees that are associated with errors.
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ package org.apache.spark.sql.catalyst.expressions.aggregate
|
|||
import java.io.{ByteArrayInputStream, ByteArrayOutputStream, DataInputStream, DataOutputStream}
|
||||
import java.util
|
||||
|
||||
import org.apache.spark.SparkException
|
||||
import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow}
|
||||
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
|
||||
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
|
||||
|
|
@ -27,7 +28,6 @@ import org.apache.spark.sql.catalyst.expressions._
|
|||
import org.apache.spark.sql.catalyst.util._
|
||||
import org.apache.spark.sql.types._
|
||||
import org.apache.spark.util.collection.OpenHashMap
|
||||
import org.apache.spark.SparkException
|
||||
|
||||
/**
|
||||
* The Percentile aggregate function returns the exact percentile(s) of numeric column `expr` at
|
||||
|
|
|
|||
|
|
@ -20,10 +20,10 @@ package org.apache.spark.sql.catalyst.expressions.aggregate
|
|||
import scala.collection.generic.Growable
|
||||
import scala.collection.mutable
|
||||
|
||||
import org.apache.spark.sql.catalyst.InternalRow
|
||||
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
|
||||
import org.apache.spark.sql.catalyst.expressions._
|
||||
import org.apache.spark.sql.catalyst.util.GenericArrayData
|
||||
import org.apache.spark.sql.catalyst.InternalRow
|
||||
import org.apache.spark.sql.types._
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -30,9 +30,9 @@ import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
|
|||
import org.apache.spark.sql.catalyst.expressions.codegen._
|
||||
import org.apache.spark.sql.catalyst.util.{ArrayData, MapData}
|
||||
import org.apache.spark.sql.types._
|
||||
import org.apache.spark.unsafe.Platform
|
||||
import org.apache.spark.unsafe.hash.Murmur3_x86_32
|
||||
import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
|
||||
import org.apache.spark.unsafe.Platform
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// This file defines all the expressions for hashing.
|
||||
|
|
|
|||
|
|
@ -24,11 +24,11 @@ import scala.util.parsing.combinator.RegexParsers
|
|||
import com.fasterxml.jackson.core._
|
||||
|
||||
import org.apache.spark.sql.AnalysisException
|
||||
import org.apache.spark.sql.catalyst.InternalRow
|
||||
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
|
||||
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
|
||||
import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
|
||||
import org.apache.spark.sql.catalyst.InternalRow
|
||||
import org.apache.spark.sql.catalyst.json._
|
||||
import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
|
||||
import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, ArrayData, BadRecordException, FailFastMode, GenericArrayData}
|
||||
import org.apache.spark.sql.types._
|
||||
import org.apache.spark.unsafe.types.UTF8String
|
||||
|
|
|
|||
|
|
@ -20,10 +20,10 @@ package org.apache.spark.sql.catalyst.expressions
|
|||
import java.{lang => jl}
|
||||
import java.util.Locale
|
||||
|
||||
import org.apache.spark.sql.catalyst.InternalRow
|
||||
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
|
||||
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
|
||||
import org.apache.spark.sql.catalyst.expressions.codegen._
|
||||
import org.apache.spark.sql.catalyst.InternalRow
|
||||
import org.apache.spark.sql.catalyst.util.NumberConverter
|
||||
import org.apache.spark.sql.types._
|
||||
import org.apache.spark.unsafe.types.UTF8String
|
||||
|
|
|
|||
|
|
@ -22,8 +22,8 @@ import scala.collection.mutable.{ArrayBuffer, Stack}
|
|||
|
||||
import org.apache.spark.sql.catalyst.analysis._
|
||||
import org.apache.spark.sql.catalyst.expressions._
|
||||
import org.apache.spark.sql.catalyst.expressions.aggregate._
|
||||
import org.apache.spark.sql.catalyst.expressions.Literal.{FalseLiteral, TrueLiteral}
|
||||
import org.apache.spark.sql.catalyst.expressions.aggregate._
|
||||
import org.apache.spark.sql.catalyst.expressions.objects.AssertNotNull
|
||||
import org.apache.spark.sql.catalyst.plans._
|
||||
import org.apache.spark.sql.catalyst.plans.logical._
|
||||
|
|
|
|||
|
|
@ -20,8 +20,8 @@ package org.apache.spark.sql.catalyst.optimizer
|
|||
import scala.collection.mutable.ArrayBuffer
|
||||
|
||||
import org.apache.spark.sql.catalyst.expressions._
|
||||
import org.apache.spark.sql.catalyst.expressions.aggregate._
|
||||
import org.apache.spark.sql.catalyst.expressions.SubExprUtils._
|
||||
import org.apache.spark.sql.catalyst.expressions.aggregate._
|
||||
import org.apache.spark.sql.catalyst.plans._
|
||||
import org.apache.spark.sql.catalyst.plans.logical._
|
||||
import org.apache.spark.sql.catalyst.rules._
|
||||
|
|
|
|||
|
|
@ -22,8 +22,8 @@ import java.math.{MathContext, RoundingMode}
|
|||
import scala.util.control.NonFatal
|
||||
|
||||
import org.apache.spark.internal.Logging
|
||||
import org.apache.spark.sql.catalyst.InternalRow
|
||||
import org.apache.spark.sql.AnalysisException
|
||||
import org.apache.spark.sql.catalyst.InternalRow
|
||||
import org.apache.spark.sql.catalyst.expressions._
|
||||
import org.apache.spark.sql.catalyst.expressions.aggregate._
|
||||
import org.apache.spark.sql.catalyst.util.DateTimeUtils
|
||||
|
|
|
|||
|
|
@ -27,10 +27,10 @@ import org.json4s.JsonAST._
|
|||
import org.json4s.JsonDSL._
|
||||
import org.json4s.jackson.JsonMethods._
|
||||
|
||||
import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogStorageFormat, CatalogTable, CatalogTableType, FunctionResource}
|
||||
import org.apache.spark.sql.catalyst.FunctionIdentifier
|
||||
import org.apache.spark.sql.catalyst.ScalaReflection._
|
||||
import org.apache.spark.sql.catalyst.TableIdentifier
|
||||
import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogStorageFormat, CatalogTable, CatalogTableType, FunctionResource}
|
||||
import org.apache.spark.sql.catalyst.errors._
|
||||
import org.apache.spark.sql.catalyst.expressions._
|
||||
import org.apache.spark.sql.catalyst.plans.JoinType
|
||||
|
|
|
|||
|
|
@ -18,8 +18,8 @@
|
|||
package org.apache.spark.sql.types
|
||||
|
||||
import scala.collection.mutable.ArrayBuffer
|
||||
import scala.util.control.NonFatal
|
||||
import scala.util.Try
|
||||
import scala.util.control.NonFatal
|
||||
|
||||
import org.json4s.JsonDSL._
|
||||
|
||||
|
|
|
|||
|
|
@ -22,8 +22,8 @@ import org.scalatest.BeforeAndAfter
|
|||
import org.apache.spark.sql.catalyst.catalog.{InMemoryCatalog, SessionCatalog}
|
||||
import org.apache.spark.sql.catalyst.dsl.expressions._
|
||||
import org.apache.spark.sql.catalyst.expressions._
|
||||
import org.apache.spark.sql.catalyst.expressions.aggregate._
|
||||
import org.apache.spark.sql.catalyst.expressions.Literal.{FalseLiteral, TrueLiteral}
|
||||
import org.apache.spark.sql.catalyst.expressions.aggregate._
|
||||
import org.apache.spark.sql.catalyst.plans.PlanTest
|
||||
import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, Project, Union}
|
||||
import org.apache.spark.sql.types._
|
||||
|
|
|
|||
|
|
@ -20,10 +20,10 @@ package org.apache.spark.sql.catalyst.encoders
|
|||
import scala.reflect.runtime.universe.TypeTag
|
||||
|
||||
import org.apache.spark.sql.AnalysisException
|
||||
import org.apache.spark.sql.catalyst.InternalRow
|
||||
import org.apache.spark.sql.catalyst.dsl.expressions._
|
||||
import org.apache.spark.sql.catalyst.plans.PlanTest
|
||||
import org.apache.spark.sql.catalyst.util.GenericArrayData
|
||||
import org.apache.spark.sql.catalyst.InternalRow
|
||||
import org.apache.spark.sql.types._
|
||||
import org.apache.spark.unsafe.types.UTF8String
|
||||
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@
|
|||
|
||||
package org.apache.spark.sql.catalyst.optimizer
|
||||
|
||||
import org.apache.spark.sql.Row
|
||||
import org.apache.spark.sql.catalyst.analysis._
|
||||
import org.apache.spark.sql.catalyst.catalog.{InMemoryCatalog, SessionCatalog}
|
||||
import org.apache.spark.sql.catalyst.dsl.expressions._
|
||||
|
|
@ -26,7 +27,6 @@ import org.apache.spark.sql.catalyst.plans.PlanTest
|
|||
import org.apache.spark.sql.catalyst.plans.logical._
|
||||
import org.apache.spark.sql.catalyst.rules._
|
||||
import org.apache.spark.sql.internal.SQLConf
|
||||
import org.apache.spark.sql.Row
|
||||
|
||||
class BooleanSimplificationSuite extends PlanTest with PredicateHelper {
|
||||
|
||||
|
|
|
|||
|
|
@ -19,8 +19,8 @@ package org.apache.spark.sql.catalyst.optimizer
|
|||
|
||||
import org.apache.spark.sql.catalyst.dsl.expressions._
|
||||
import org.apache.spark.sql.catalyst.dsl.plans._
|
||||
import org.apache.spark.sql.catalyst.plans.logical._
|
||||
import org.apache.spark.sql.catalyst.plans.PlanTest
|
||||
import org.apache.spark.sql.catalyst.plans.logical._
|
||||
import org.apache.spark.sql.catalyst.rules._
|
||||
|
||||
class CombiningLimitsSuite extends PlanTest {
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue