[SPARK-6459][SQL] Warn when constructing trivially true equals predicate
For example, one might expect the following code to work, but it does not. Now you will at least get a warning with a suggestion to use aliases. ```scala val df = sqlContext.load(path, "parquet") val txns = df.groupBy("cust_id").agg($"cust_id", countDistinct($"day_num").as("txns")) val spend = df.groupBy("cust_id").agg($"cust_id", sum($"extended_price").as("spend")) val rmJoin = txns.join(spend, txns("cust_id") === spend("cust_id"), "inner") ``` Author: Michael Armbrust <michael@databricks.com> Closes #5163 from marmbrus/selfJoinError and squashes the following commits: 16c1f0b [Michael Armbrust] fix visibility 1b57e8d [Michael Armbrust] Warn when constructing trivially true equals predicate
This commit is contained in:
parent
6bdddb6f6f
commit
32efadd050
|
@ -20,6 +20,7 @@ package org.apache.spark.sql
|
|||
import scala.language.implicitConversions
|
||||
|
||||
import org.apache.spark.annotation.Experimental
|
||||
import org.apache.spark.Logging
|
||||
import org.apache.spark.sql.functions.lit
|
||||
import org.apache.spark.sql.catalyst.expressions._
|
||||
import org.apache.spark.sql.catalyst.analysis.{UnresolvedAttribute, UnresolvedStar, UnresolvedGetField}
|
||||
|
@ -46,7 +47,7 @@ private[sql] object Column {
|
|||
* @groupname Ungrouped Support functions for DataFrames.
|
||||
*/
|
||||
@Experimental
|
||||
class Column(protected[sql] val expr: Expression) {
|
||||
class Column(protected[sql] val expr: Expression) extends Logging {
|
||||
|
||||
def this(name: String) = this(name match {
|
||||
case "*" => UnresolvedStar(None)
|
||||
|
@ -109,7 +110,15 @@ class Column(protected[sql] val expr: Expression) {
|
|||
*
|
||||
* @group expr_ops
|
||||
*/
|
||||
def === (other: Any): Column = EqualTo(expr, lit(other).expr)
|
||||
def === (other: Any): Column = {
|
||||
val right = lit(other).expr
|
||||
if (this.expr == right) {
|
||||
logWarning(
|
||||
s"Constructing trivially true equals predicate, '${this.expr} = $right'. " +
|
||||
"Perhaps you need to use aliases.")
|
||||
}
|
||||
EqualTo(expr, right)
|
||||
}
|
||||
|
||||
/**
|
||||
* Equality test.
|
||||
|
|
Loading…
Reference in a new issue