ColumnName

final def !=(arg0: Any): Boolean

final def ##(): Int

def %(other: Any): Column

Modulo (a.k.a.

def &&(other: Any): Column

Boolean AND.

// Scala: The following selects people that are in school and employed at the same time.
people.select( people("inSchool") && people("isEmployed") )

// Java:
people.select( people.col("inSchool").and(people.col("isEmployed")) );

def *(other: Any): Column

Multiplication of this expression and another expression.

// Scala: The following multiplies a person's height by their weight.
people.select( people("height") * people("weight") )

// Java:
people.select( people.col("height").multiply(people.col("weight")) );

def +(other: Any): Column

Sum of this expression and another expression.

// Scala: The following selects the sum of a person's height and weight.
people.select( people("height") + people("weight") )

// Java:
people.select( people.col("height").plus(people.col("weight")) );

def -(other: Any): Column

Subtraction.

// Scala: The following selects the difference between people's height and their weight.
people.select( people("height") - people("weight") )

// Java:
people.select( people.col("height").minus(people.col("weight")) );

def /(other: Any): Column

Division this expression by another expression.

// Scala: The following divides a person's height by their weight.
people.select( people("height") / people("weight") )

// Java:
people.select( people.col("height").divide(people.col("weight")) );

def <(other: Any): Column

Less than.

// Scala: The following selects people younger than 21.
people.select( people("age") < 21 )

// Java:
people.select( people.col("age").lt(21) );

def <=(other: Any): Column

Less than or equal to.

// Scala: The following selects people age 21 or younger than 21.
people.select( people("age") <= 21 )

// Java:
people.select( people.col("age").leq(21) );

def <=>(other: Any): Column

Equality test that is safe for null values.

def =!=(other: Any): Column

Inequality test.

// Scala:
df.select( df("colA") =!= df("colB") )
df.select( !(df("colA") === df("colB")) )

// Java:
import static org.apache.spark.sql.functions.*;
df.filter( col("colA").notEqual(col("colB")) );

final def ==(arg0: Any): Boolean

def ===(other: Any): Column

Equality test.

// Scala:
df.filter( df("colA") === df("colB") )

// Java
import static org.apache.spark.sql.functions.*;
df.filter( col("colA").equalTo(col("colB")) );

def >(other: Any): Column

Greater than.

// Scala: The following selects people older than 21.
people.select( people("age") > 21 )

// Java:
import static org.apache.spark.sql.functions.*;
people.select( people.col("age").gt(21) );

def >=(other: Any): Column

Greater than or equal to an expression.

// Scala: The following selects people age 21 or older than 21.
people.select( people("age") >= 21 )

// Java:
people.select( people.col("age").geq(21) )

def alias(alias: String): Column

Gives the column an alias.

// Renames colA to colB in select output.
df.select($"colA".alias("colB"))

def and(other: Column): Column

Boolean AND.

// Scala: The following selects people that are in school and employed at the same time.
people.select( people("inSchool") && people("isEmployed") )

// Java:
people.select( people.col("inSchool").and(people.col("isEmployed")) );

def apply(extraction: Any): Column

Extracts a value or values from a complex type.

def array(dataType: DataType): StructField

Creates a new StructField of type array.

def as(alias: String, metadata: Metadata): Column

Gives the column an alias with metadata.

val metadata: Metadata = ...
df.select($"colA".as("colB", metadata))

def as(alias: Symbol): Column

Gives the column an alias.

// Renames colA to colB in select output.
df.select($"colA".as('colB))

def as(aliases: Array[String]): Column

Assigns the given aliases to the results of a table generating function.

// Renames colA to colB in select output.
df.select(explode($"myMap").as("key" :: "value" :: Nil))

def as(aliases: Seq[String]): Column

(Scala-specific) Assigns the given aliases to the results of a table generating function.

// Renames colA to colB in select output.
df.select(explode($"myMap").as("key" :: "value" :: Nil))

def as(alias: String): Column

Gives the column an alias.

// Renames colA to colB in select output.
df.select($"colA".as("colB"))

def as[U](implicit arg0: Encoder[U]): TypedColumn[Any, U]

Provides a type hint about the expected return value of this column.

final def asInstanceOf[T0]: T0

def asc: Column

Returns a sort expression based on ascending order of the column.

// Scala: sort a DataFrame by age column in ascending order.
df.sort(df("age").asc)

// Java
df.sort(df.col("age").asc());

def asc_nulls_first: Column

Returns a sort expression based on ascending order of the column, and null values return before non-null values.

// Scala: sort a DataFrame by age column in ascending order and null values appearing first.
df.sort(df("age").asc_nulls_first)

// Java
df.sort(df.col("age").asc_nulls_first());

def asc_nulls_last: Column

Returns a sort expression based on ascending order of the column, and null values appear after non-null values.

// Scala: sort a DataFrame by age column in ascending order and null values appearing last.
df.sort(df("age").asc_nulls_last)

// Java
df.sort(df.col("age").asc_nulls_last());

def between(lowerBound: Any, upperBound: Any): Column

True if the current column is between the lower bound and upper bound, inclusive.

def binary: StructField

Creates a new StructField of type binary.

def bitwiseAND(other: Any): Column

Compute bitwise AND of this expression with another expression.

df.select($"colA".bitwiseAND($"colB"))

def bitwiseOR(other: Any): Column

Compute bitwise OR of this expression with another expression.

df.select($"colA".bitwiseOR($"colB"))

def bitwiseXOR(other: Any): Column

Compute bitwise XOR of this expression with another expression.

df.select($"colA".bitwiseXOR($"colB"))

def boolean: StructField

Creates a new StructField of type boolean.

def byte: StructField

Creates a new StructField of type byte.

def cast(to: String): Column

Casts the column to a different data type, using the canonical string representation of the type.

// Casts colA to integer.
df.select(df("colA").cast("int"))

def cast(to: DataType): Column

Casts the column to a different data type.

// Casts colA to IntegerType.
import org.apache.spark.sql.types.IntegerType
df.select(df("colA").cast(IntegerType))

// equivalent to
df.select(df("colA").cast("int"))

def clone(): AnyRef

def contains(other: Any): Column

Contains the other element.

def date: StructField

Creates a new StructField of type date.

def decimal(precision: Int, scale: Int): StructField

Creates a new StructField of type decimal.

def decimal: StructField

Creates a new StructField of type decimal.

def desc: Column

Returns a sort expression based on the descending order of the column.

// Scala
df.sort(df("age").desc)

// Java
df.sort(df.col("age").desc());

def desc_nulls_first: Column

Returns a sort expression based on the descending order of the column, and null values appear before non-null values.

// Scala: sort a DataFrame by age column in descending order and null values appearing first.
df.sort(df("age").desc_nulls_first)

// Java
df.sort(df.col("age").desc_nulls_first());

def desc_nulls_last: Column

Returns a sort expression based on the descending order of the column, and null values appear after non-null values.

// Scala: sort a DataFrame by age column in descending order and null values appearing last.
df.sort(df("age").desc_nulls_last)

// Java
df.sort(df.col("age").desc_nulls_last());

def divide(other: Any): Column

Division this expression by another expression.

// Scala: The following divides a person's height by their weight.
people.select( people("height") / people("weight") )

// Java:
people.select( people.col("height").divide(people.col("weight")) );

def double: StructField

Creates a new StructField of type double.

def endsWith(literal: String): Column

String ends with another string literal.

def endsWith(other: Column): Column

String ends with.

final def eq(arg0: AnyRef): Boolean

def eqNullSafe(other: Any): Column

Equality test that is safe for null values.

def equalTo(other: Any): Column

Equality test.

// Scala:
df.filter( df("colA") === df("colB") )

// Java
import static org.apache.spark.sql.functions.*;
df.filter( col("colA").equalTo(col("colB")) );

def equals(that: Any): Boolean

def explain(extended: Boolean): Unit

Prints the expression to the console for debugging purposes.

val expr: Expression

def finalize(): Unit

def float: StructField

Creates a new StructField of type float.

def geq(other: Any): Column

Greater than or equal to an expression.

// Scala: The following selects people age 21 or older than 21.
people.select( people("age") >= 21 )

// Java:
people.select( people.col("age").geq(21) )

final def getClass(): Class[_]

def getField(fieldName: String): Column

An expression that gets a field by name in a StructType.

def getItem(key: Any): Column

An expression that gets an item at position ordinal out of an array, or gets a value by key key in a MapType.

def gt(other: Any): Column

Greater than.

// Scala: The following selects people older than 21.
people.select( people("age") > lit(21) )

// Java:
import static org.apache.spark.sql.functions.*;
people.select( people.col("age").gt(21) );

def hashCode(): Int

def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean = false): Boolean

def initializeLogIfNecessary(isInterpreter: Boolean): Unit

def int: StructField

Creates a new StructField of type int.

def isInCollection(values: Iterable[_]): Column

A boolean expression that is evaluated to true if the value of this expression is contained by the provided collection.

def isInCollection(values: Iterable[_]): Column

A boolean expression that is evaluated to true if the value of this expression is contained by the provided collection.

final def isInstanceOf[T0]: Boolean

def isNaN: Column

True if the current expression is NaN.

def isNotNull: Column

True if the current expression is NOT null.

def isNull: Column

True if the current expression is null.

def isTraceEnabled(): Boolean

def isin(list: Any*): Column

A boolean expression that is evaluated to true if the value of this expression is contained by the evaluated values of the arguments.

def leq(other: Any): Column

Less than or equal to.

// Scala: The following selects people age 21 or younger than 21.
people.select( people("age") <= 21 )

// Java:
people.select( people.col("age").leq(21) );

def like(literal: String): Column

SQL like expression.

def log: Logger

def logDebug(msg: ⇒ String, throwable: Throwable): Unit

def logDebug(msg: ⇒ String): Unit

def logError(msg: ⇒ String, throwable: Throwable): Unit

def logError(msg: ⇒ String): Unit

def logInfo(msg: ⇒ String, throwable: Throwable): Unit

def logInfo(msg: ⇒ String): Unit

def logName: String

def logTrace(msg: ⇒ String, throwable: Throwable): Unit

def logTrace(msg: ⇒ String): Unit

def logWarning(msg: ⇒ String, throwable: Throwable): Unit

def logWarning(msg: ⇒ String): Unit

def long: StructField

Creates a new StructField of type long.

def lt(other: Any): Column

Less than.

// Scala: The following selects people younger than 21.
people.select( people("age") < 21 )

// Java:
people.select( people.col("age").lt(21) );

def map(mapType: MapType): StructField

def map(keyType: DataType, valueType: DataType): StructField

Creates a new StructField of type map.

def minus(other: Any): Column

Subtraction.

// Scala: The following selects the difference between people's height and their weight.
people.select( people("height") - people("weight") )

// Java:
people.select( people.col("height").minus(people.col("weight")) );

def mod(other: Any): Column

Modulo (a.k.a.

def multiply(other: Any): Column

Multiplication of this expression and another expression.

// Scala: The following multiplies a person's height by their weight.
people.select( people("height") * people("weight") )

// Java:
people.select( people.col("height").multiply(people.col("weight")) );

def name(alias: String): Column

Gives the column a name (alias).

// Renames colA to colB in select output.
df.select($"colA".name("colB"))

final def ne(arg0: AnyRef): Boolean

def notEqual(other: Any): Column

Inequality test.

// Scala:
df.select( df("colA") !== df("colB") )
df.select( !(df("colA") === df("colB")) )

// Java:
import static org.apache.spark.sql.functions.*;
df.filter( col("colA").notEqual(col("colB")) );

final def notify(): Unit

final def notifyAll(): Unit

def or(other: Column): Column

Boolean OR.

// Scala: The following selects people that are in school or employed.
people.filter( people("inSchool") || people("isEmployed") )

// Java:
people.filter( people.col("inSchool").or(people.col("isEmployed")) );

def otherwise(value: Any): Column

Evaluates a list of conditions and returns one of multiple possible result expressions.

// Example: encoding gender string column into integer.

// Scala:
people.select(when(people("gender") === "male", 0)
  .when(people("gender") === "female", 1)
  .otherwise(2))

// Java:
people.select(when(col("gender").equalTo("male"), 0)
  .when(col("gender").equalTo("female"), 1)
  .otherwise(2))

def over(): Column

Defines an empty analytic clause.

df.select(
  sum("price").over(),
  avg("price").over()
)

def over(window: WindowSpec): Column

Defines a windowing column.

val w = Window.partitionBy("name").orderBy("id")
df.select(
  sum("price").over(w.rangeBetween(Window.unboundedPreceding, 2)),
  avg("price").over(w.rowsBetween(Window.currentRow, 4))
)

def plus(other: Any): Column

Sum of this expression and another expression.

// Scala: The following selects the sum of a person's height and weight.
people.select( people("height") + people("weight") )

// Java:
people.select( people.col("height").plus(people.col("weight")) );

def rlike(literal: String): Column

SQL RLIKE expression (LIKE with Regex).

def short: StructField

Creates a new StructField of type short.

def startsWith(literal: String): Column

String starts with another string literal.

def startsWith(other: Column): Column

String starts with.

def string: StructField

Creates a new StructField of type string.

def struct(structType: StructType): StructField

Creates a new StructField of type struct.

def struct(fields: StructField*): StructField

Creates a new StructField of type struct.

def substr(startPos: Int, len: Int): Column

An expression that returns a substring.

def substr(startPos: Column, len: Column): Column

An expression that returns a substring.

final def synchronized[T0](arg0: ⇒ T0): T0

def timestamp: StructField

Creates a new StructField of type timestamp.

def toString(): String

def unary_!: Column

Inversion of boolean expression, i.e.

// Scala: select rows that are not active (isActive === false)
df.filter( !df("isActive") )

// Java:
import static org.apache.spark.sql.functions.*;
df.filter( not(df.col("isActive")) );

def unary_-: Column

Unary minus, i.e.

// Scala: select the amount column and negates all values.
df.select( -df("amount") )

// Java:
import static org.apache.spark.sql.functions.*;
df.select( negate(col("amount") );

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

def when(condition: Column, value: Any): Column

Evaluates a list of conditions and returns one of multiple possible result expressions.

// Example: encoding gender string column into integer.

// Scala:
people.select(when(people("gender") === "male", 0)
  .when(people("gender") === "female", 1)
  .otherwise(2))

// Java:
people.select(when(col("gender").equalTo("male"), 0)
  .when(col("gender").equalTo("female"), 1)
  .otherwise(2))

def ||(other: Any): Column

Boolean OR.

// Scala: The following selects people that are in school or employed.
people.filter( people("inSchool") || people("isEmployed") )

// Java:
people.filter( people.col("inSchool").or(people.col("isEmployed")) );

def !==(other: Any): Column

Inequality test.

// Scala:
df.select( df("colA") !== df("colB") )
df.select( !(df("colA") === df("colB")) )

// Java:
import static org.apache.spark.sql.functions.*;
df.filter( col("colA").notEqual(col("colB")) );