functions

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def abs(e: Column): Column

Computes the absolute value.
Computes the absolute value.

Since
1.3.0
def acos(columnName: String): Column

Computes the cosine inverse of the given column; the returned angle is in the range 0.0 through pi.
Computes the cosine inverse of the given column; the returned angle is in the range 0.0 through pi.

Since
1.4.0
def acos(e: Column): Column

Computes the cosine inverse of the given value; the returned angle is in the range 0.0 through pi.
Computes the cosine inverse of the given value; the returned angle is in the range 0.0 through pi.

Since
1.4.0
def add_months(startDate: Column, numMonths: Int): Column

Returns the date that is numMonths after startDate.
Returns the date that is numMonths after startDate.

Since
1.5.0
def approx_count_distinct(columnName: String, rsd: Double): Column

Aggregate function: returns the approximate number of distinct items in a group.
Aggregate function: returns the approximate number of distinct items in a group.
rsd
maximum estimation error allowed (default = 0.05)

Since
2.1.0
def approx_count_distinct(e: Column, rsd: Double): Column

Aggregate function: returns the approximate number of distinct items in a group.
Aggregate function: returns the approximate number of distinct items in a group.
rsd
maximum estimation error allowed (default = 0.05)

Since
2.1.0
def approx_count_distinct(columnName: String): Column

Aggregate function: returns the approximate number of distinct items in a group.
Aggregate function: returns the approximate number of distinct items in a group.

Since
2.1.0
def approx_count_distinct(e: Column): Column

Aggregate function: returns the approximate number of distinct items in a group.
Aggregate function: returns the approximate number of distinct items in a group.

Since
2.1.0
def array(colName: String, colNames: String*): Column

Creates a new array column.
Creates a new array column. The input columns must all have the same data type.

Annotations
@varargs()
Since
1.4.0
def array(cols: Column*): Column

Creates a new array column.
Creates a new array column. The input columns must all have the same data type.

Annotations
@varargs()
Since
1.4.0
def array_contains(column: Column, value: Any): Column

Returns null if the array is null, true if the array contains value, and false otherwise.
Returns null if the array is null, true if the array contains value, and false otherwise.

Since
1.5.0
final def asInstanceOf[T0]: T0

Definition Classes
Any
def asc(columnName: String): Column

Returns a sort expression based on ascending order of the column.
Returns a sort expression based on ascending order of the column.
```
df.sort(asc("dept"), desc("age"))
```
Since
1.3.0
def asc_nulls_first(columnName: String): Column

Returns a sort expression based on ascending order of the column, and null values return before non-null values.
Returns a sort expression based on ascending order of the column, and null values return before non-null values.
```
df.sort(asc_nulls_last("dept"), desc("age"))
```
Since
2.1.0
def asc_nulls_last(columnName: String): Column

Returns a sort expression based on ascending order of the column, and null values appear after non-null values.
Returns a sort expression based on ascending order of the column, and null values appear after non-null values.
```
df.sort(asc_nulls_last("dept"), desc("age"))
```
Since
2.1.0
def ascii(e: Column): Column

Computes the numeric value of the first character of the string column, and returns the result as an int column.
Computes the numeric value of the first character of the string column, and returns the result as an int column.

Since
1.5.0
def asin(columnName: String): Column

Computes the sine inverse of the given column; the returned angle is in the range -pi/2 through pi/2.
Computes the sine inverse of the given column; the returned angle is in the range -pi/2 through pi/2.

Since
1.4.0
def asin(e: Column): Column

Computes the sine inverse of the given value; the returned angle is in the range -pi/2 through pi/2.
Computes the sine inverse of the given value; the returned angle is in the range -pi/2 through pi/2.

Since
1.4.0
def atan(columnName: String): Column

Computes the tangent inverse of the given column; the returned angle is in the range -pi/2 through pi/2
Computes the tangent inverse of the given column; the returned angle is in the range -pi/2 through pi/2

Since
1.4.0
def atan(e: Column): Column

Computes the tangent inverse of the given column; the returned angle is in the range -pi/2 through pi/2
Computes the tangent inverse of the given column; the returned angle is in the range -pi/2 through pi/2

Since
1.4.0
def atan2(l: Double, rightName: String): Column

Returns the angle theta from the conversion of rectangular coordinates (x, y) to polar coordinates (r, theta).
Returns the angle theta from the conversion of rectangular coordinates (x, y) to polar coordinates (r, theta).

Since
1.4.0
def atan2(l: Double, r: Column): Column

Returns the angle theta from the conversion of rectangular coordinates (x, y) to polar coordinates (r, theta).
Returns the angle theta from the conversion of rectangular coordinates (x, y) to polar coordinates (r, theta).

Since
1.4.0
def atan2(leftName: String, r: Double): Column

Returns the angle theta from the conversion of rectangular coordinates (x, y) to polar coordinates (r, theta).
Returns the angle theta from the conversion of rectangular coordinates (x, y) to polar coordinates (r, theta).

Since
1.4.0
def atan2(l: Column, r: Double): Column

Returns the angle theta from the conversion of rectangular coordinates (x, y) to polar coordinates (r, theta).
Returns the angle theta from the conversion of rectangular coordinates (x, y) to polar coordinates (r, theta).

Since
1.4.0
def atan2(leftName: String, rightName: String): Column

Returns the angle theta from the conversion of rectangular coordinates (x, y) to polar coordinates (r, theta).
Returns the angle theta from the conversion of rectangular coordinates (x, y) to polar coordinates (r, theta).

Since
1.4.0
def atan2(leftName: String, r: Column): Column

Returns the angle theta from the conversion of rectangular coordinates (x, y) to polar coordinates (r, theta).
Returns the angle theta from the conversion of rectangular coordinates (x, y) to polar coordinates (r, theta).

Since
1.4.0
def atan2(l: Column, rightName: String): Column

Returns the angle theta from the conversion of rectangular coordinates (x, y) to polar coordinates (r, theta).
Returns the angle theta from the conversion of rectangular coordinates (x, y) to polar coordinates (r, theta).

Since
1.4.0
def atan2(l: Column, r: Column): Column

Returns the angle theta from the conversion of rectangular coordinates (x, y) to polar coordinates (r, theta).
Returns the angle theta from the conversion of rectangular coordinates (x, y) to polar coordinates (r, theta). Units in radians.

Since
1.4.0
def avg(columnName: String): Column

Aggregate function: returns the average of the values in a group.
Aggregate function: returns the average of the values in a group.

Since
1.3.0
def avg(e: Column): Column

Aggregate function: returns the average of the values in a group.
Aggregate function: returns the average of the values in a group.

Since
1.3.0
def base64(e: Column): Column

Computes the BASE64 encoding of a binary column and returns it as a string column.
Computes the BASE64 encoding of a binary column and returns it as a string column. This is the reverse of unbase64.

Since
1.5.0
def bin(columnName: String): Column

An expression that returns the string representation of the binary value of the given long column.
An expression that returns the string representation of the binary value of the given long column. For example, bin("12") returns "1100".

Since
1.5.0
def bin(e: Column): Column

An expression that returns the string representation of the binary value of the given long column.
An expression that returns the string representation of the binary value of the given long column. For example, bin("12") returns "1100".

Since
1.5.0
def bitwiseNOT(e: Column): Column

Computes bitwise NOT.
Computes bitwise NOT.

Since
1.4.0
def broadcast[T](df: Dataset[T]): Dataset[T]

Marks a DataFrame as small enough for use in broadcast joins.
Marks a DataFrame as small enough for use in broadcast joins.
The following example marks the right DataFrame for broadcast hash join using joinKey.
```
// left and right are DataFrames
left.join(broadcast(right), "joinKey")
```
Since
1.5.0
def bround(e: Column, scale: Int): Column

Round the value of e to scale decimal places with HALF_EVEN round mode if scale is greater than or equal to 0 or at integral part when scale is less than 0.
Round the value of e to scale decimal places with HALF_EVEN round mode if scale is greater than or equal to 0 or at integral part when scale is less than 0.

Since
2.0.0
def bround(e: Column): Column

Returns the value of the column e rounded to 0 decimal places with HALF_EVEN round mode.
Returns the value of the column e rounded to 0 decimal places with HALF_EVEN round mode.

Since
2.0.0

def callUDF(udfName: String, cols: Column*): Column

Call an user-defined function.

Call an user-defined function. Example:

import org.apache.spark.sql._

val df = Seq(("id1", 1), ("id2", 4), ("id3", 5)).toDF("id", "value")
val spark = df.sparkSession
spark.udf.register("simpleUDF", (v: Int) => v * v)
df.select($"id", callUDF("simpleUDF", $"value"))

Annotations: @varargs()
Since: 1.5.0

def cbrt(columnName: String): Column

Computes the cube-root of the given column.
Computes the cube-root of the given column.

Since
1.4.0
def cbrt(e: Column): Column

Computes the cube-root of the given value.
Computes the cube-root of the given value.

Since
1.4.0
def ceil(columnName: String): Column

Computes the ceiling of the given column.
Computes the ceiling of the given column.

Since
1.4.0
def ceil(e: Column): Column

Computes the ceiling of the given value.
Computes the ceiling of the given value.

Since
1.4.0
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def coalesce(e: Column*): Column

Returns the first column that is not null, or null if all inputs are null.
Returns the first column that is not null, or null if all inputs are null.
For example, coalesce(a, b, c) will return a if a is not null, or b if a is null and b is not null, or c if both a and b are null but c is not null.

Annotations
@varargs()
Since
1.3.0
def col(colName: String): Column

Returns a Column based on the given column name.
Returns a Column based on the given column name.

Since
1.3.0
def collect_list(columnName: String): Column

Aggregate function: returns a list of objects with duplicates.
Aggregate function: returns a list of objects with duplicates.

Since
1.6.0
def collect_list(e: Column): Column

Aggregate function: returns a list of objects with duplicates.
Aggregate function: returns a list of objects with duplicates.

Since
1.6.0
def collect_set(columnName: String): Column

Aggregate function: returns a set of objects with duplicate elements eliminated.
Aggregate function: returns a set of objects with duplicate elements eliminated.

Since
1.6.0
def collect_set(e: Column): Column

Aggregate function: returns a set of objects with duplicate elements eliminated.
Aggregate function: returns a set of objects with duplicate elements eliminated.

Since
1.6.0
def column(colName: String): Column

Returns a Column based on the given column name.
Returns a Column based on the given column name. Alias of col.

Since
1.3.0
def concat(exprs: Column*): Column

Concatenates multiple input columns together into a single column.
Concatenates multiple input columns together into a single column. If all inputs are binary, concat returns an output as binary. Otherwise, it returns as string.

Annotations
@varargs()
Since
1.5.0
def concat_ws(sep: String, exprs: Column*): Column

Concatenates multiple input string columns together into a single string column, using the given separator.
Concatenates multiple input string columns together into a single string column, using the given separator.

Annotations
@varargs()
Since
1.5.0
def conv(num: Column, fromBase: Int, toBase: Int): Column

Convert a number in a string column from one base to another.
Convert a number in a string column from one base to another.

Since
1.5.0
def corr(columnName1: String, columnName2: String): Column

Aggregate function: returns the Pearson Correlation Coefficient for two columns.
Aggregate function: returns the Pearson Correlation Coefficient for two columns.

Since
1.6.0
def corr(column1: Column, column2: Column): Column

Aggregate function: returns the Pearson Correlation Coefficient for two columns.
Aggregate function: returns the Pearson Correlation Coefficient for two columns.

Since
1.6.0
def cos(columnName: String): Column

Computes the cosine of the given column.
Computes the cosine of the given column.

Since
1.4.0
def cos(e: Column): Column

Computes the cosine of the given value.
Computes the cosine of the given value. Units in radians.

Since
1.4.0
def cosh(columnName: String): Column

Computes the hyperbolic cosine of the given column.
Computes the hyperbolic cosine of the given column.

Since
1.4.0
def cosh(e: Column): Column

Computes the hyperbolic cosine of the given value.
Computes the hyperbolic cosine of the given value.

Since
1.4.0
def count(columnName: String): TypedColumn[Any, Long]

Aggregate function: returns the number of items in a group.
Aggregate function: returns the number of items in a group.

Since
1.3.0
def count(e: Column): Column

Aggregate function: returns the number of items in a group.
Aggregate function: returns the number of items in a group.

Since
1.3.0
def countDistinct(columnName: String, columnNames: String*): Column

Aggregate function: returns the number of distinct items in a group.
Aggregate function: returns the number of distinct items in a group.

Annotations
@varargs()
Since
1.3.0
def countDistinct(expr: Column, exprs: Column*): Column

Aggregate function: returns the number of distinct items in a group.
Aggregate function: returns the number of distinct items in a group.

Annotations
@varargs()
Since
1.3.0
def covar_pop(columnName1: String, columnName2: String): Column

Aggregate function: returns the population covariance for two columns.
Aggregate function: returns the population covariance for two columns.

Since
2.0.0
def covar_pop(column1: Column, column2: Column): Column

Aggregate function: returns the population covariance for two columns.
Aggregate function: returns the population covariance for two columns.

Since
2.0.0
def covar_samp(columnName1: String, columnName2: String): Column

Aggregate function: returns the sample covariance for two columns.
Aggregate function: returns the sample covariance for two columns.

Since
2.0.0
def covar_samp(column1: Column, column2: Column): Column

Aggregate function: returns the sample covariance for two columns.
Aggregate function: returns the sample covariance for two columns.

Since
2.0.0
def crc32(e: Column): Column

Calculates the cyclic redundancy check value (CRC32) of a binary column and returns the value as a bigint.
Calculates the cyclic redundancy check value (CRC32) of a binary column and returns the value as a bigint.

Since
1.5.0
def cume_dist(): Column

Window function: returns the cumulative distribution of values within a window partition, i.e.
Window function: returns the cumulative distribution of values within a window partition, i.e. the fraction of rows that are below the current row.
```
N = total number of rows in the partition
cumeDist(x) = number of values before (and including) x / N
```
Since
1.6.0
def currentRow(): Column

Window function: returns the special frame boundary that represents the current row in the window partition.
Window function: returns the special frame boundary that represents the current row in the window partition.

Since
2.3.0
def current_date(): Column

Returns the current date as a date column.
Returns the current date as a date column.

Since
1.5.0
def current_timestamp(): Column

Returns the current timestamp as a timestamp column.
Returns the current timestamp as a timestamp column.

Since
1.5.0
def date_add(start: Column, days: Int): Column

Returns the date that is days days after start
Returns the date that is days days after start

Since
1.5.0
def date_format(dateExpr: Column, format: String): Column

Converts a date/timestamp/string to a value of string in the format specified by the date format given by the second argument.
Converts a date/timestamp/string to a value of string in the format specified by the date format given by the second argument.
A pattern dd.MM.yyyy would return a string like 18.03.1993. All pattern letters of java.text.SimpleDateFormat can be used.

Since
1.5.0
Note
Use specialized functions like year whenever possible as they benefit from a specialized implementation.
def date_sub(start: Column, days: Int): Column

Returns the date that is days days before start
Returns the date that is days days before start

Since
1.5.0
def date_trunc(format: String, timestamp: Column): Column

Returns timestamp truncated to the unit specified by the format.
Returns timestamp truncated to the unit specified by the format.

Since
2.3.0
def datediff(end: Column, start: Column): Column

Returns the number of days from start to end.
Returns the number of days from start to end.

Since
1.5.0
def dayofmonth(e: Column): Column

Extracts the day of the month as an integer from a given date/timestamp/string.
Extracts the day of the month as an integer from a given date/timestamp/string.

Since
1.5.0
def dayofweek(e: Column): Column

Extracts the day of the week as an integer from a given date/timestamp/string.
Extracts the day of the week as an integer from a given date/timestamp/string.

Since
2.3.0
def dayofyear(e: Column): Column

Extracts the day of the year as an integer from a given date/timestamp/string.
Extracts the day of the year as an integer from a given date/timestamp/string.

Since
1.5.0
def decode(value: Column, charset: String): Column

Computes the first argument into a string from a binary using the provided character set (one of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16').
Computes the first argument into a string from a binary using the provided character set (one of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16'). If either argument is null, the result will also be null.

Since
1.5.0
def degrees(columnName: String): Column

Converts an angle measured in radians to an approximately equivalent angle measured in degrees.
Converts an angle measured in radians to an approximately equivalent angle measured in degrees.

Since
2.1.0
def degrees(e: Column): Column

Converts an angle measured in radians to an approximately equivalent angle measured in degrees.
Converts an angle measured in radians to an approximately equivalent angle measured in degrees.

Since
2.1.0
def dense_rank(): Column

Window function: returns the rank of rows within a window partition, without any gaps.
Window function: returns the rank of rows within a window partition, without any gaps.
The difference between rank and dense_rank is that denseRank leaves no gaps in ranking sequence when there are ties. That is, if you were ranking a competition using dense_rank and had three people tie for second place, you would say that all three were in second place and that the next person came in third. Rank would give me sequential numbers, making the person that came in third place (after the ties) would register as coming in fifth.
This is equivalent to the DENSE_RANK function in SQL.

Since
1.6.0
def desc(columnName: String): Column

Returns a sort expression based on the descending order of the column.
Returns a sort expression based on the descending order of the column.
```
df.sort(asc("dept"), desc("age"))
```
Since
1.3.0
def desc_nulls_first(columnName: String): Column

Returns a sort expression based on the descending order of the column, and null values appear before non-null values.
Returns a sort expression based on the descending order of the column, and null values appear before non-null values.
```
df.sort(asc("dept"), desc_nulls_first("age"))
```
Since
2.1.0
def desc_nulls_last(columnName: String): Column

Returns a sort expression based on the descending order of the column, and null values appear after non-null values.
Returns a sort expression based on the descending order of the column, and null values appear after non-null values.
```
df.sort(asc("dept"), desc_nulls_last("age"))
```
Since
2.1.0
def encode(value: Column, charset: String): Column

Computes the first argument into a binary from a string using the provided character set (one of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16').
Computes the first argument into a binary from a string using the provided character set (one of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16'). If either argument is null, the result will also be null.

Since
1.5.0
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def exp(columnName: String): Column

Computes the exponential of the given column.
Computes the exponential of the given column.

Since
1.4.0
def exp(e: Column): Column

Computes the exponential of the given value.
Computes the exponential of the given value.

Since
1.4.0
def explode(e: Column): Column

Creates a new row for each element in the given array or map column.
Creates a new row for each element in the given array or map column.

Since
1.3.0
def explode_outer(e: Column): Column

Creates a new row for each element in the given array or map column.
Creates a new row for each element in the given array or map column. Unlike explode, if the array/map is null or empty then null is produced.

Since
2.2.0
def expm1(columnName: String): Column

Computes the exponential of the given column.
Computes the exponential of the given column.

Since
1.4.0
def expm1(e: Column): Column

Computes the exponential of the given value minus one.
Computes the exponential of the given value minus one.

Since
1.4.0
def expr(expr: String): Column

Parses the expression string into the column that it represents, similar to Dataset#selectExpr.
Parses the expression string into the column that it represents, similar to Dataset#selectExpr.
```
// get the number of words of each length
df.groupBy(expr("length(word)")).count()
```
def factorial(e: Column): Column

Computes the factorial of the given value.
Computes the factorial of the given value.

Since
1.5.0
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
def first(columnName: String): Column

Aggregate function: returns the first value of a column in a group.
Aggregate function: returns the first value of a column in a group.
The function by default returns the first values it sees. It will return the first non-null value it sees when ignoreNulls is set to true. If all values are null, then null is returned.

Since
1.3.0
def first(e: Column): Column

Aggregate function: returns the first value in a group.
Aggregate function: returns the first value in a group.
The function by default returns the first values it sees. It will return the first non-null value it sees when ignoreNulls is set to true. If all values are null, then null is returned.

Since
1.3.0
def first(columnName: String, ignoreNulls: Boolean): Column

Aggregate function: returns the first value of a column in a group.
Aggregate function: returns the first value of a column in a group.
The function by default returns the first values it sees. It will return the first non-null value it sees when ignoreNulls is set to true. If all values are null, then null is returned.

Since
2.0.0
def first(e: Column, ignoreNulls: Boolean): Column

Aggregate function: returns the first value in a group.
Aggregate function: returns the first value in a group.
The function by default returns the first values it sees. It will return the first non-null value it sees when ignoreNulls is set to true. If all values are null, then null is returned.

Since
2.0.0
def floor(columnName: String): Column

Computes the floor of the given column.
Computes the floor of the given column.

Since
1.4.0
def floor(e: Column): Column

Computes the floor of the given value.
Computes the floor of the given value.

Since
1.4.0
def format_number(x: Column, d: Int): Column

Formats numeric column x to a format like '#,###,###.##', rounded to d decimal places with HALF_EVEN round mode, and returns the result as a string column.
Formats numeric column x to a format like '#,###,###.##', rounded to d decimal places with HALF_EVEN round mode, and returns the result as a string column.
If d is 0, the result has no decimal point or fractional part. If d is less than 0, the result will be null.

Since
1.5.0
def format_string(format: String, arguments: Column*): Column

Formats the arguments in printf-style and returns the result as a string column.
Formats the arguments in printf-style and returns the result as a string column.

Annotations
@varargs()
Since
1.5.0
def from_json(e: Column, schema: String, options: Map[String, String]): Column

(Scala-specific) Parses a column containing a JSON string into a StructType or ArrayType of StructTypes with the specified schema.
(Scala-specific) Parses a column containing a JSON string into a StructType or ArrayType of StructTypes with the specified schema. Returns null, in the case of an unparseable string.
e
a string column containing JSON data.
schema
the schema to use when parsing the json string as a json string, it could be a JSON format string or a DDL-formatted string.

Since
2.3.0
def from_json(e: Column, schema: String, options: Map[String, String]): Column

(Java-specific) Parses a column containing a JSON string into a StructType or ArrayType of StructTypes with the specified schema.
(Java-specific) Parses a column containing a JSON string into a StructType or ArrayType of StructTypes with the specified schema. Returns null, in the case of an unparseable string.
e
a string column containing JSON data.
schema
the schema to use when parsing the json string as a json string. In Spark 2.1, the user-provided schema has to be in JSON format. Since Spark 2.2, the DDL format is also supported for the schema.

Since
2.1.0
def from_json(e: Column, schema: DataType): Column

Parses a column containing a JSON string into a StructType or ArrayType of StructTypes with the specified schema.
Parses a column containing a JSON string into a StructType or ArrayType of StructTypes with the specified schema. Returns null, in the case of an unparseable string.
e
a string column containing JSON data.
schema
the schema to use when parsing the json string

Since
2.2.0
def from_json(e: Column, schema: StructType): Column

Parses a column containing a JSON string into a StructType with the specified schema.
Parses a column containing a JSON string into a StructType with the specified schema. Returns null, in the case of an unparseable string.
e
a string column containing JSON data.
schema
the schema to use when parsing the json string

Since
2.1.0
def from_json(e: Column, schema: DataType, options: Map[String, String]): Column

(Java-specific) Parses a column containing a JSON string into a StructType or ArrayType of StructTypes with the specified schema.
(Java-specific) Parses a column containing a JSON string into a StructType or ArrayType of StructTypes with the specified schema. Returns null, in the case of an unparseable string.
e
a string column containing JSON data.
schema
the schema to use when parsing the json string
options
options to control how the json is parsed. accepts the same options and the json data source.

Since
2.2.0
def from_json(e: Column, schema: StructType, options: Map[String, String]): Column

(Java-specific) Parses a column containing a JSON string into a StructType with the specified schema.
(Java-specific) Parses a column containing a JSON string into a StructType with the specified schema. Returns null, in the case of an unparseable string.
e
a string column containing JSON data.
schema
the schema to use when parsing the json string
options
options to control how the json is parsed. accepts the same options and the json data source.

Since
2.1.0
def from_json(e: Column, schema: DataType, options: Map[String, String]): Column

(Scala-specific) Parses a column containing a JSON string into a StructType or ArrayType of StructTypes with the specified schema.
(Scala-specific) Parses a column containing a JSON string into a StructType or ArrayType of StructTypes with the specified schema. Returns null, in the case of an unparseable string.
e
a string column containing JSON data.
schema
the schema to use when parsing the json string
options
options to control how the json is parsed. accepts the same options and the json data source.

Since
2.2.0
def from_json(e: Column, schema: StructType, options: Map[String, String]): Column

(Scala-specific) Parses a column containing a JSON string into a StructType with the specified schema.
(Scala-specific) Parses a column containing a JSON string into a StructType with the specified schema. Returns null, in the case of an unparseable string.
e
a string column containing JSON data.
schema
the schema to use when parsing the json string
options
options to control how the json is parsed. Accepts the same options as the json data source.

Since
2.1.0
def from_unixtime(ut: Column, f: String): Column

Converts the number of seconds from unix epoch (1970-01-01 00:00:00 UTC) to a string representing the timestamp of that moment in the current system time zone in the given format.
Converts the number of seconds from unix epoch (1970-01-01 00:00:00 UTC) to a string representing the timestamp of that moment in the current system time zone in the given format.

Since
1.5.0
def from_unixtime(ut: Column): Column

Converts the number of seconds from unix epoch (1970-01-01 00:00:00 UTC) to a string representing the timestamp of that moment in the current system time zone in the given format.
Converts the number of seconds from unix epoch (1970-01-01 00:00:00 UTC) to a string representing the timestamp of that moment in the current system time zone in the given format.

Since
1.5.0
def from_utc_timestamp(ts: Column, tz: String): Column

Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in UTC, and renders that time as a timestamp in the given time zone.
Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in UTC, and renders that time as a timestamp in the given time zone. For example, 'GMT+1' would yield '2017-07-14 03:40:00.0'.

Since
1.5.0
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def get_json_object(e: Column, path: String): Column

Extracts json object from a json string based on json path specified, and returns json string of the extracted json object.
Extracts json object from a json string based on json path specified, and returns json string of the extracted json object. It will return null if the input json string is invalid.

Since
1.6.0
def greatest(columnName: String, columnNames: String*): Column

Returns the greatest value of the list of column names, skipping null values.
Returns the greatest value of the list of column names, skipping null values. This function takes at least 2 parameters. It will return null iff all parameters are null.

Annotations
@varargs()
Since
1.5.0
def greatest(exprs: Column*): Column

Returns the greatest value of the list of values, skipping null values.
Returns the greatest value of the list of values, skipping null values. This function takes at least 2 parameters. It will return null iff all parameters are null.

Annotations
@varargs()
Since
1.5.0
def grouping(columnName: String): Column

Aggregate function: indicates whether a specified column in a GROUP BY list is aggregated or not, returns 1 for aggregated or 0 for not aggregated in the result set.
Aggregate function: indicates whether a specified column in a GROUP BY list is aggregated or not, returns 1 for aggregated or 0 for not aggregated in the result set.

Since
2.0.0
def grouping(e: Column): Column

Aggregate function: indicates whether a specified column in a GROUP BY list is aggregated or not, returns 1 for aggregated or 0 for not aggregated in the result set.
Aggregate function: indicates whether a specified column in a GROUP BY list is aggregated or not, returns 1 for aggregated or 0 for not aggregated in the result set.

Since
2.0.0
def grouping_id(colName: String, colNames: String*): Column

Aggregate function: returns the level of grouping, equals to
Aggregate function: returns the level of grouping, equals to
```
(grouping(c1) <<; (n-1)) + (grouping(c2) <<; (n-2)) + ... + grouping(cn)
```
Since
2.0.0
Note
The list of columns should match with grouping columns exactly.
def grouping_id(cols: Column*): Column

Aggregate function: returns the level of grouping, equals to
Aggregate function: returns the level of grouping, equals to
```
(grouping(c1) <<; (n-1)) + (grouping(c2) <<; (n-2)) + ... + grouping(cn)
```
Since
2.0.0
Note
The list of columns should match with grouping columns exactly, or empty (means all the grouping columns).
def hash(cols: Column*): Column

Calculates the hash code of given columns, and returns the result as an int column.
Calculates the hash code of given columns, and returns the result as an int column.

Annotations
@varargs()
Since
2.0.0
def hashCode(): Int

Definition Classes
AnyRef → Any
def hex(column: Column): Column

Computes hex value of the given column.
Computes hex value of the given column.

Since
1.5.0
def hour(e: Column): Column

Extracts the hours as an integer from a given date/timestamp/string.
Extracts the hours as an integer from a given date/timestamp/string.

Since
1.5.0
def hypot(l: Double, rightName: String): Column

Computes sqrt(a² + b²) without intermediate overflow or underflow.
Computes sqrt(a² + b²) without intermediate overflow or underflow.

Since
1.4.0
def hypot(l: Double, r: Column): Column

Computes sqrt(a² + b²) without intermediate overflow or underflow.
Computes sqrt(a² + b²) without intermediate overflow or underflow.

Since
1.4.0
def hypot(leftName: String, r: Double): Column

Computes sqrt(a² + b²) without intermediate overflow or underflow.
Computes sqrt(a² + b²) without intermediate overflow or underflow.

Since
1.4.0
def hypot(l: Column, r: Double): Column

Computes sqrt(a² + b²) without intermediate overflow or underflow.
Computes sqrt(a² + b²) without intermediate overflow or underflow.

Since
1.4.0
def hypot(leftName: String, rightName: String): Column

Computes sqrt(a² + b²) without intermediate overflow or underflow.
Computes sqrt(a² + b²) without intermediate overflow or underflow.

Since
1.4.0
def hypot(leftName: String, r: Column): Column

Computes sqrt(a² + b²) without intermediate overflow or underflow.
Computes sqrt(a² + b²) without intermediate overflow or underflow.

Since
1.4.0
def hypot(l: Column, rightName: String): Column

Computes sqrt(a² + b²) without intermediate overflow or underflow.
Computes sqrt(a² + b²) without intermediate overflow or underflow.

Since
1.4.0
def hypot(l: Column, r: Column): Column

Computes sqrt(a² + b²) without intermediate overflow or underflow.
Computes sqrt(a² + b²) without intermediate overflow or underflow.

Since
1.4.0
def initcap(e: Column): Column

Returns a new string column by converting the first letter of each word to uppercase.
Returns a new string column by converting the first letter of each word to uppercase. Words are delimited by whitespace.
For example, "hello world" will become "Hello World".

Since
1.5.0
def input_file_name(): Column

Creates a string column for the file name of the current Spark task.
Creates a string column for the file name of the current Spark task.

Since
1.6.0
def instr(str: Column, substring: String): Column

Locate the position of the first occurrence of substr column in the given string.
Locate the position of the first occurrence of substr column in the given string. Returns null if either of the arguments are null.

Since
1.5.0
Note
The position is not zero based, but 1 based index. Returns 0 if substr could not be found in str.
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def isnan(e: Column): Column

Return true iff the column is NaN.
Return true iff the column is NaN.

Since
1.6.0
def isnull(e: Column): Column

Return true iff the column is null.
Return true iff the column is null.

Since
1.6.0
def json_tuple(json: Column, fields: String*): Column

Creates a new row for a json column according to the given field names.
Creates a new row for a json column according to the given field names.

Annotations
@varargs()
Since
1.6.0
def kurtosis(columnName: String): Column

Aggregate function: returns the kurtosis of the values in a group.
Aggregate function: returns the kurtosis of the values in a group.

Since
1.6.0
def kurtosis(e: Column): Column

Aggregate function: returns the kurtosis of the values in a group.
Aggregate function: returns the kurtosis of the values in a group.

Since
1.6.0
def lag(e: Column, offset: Int, defaultValue: Any): Column

Window function: returns the value that is offset rows before the current row, and defaultValue if there is less than offset rows before the current row.
Window function: returns the value that is offset rows before the current row, and defaultValue if there is less than offset rows before the current row. For example, an offset of one will return the previous row at any given point in the window partition.
This is equivalent to the LAG function in SQL.

Since
1.4.0
def lag(columnName: String, offset: Int, defaultValue: Any): Column

Window function: returns the value that is offset rows before the current row, and defaultValue if there is less than offset rows before the current row.
Window function: returns the value that is offset rows before the current row, and defaultValue if there is less than offset rows before the current row. For example, an offset of one will return the previous row at any given point in the window partition.
This is equivalent to the LAG function in SQL.

Since
1.4.0
def lag(columnName: String, offset: Int): Column

Window function: returns the value that is offset rows before the current row, and null if there is less than offset rows before the current row.
Window function: returns the value that is offset rows before the current row, and null if there is less than offset rows before the current row. For example, an offset of one will return the previous row at any given point in the window partition.
This is equivalent to the LAG function in SQL.

Since
1.4.0
def lag(e: Column, offset: Int): Column

Window function: returns the value that is offset rows before the current row, and null if there is less than offset rows before the current row.
Window function: returns the value that is offset rows before the current row, and null if there is less than offset rows before the current row. For example, an offset of one will return the previous row at any given point in the window partition.
This is equivalent to the LAG function in SQL.

Since
1.4.0
def last(columnName: String): Column

Aggregate function: returns the last value of the column in a group.
Aggregate function: returns the last value of the column in a group.
The function by default returns the last values it sees. It will return the last non-null value it sees when ignoreNulls is set to true. If all values are null, then null is returned.

Since
1.3.0
def last(e: Column): Column

Aggregate function: returns the last value in a group.
Aggregate function: returns the last value in a group.
The function by default returns the last values it sees. It will return the last non-null value it sees when ignoreNulls is set to true. If all values are null, then null is returned.

Since
1.3.0
def last(columnName: String, ignoreNulls: Boolean): Column

Aggregate function: returns the last value of the column in a group.
Aggregate function: returns the last value of the column in a group.
The function by default returns the last values it sees. It will return the last non-null value it sees when ignoreNulls is set to true. If all values are null, then null is returned.

Since
2.0.0
def last(e: Column, ignoreNulls: Boolean): Column

Aggregate function: returns the last value in a group.
Aggregate function: returns the last value in a group.
The function by default returns the last values it sees. It will return the last non-null value it sees when ignoreNulls is set to true. If all values are null, then null is returned.

Since
2.0.0
def last_day(e: Column): Column

Given a date column, returns the last day of the month which the given date belongs to.
Given a date column, returns the last day of the month which the given date belongs to. For example, input "2015-07-27" returns "2015-07-31" since July 31 is the last day of the month in July 2015.

Since
1.5.0
def lead(e: Column, offset: Int, defaultValue: Any): Column

Window function: returns the value that is offset rows after the current row, and defaultValue if there is less than offset rows after the current row.
Window function: returns the value that is offset rows after the current row, and defaultValue if there is less than offset rows after the current row. For example, an offset of one will return the next row at any given point in the window partition.
This is equivalent to the LEAD function in SQL.

Since
1.4.0
def lead(columnName: String, offset: Int, defaultValue: Any): Column

Window function: returns the value that is offset rows after the current row, and defaultValue if there is less than offset rows after the current row.
Window function: returns the value that is offset rows after the current row, and defaultValue if there is less than offset rows after the current row. For example, an offset of one will return the next row at any given point in the window partition.
This is equivalent to the LEAD function in SQL.

Since
1.4.0
def lead(e: Column, offset: Int): Column

Window function: returns the value that is offset rows after the current row, and null if there is less than offset rows after the current row.
Window function: returns the value that is offset rows after the current row, and null if there is less than offset rows after the current row. For example, an offset of one will return the next row at any given point in the window partition.
This is equivalent to the LEAD function in SQL.

Since
1.4.0
def lead(columnName: String, offset: Int): Column

Window function: returns the value that is offset rows after the current row, and null if there is less than offset rows after the current row.
Window function: returns the value that is offset rows after the current row, and null if there is less than offset rows after the current row. For example, an offset of one will return the next row at any given point in the window partition.
This is equivalent to the LEAD function in SQL.

Since
1.4.0
def least(columnName: String, columnNames: String*): Column

Returns the least value of the list of column names, skipping null values.
Returns the least value of the list of column names, skipping null values. This function takes at least 2 parameters. It will return null iff all parameters are null.

Annotations
@varargs()
Since
1.5.0
def least(exprs: Column*): Column

Returns the least value of the list of values, skipping null values.
Returns the least value of the list of values, skipping null values. This function takes at least 2 parameters. It will return null iff all parameters are null.

Annotations
@varargs()
Since
1.5.0
def length(e: Column): Column

Computes the character length of a given string or number of bytes of a binary string.
Computes the character length of a given string or number of bytes of a binary string. The length of character strings include the trailing spaces. The length of binary strings includes binary zeros.

Since
1.5.0
def levenshtein(l: Column, r: Column): Column

Computes the Levenshtein distance of the two given string columns.
Computes the Levenshtein distance of the two given string columns.

Since
1.5.0
def lit(literal: Any): Column

Creates a Column of literal value.
Creates a Column of literal value.
The passed in object is returned directly if it is already a Column. If the object is a Scala Symbol, it is converted into a Column also. Otherwise, a new Column is created to represent the literal value.

Since
1.3.0
def locate(substr: String, str: Column, pos: Int): Column

Locate the position of the first occurrence of substr in a string column, after position pos.
Locate the position of the first occurrence of substr in a string column, after position pos.

Since
1.5.0
Note
The position is not zero based, but 1 based index. returns 0 if substr could not be found in str.
def locate(substr: String, str: Column): Column

Locate the position of the first occurrence of substr.
Locate the position of the first occurrence of substr.

Since
1.5.0
Note
The position is not zero based, but 1 based index. Returns 0 if substr could not be found in str.
def log(base: Double, columnName: String): Column

Returns the first argument-base logarithm of the second argument.
Returns the first argument-base logarithm of the second argument.

Since
1.4.0
def log(base: Double, a: Column): Column

Returns the first argument-base logarithm of the second argument.
Returns the first argument-base logarithm of the second argument.

Since
1.4.0
def log(columnName: String): Column

Computes the natural logarithm of the given column.
Computes the natural logarithm of the given column.

Since
1.4.0
def log(e: Column): Column

Computes the natural logarithm of the given value.
Computes the natural logarithm of the given value.

Since
1.4.0
def log10(columnName: String): Column

Computes the logarithm of the given value in base 10.
Computes the logarithm of the given value in base 10.

Since
1.4.0
def log10(e: Column): Column

Computes the logarithm of the given value in base 10.
Computes the logarithm of the given value in base 10.

Since
1.4.0
def log1p(columnName: String): Column

Computes the natural logarithm of the given column plus one.
Computes the natural logarithm of the given column plus one.

Since
1.4.0
def log1p(e: Column): Column

Computes the natural logarithm of the given value plus one.
Computes the natural logarithm of the given value plus one.

Since
1.4.0
def log2(columnName: String): Column

Computes the logarithm of the given value in base 2.
Computes the logarithm of the given value in base 2.

Since
1.5.0
def log2(expr: Column): Column

Computes the logarithm of the given column in base 2.
Computes the logarithm of the given column in base 2.

Since
1.5.0
def lower(e: Column): Column

Converts a string column to lower case.
Converts a string column to lower case.

Since
1.3.0
def lpad(str: Column, len: Int, pad: String): Column

Left-pad the string column with pad to a length of len.
Left-pad the string column with pad to a length of len. If the string column is longer than len, the return value is shortened to len characters.

Since
1.5.0
def ltrim(e: Column, trimString: String): Column

Trim the specified character string from left end for the specified string column.
Trim the specified character string from left end for the specified string column.

Since
2.3.0
def ltrim(e: Column): Column

Trim the spaces from left end for the specified string value.
Trim the spaces from left end for the specified string value.

Since
1.5.0
def map(cols: Column*): Column

Creates a new map column.
Creates a new map column. The input columns must be grouped as key-value pairs, e.g. (key1, value1, key2, value2, ...). The key columns must all have the same data type, and can't be null. The value columns must all have the same data type.

Annotations
@varargs()
Since
2.0
def map_keys(e: Column): Column

Returns an unordered array containing the keys of the map.
Returns an unordered array containing the keys of the map.

Since
2.3.0
def map_values(e: Column): Column

Returns an unordered array containing the values of the map.
Returns an unordered array containing the values of the map.

Since
2.3.0
def max(columnName: String): Column

Aggregate function: returns the maximum value of the column in a group.
Aggregate function: returns the maximum value of the column in a group.

Since
1.3.0
def max(e: Column): Column

Aggregate function: returns the maximum value of the expression in a group.
Aggregate function: returns the maximum value of the expression in a group.

Since
1.3.0
def md5(e: Column): Column

Calculates the MD5 digest of a binary column and returns the value as a 32 character hex string.
Calculates the MD5 digest of a binary column and returns the value as a 32 character hex string.

Since
1.5.0
def mean(columnName: String): Column

Aggregate function: returns the average of the values in a group.
Aggregate function: returns the average of the values in a group. Alias for avg.

Since
1.4.0
def mean(e: Column): Column

Aggregate function: returns the average of the values in a group.
Aggregate function: returns the average of the values in a group. Alias for avg.

Since
1.4.0
def min(columnName: String): Column

Aggregate function: returns the minimum value of the column in a group.
Aggregate function: returns the minimum value of the column in a group.

Since
1.3.0
def min(e: Column): Column

Aggregate function: returns the minimum value of the expression in a group.
Aggregate function: returns the minimum value of the expression in a group.

Since
1.3.0
def minute(e: Column): Column

Extracts the minutes as an integer from a given date/timestamp/string.
Extracts the minutes as an integer from a given date/timestamp/string.

Since
1.5.0
def monotonically_increasing_id(): Column

A column expression that generates monotonically increasing 64-bit integers.
A column expression that generates monotonically increasing 64-bit integers.
The generated ID is guaranteed to be monotonically increasing and unique, but not consecutive. The current implementation puts the partition ID in the upper 31 bits, and the record number within each partition in the lower 33 bits. The assumption is that the data frame has less than 1 billion partitions, and each partition has less than 8 billion records.
As an example, consider a DataFrame with two partitions, each with 3 records. This expression would return the following IDs:
```
0, 1, 2, 8589934592 (1L << 33), 8589934593, 8589934594.
```
Since
1.6.0
def month(e: Column): Column

Extracts the month as an integer from a given date/timestamp/string.
Extracts the month as an integer from a given date/timestamp/string.

Since
1.5.0
def months_between(date1: Column, date2: Column): Column

Returns number of months between dates date1 and date2.
Returns number of months between dates date1 and date2.

Since
1.5.0
def nanvl(col1: Column, col2: Column): Column

Returns col1 if it is not NaN, or col2 if col1 is NaN.
Returns col1 if it is not NaN, or col2 if col1 is NaN.
Both inputs should be floating point columns (DoubleType or FloatType).

Since
1.5.0
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef

def negate(e: Column): Column

Unary minus, i.e.

Unary minus, i.e. negate the expression.

// Select the amount column and negates all values.
// Scala:
df.select( -df("amount") )

// Java:
df.select( negate(df.col("amount")) );

Since: 1.3.0

def next_day(date: Column, dayOfWeek: String): Column

Given a date column, returns the first date which is later than the value of the date column that is on the specified day of the week.
Given a date column, returns the first date which is later than the value of the date column that is on the specified day of the week.
For example, next_day('2015-07-27', "Sunday") returns 2015-08-02 because that is the first Sunday after 2015-07-27.
Day of the week parameter is case insensitive, and accepts: "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun".

Since
1.5.0

def not(e: Column): Column

Inversion of boolean expression, i.e.

Inversion of boolean expression, i.e. NOT.

// Scala: select rows that are not active (isActive === false)
df.filter( !df("isActive") )

// Java:
df.filter( not(df.col("isActive")) );

Since: 1.3.0

final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def ntile(n: Int): Column

Window function: returns the ntile group id (from 1 to n inclusive) in an ordered window partition.
Window function: returns the ntile group id (from 1 to n inclusive) in an ordered window partition. For example, if n is 4, the first quarter of the rows will get value 1, the second quarter will get 2, the third quarter will get 3, and the last quarter will get 4.
This is equivalent to the NTILE function in SQL.

Since
1.4.0
def percent_rank(): Column

Window function: returns the relative rank (i.e.
Window function: returns the relative rank (i.e. percentile) of rows within a window partition.
This is computed by:
```
(rank of row in its partition - 1) / (number of rows in the partition - 1)
```
This is equivalent to the PERCENT_RANK function in SQL.
Since
1.6.0
def pmod(dividend: Column, divisor: Column): Column

Returns the positive value of dividend mod divisor.
Returns the positive value of dividend mod divisor.

Since
1.5.0
def posexplode(e: Column): Column

Creates a new row for each element with position in the given array or map column.
Creates a new row for each element with position in the given array or map column.

Since
2.1.0
def posexplode_outer(e: Column): Column

Creates a new row for each element with position in the given array or map column.
Creates a new row for each element with position in the given array or map column. Unlike posexplode, if the array/map is null or empty then the row (null, null) is produced.

Since
2.2.0
def pow(l: Double, rightName: String): Column

Returns the value of the first argument raised to the power of the second argument.
Returns the value of the first argument raised to the power of the second argument.

Since
1.4.0
def pow(l: Double, r: Column): Column

Returns the value of the first argument raised to the power of the second argument.
Returns the value of the first argument raised to the power of the second argument.

Since
1.4.0
def pow(leftName: String, r: Double): Column

Returns the value of the first argument raised to the power of the second argument.
Returns the value of the first argument raised to the power of the second argument.

Since
1.4.0
def pow(l: Column, r: Double): Column

Returns the value of the first argument raised to the power of the second argument.
Returns the value of the first argument raised to the power of the second argument.

Since
1.4.0
def pow(leftName: String, rightName: String): Column

Returns the value of the first argument raised to the power of the second argument.
Returns the value of the first argument raised to the power of the second argument.

Since
1.4.0
def pow(leftName: String, r: Column): Column

Returns the value of the first argument raised to the power of the second argument.
Returns the value of the first argument raised to the power of the second argument.

Since
1.4.0
def pow(l: Column, rightName: String): Column

Returns the value of the first argument raised to the power of the second argument.
Returns the value of the first argument raised to the power of the second argument.

Since
1.4.0
def pow(l: Column, r: Column): Column

Returns the value of the first argument raised to the power of the second argument.
Returns the value of the first argument raised to the power of the second argument.

Since
1.4.0
def quarter(e: Column): Column

Extracts the quarter as an integer from a given date/timestamp/string.
Extracts the quarter as an integer from a given date/timestamp/string.

Since
1.5.0
def radians(columnName: String): Column

Converts an angle measured in degrees to an approximately equivalent angle measured in radians.
Converts an angle measured in degrees to an approximately equivalent angle measured in radians.

Since
2.1.0
def radians(e: Column): Column

Converts an angle measured in degrees to an approximately equivalent angle measured in radians.
Converts an angle measured in degrees to an approximately equivalent angle measured in radians.

Since
2.1.0
def rand(): Column

Generate a random column with independent and identically distributed (i.i.d.) samples from U[0.0, 1.0].
Generate a random column with independent and identically distributed (i.i.d.) samples from U[0.0, 1.0].

Since
1.4.0
def rand(seed: Long): Column

Generate a random column with independent and identically distributed (i.i.d.) samples from U[0.0, 1.0].
Generate a random column with independent and identically distributed (i.i.d.) samples from U[0.0, 1.0].

Since
1.4.0
Note
This is indeterministic when data partitions are not fixed.
def randn(): Column

Generate a column with independent and identically distributed (i.i.d.) samples from the standard normal distribution.
Generate a column with independent and identically distributed (i.i.d.) samples from the standard normal distribution.

Since
1.4.0
def randn(seed: Long): Column

Generate a column with independent and identically distributed (i.i.d.) samples from the standard normal distribution.
Generate a column with independent and identically distributed (i.i.d.) samples from the standard normal distribution.

Since
1.4.0
Note
This is indeterministic when data partitions are not fixed.
def rank(): Column

Window function: returns the rank of rows within a window partition.
Window function: returns the rank of rows within a window partition.
The difference between rank and dense_rank is that dense_rank leaves no gaps in ranking sequence when there are ties. That is, if you were ranking a competition using dense_rank and had three people tie for second place, you would say that all three were in second place and that the next person came in third. Rank would give me sequential numbers, making the person that came in third place (after the ties) would register as coming in fifth.
This is equivalent to the RANK function in SQL.

Since
1.4.0
def regexp_extract(e: Column, exp: String, groupIdx: Int): Column

Extract a specific group matched by a Java regex, from the specified string column.
Extract a specific group matched by a Java regex, from the specified string column. If the regex did not match, or the specified group did not match, an empty string is returned.

Since
1.5.0
def regexp_replace(e: Column, pattern: Column, replacement: Column): Column

Replace all substrings of the specified string value that match regexp with rep.
Replace all substrings of the specified string value that match regexp with rep.

Since
2.1.0
def regexp_replace(e: Column, pattern: String, replacement: String): Column

Replace all substrings of the specified string value that match regexp with rep.
Replace all substrings of the specified string value that match regexp with rep.

Since
1.5.0
def repeat(str: Column, n: Int): Column

Repeats a string column n times, and returns it as a new string column.
Repeats a string column n times, and returns it as a new string column.

Since
1.5.0
def reverse(str: Column): Column

Reverses the string column and returns it as a new string column.
Reverses the string column and returns it as a new string column.

Since
1.5.0
def rint(columnName: String): Column

Returns the double value that is closest in value to the argument and is equal to a mathematical integer.
Returns the double value that is closest in value to the argument and is equal to a mathematical integer.

Since
1.4.0
def rint(e: Column): Column

Returns the double value that is closest in value to the argument and is equal to a mathematical integer.
Returns the double value that is closest in value to the argument and is equal to a mathematical integer.

Since
1.4.0
def round(e: Column, scale: Int): Column

Round the value of e to scale decimal places with HALF_UP round mode if scale is greater than or equal to 0 or at integral part when scale is less than 0.
Round the value of e to scale decimal places with HALF_UP round mode if scale is greater than or equal to 0 or at integral part when scale is less than 0.

Since
1.5.0
def round(e: Column): Column

Returns the value of the column e rounded to 0 decimal places with HALF_UP round mode.
Returns the value of the column e rounded to 0 decimal places with HALF_UP round mode.

Since
1.5.0
def row_number(): Column

Window function: returns a sequential number starting at 1 within a window partition.
Window function: returns a sequential number starting at 1 within a window partition.

Since
1.6.0
def rpad(str: Column, len: Int, pad: String): Column

Right-pad the string column with pad to a length of len.
Right-pad the string column with pad to a length of len. If the string column is longer than len, the return value is shortened to len characters.

Since
1.5.0
def rtrim(e: Column, trimString: String): Column

Trim the specified character string from right end for the specified string column.
Trim the specified character string from right end for the specified string column.

Since
2.3.0
def rtrim(e: Column): Column

Trim the spaces from right end for the specified string value.
Trim the spaces from right end for the specified string value.

Since
1.5.0
def second(e: Column): Column

Extracts the seconds as an integer from a given date/timestamp/string.
Extracts the seconds as an integer from a given date/timestamp/string.

Since
1.5.0
def sha1(e: Column): Column

Calculates the SHA-1 digest of a binary column and returns the value as a 40 character hex string.
Calculates the SHA-1 digest of a binary column and returns the value as a 40 character hex string.

Since
1.5.0
def sha2(e: Column, numBits: Int): Column

Calculates the SHA-2 family of hash functions of a binary column and returns the value as a hex string.
Calculates the SHA-2 family of hash functions of a binary column and returns the value as a hex string.
e
column to compute SHA-2 on.
numBits
one of 224, 256, 384, or 512.

Since
1.5.0
def shiftLeft(e: Column, numBits: Int): Column

Shift the given value numBits left.
Shift the given value numBits left. If the given value is a long value, this function will return a long value else it will return an integer value.

Since
1.5.0
def shiftRight(e: Column, numBits: Int): Column

(Signed) shift the given value numBits right.
(Signed) shift the given value numBits right. If the given value is a long value, it will return a long value else it will return an integer value.

Since
1.5.0
def shiftRightUnsigned(e: Column, numBits: Int): Column

Unsigned shift the given value numBits right.
Unsigned shift the given value numBits right. If the given value is a long value, it will return a long value else it will return an integer value.

Since
1.5.0
def signum(columnName: String): Column

Computes the signum of the given column.
Computes the signum of the given column.

Since
1.4.0
def signum(e: Column): Column

Computes the signum of the given value.
Computes the signum of the given value.

Since
1.4.0
def sin(columnName: String): Column

Computes the sine of the given column.
Computes the sine of the given column.

Since
1.4.0
def sin(e: Column): Column

Computes the sine of the given value.
Computes the sine of the given value. Units in radians.

Since
1.4.0
def sinh(columnName: String): Column

Computes the hyperbolic sine of the given column.
Computes the hyperbolic sine of the given column.

Since
1.4.0
def sinh(e: Column): Column

Computes the hyperbolic sine of the given value.
Computes the hyperbolic sine of the given value.

Since
1.4.0
def size(e: Column): Column

Returns length of array or map.
Returns length of array or map.

Since
1.5.0
def skewness(columnName: String): Column

Aggregate function: returns the skewness of the values in a group.
Aggregate function: returns the skewness of the values in a group.

Since
1.6.0
def skewness(e: Column): Column

Aggregate function: returns the skewness of the values in a group.
Aggregate function: returns the skewness of the values in a group.

Since
1.6.0
def sort_array(e: Column, asc: Boolean): Column

Sorts the input array for the given column in ascending or descending order, according to the natural ordering of the array elements.
Sorts the input array for the given column in ascending or descending order, according to the natural ordering of the array elements.

Since
1.5.0
def sort_array(e: Column): Column

Sorts the input array for the given column in ascending order, according to the natural ordering of the array elements.
Sorts the input array for the given column in ascending order, according to the natural ordering of the array elements.

Since
1.5.0
def soundex(e: Column): Column

Returns the soundex code for the specified expression.
Returns the soundex code for the specified expression.

Since
1.5.0
def spark_partition_id(): Column

Partition ID.
Partition ID.

Since
1.6.0
Note
This is indeterministic because it depends on data partitioning and task scheduling.
def split(str: Column, pattern: String): Column

Splits str around pattern (pattern is a regular expression).
Splits str around pattern (pattern is a regular expression).

Since
1.5.0
Note
Pattern is a string representation of the regular expression.
def sqrt(colName: String): Column

Computes the square root of the specified float value.
Computes the square root of the specified float value.

Since
1.5.0
def sqrt(e: Column): Column

Computes the square root of the specified float value.
Computes the square root of the specified float value.

Since
1.3.0
def stddev(columnName: String): Column

Aggregate function: alias for stddev_samp.
Aggregate function: alias for stddev_samp.

Since
1.6.0
def stddev(e: Column): Column

Aggregate function: alias for stddev_samp.
Aggregate function: alias for stddev_samp.

Since
1.6.0
def stddev_pop(columnName: String): Column

Aggregate function: returns the population standard deviation of the expression in a group.
Aggregate function: returns the population standard deviation of the expression in a group.

Since
1.6.0
def stddev_pop(e: Column): Column

Aggregate function: returns the population standard deviation of the expression in a group.
Aggregate function: returns the population standard deviation of the expression in a group.

Since
1.6.0
def stddev_samp(columnName: String): Column

Aggregate function: returns the sample standard deviation of the expression in a group.
Aggregate function: returns the sample standard deviation of the expression in a group.

Since
1.6.0
def stddev_samp(e: Column): Column

Aggregate function: returns the sample standard deviation of the expression in a group.
Aggregate function: returns the sample standard deviation of the expression in a group.

Since
1.6.0
def struct(colName: String, colNames: String*): Column

Creates a new struct column that composes multiple input columns.
Creates a new struct column that composes multiple input columns.

Annotations
@varargs()
Since
1.4.0
def struct(cols: Column*): Column

Creates a new struct column.
Creates a new struct column. If the input column is a column in a DataFrame, or a derived column expression that is named (i.e. aliased), its name would be retained as the StructField's name, otherwise, the newly generated StructField's name would be auto generated as col with a suffix index + 1, i.e. col1, col2, col3, ...

Annotations
@varargs()
Since
1.4.0
def substring(str: Column, pos: Int, len: Int): Column

Substring starts at pos and is of length len when str is String type or returns the slice of byte array that starts at pos in byte and is of length len when str is Binary type
Substring starts at pos and is of length len when str is String type or returns the slice of byte array that starts at pos in byte and is of length len when str is Binary type

Since
1.5.0
Note
The position is not zero based, but 1 based index.
def substring_index(str: Column, delim: String, count: Int): Column

Returns the substring from string str before count occurrences of the delimiter delim.
Returns the substring from string str before count occurrences of the delimiter delim. If count is positive, everything the left of the final delimiter (counting from left) is returned. If count is negative, every to the right of the final delimiter (counting from the right) is returned. substring_index performs a case-sensitive match when searching for delim.
def sum(columnName: String): Column

Aggregate function: returns the sum of all values in the given column.
Aggregate function: returns the sum of all values in the given column.

Since
1.3.0
def sum(e: Column): Column

Aggregate function: returns the sum of all values in the expression.
Aggregate function: returns the sum of all values in the expression.

Since
1.3.0
def sumDistinct(columnName: String): Column

Aggregate function: returns the sum of distinct values in the expression.
Aggregate function: returns the sum of distinct values in the expression.

Since
1.3.0
def sumDistinct(e: Column): Column

Aggregate function: returns the sum of distinct values in the expression.
Aggregate function: returns the sum of distinct values in the expression.

Since
1.3.0
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def tan(columnName: String): Column

Computes the tangent of the given column.
Computes the tangent of the given column.

Since
1.4.0
def tan(e: Column): Column

Computes the tangent of the given value.
Computes the tangent of the given value. Units in radians.

Since
1.4.0
def tanh(columnName: String): Column

Computes the hyperbolic tangent of the given column.
Computes the hyperbolic tangent of the given column.

Since
1.4.0
def tanh(e: Column): Column

Computes the hyperbolic tangent of the given value.
Computes the hyperbolic tangent of the given value.

Since
1.4.0
def toString(): String

Definition Classes
AnyRef → Any
def to_date(e: Column, fmt: String): Column

Converts the column into a DateType with a specified format (see [http://docs.oracle.com/javase/tutorial/i18n/format/simpleDateFormat.html]) return null if fail.
Converts the column into a DateType with a specified format (see [http://docs.oracle.com/javase/tutorial/i18n/format/simpleDateFormat.html]) return null if fail.

Since
2.2.0
def to_date(e: Column): Column

Converts the column into DateType by casting rules to DateType.
Converts the column into DateType by casting rules to DateType.

Since
1.5.0
def to_json(e: Column): Column

Converts a column containing a StructType, ArrayType of StructTypes, a MapType or ArrayType of MapTypes into a JSON string with the specified schema.
Converts a column containing a StructType, ArrayType of StructTypes, a MapType or ArrayType of MapTypes into a JSON string with the specified schema. Throws an exception, in the case of an unsupported type.
e
a column containing a struct or array of the structs.

Since
2.1.0
def to_json(e: Column, options: Map[String, String]): Column

(Java-specific) Converts a column containing a StructType, ArrayType of StructTypes, a MapType or ArrayType of MapTypes into a JSON string with the specified schema.
(Java-specific) Converts a column containing a StructType, ArrayType of StructTypes, a MapType or ArrayType of MapTypes into a JSON string with the specified schema. Throws an exception, in the case of an unsupported type.
e
a column containing a struct or array of the structs.
options
options to control how the struct column is converted into a json string. accepts the same options and the json data source.

Since
2.1.0
def to_json(e: Column, options: Map[String, String]): Column

(Scala-specific) Converts a column containing a StructType, ArrayType of StructTypes, a MapType or ArrayType of MapTypes into a JSON string with the specified schema.
(Scala-specific) Converts a column containing a StructType, ArrayType of StructTypes, a MapType or ArrayType of MapTypes into a JSON string with the specified schema. Throws an exception, in the case of an unsupported type.
e
a column containing a struct or array of the structs.
options
options to control how the struct column is converted into a json string. accepts the same options and the json data source.

Since
2.1.0
def to_timestamp(s: Column, fmt: String): Column

Convert time string to a Unix timestamp (in seconds) with a specified format (see [http://docs.oracle.com/javase/tutorial/i18n/format/simpleDateFormat.html]) to Unix timestamp (in seconds), return null if fail.
Convert time string to a Unix timestamp (in seconds) with a specified format (see [http://docs.oracle.com/javase/tutorial/i18n/format/simpleDateFormat.html]) to Unix timestamp (in seconds), return null if fail.

Since
2.2.0
def to_timestamp(s: Column): Column

Convert time string to a Unix timestamp (in seconds) by casting rules to TimestampType.
Convert time string to a Unix timestamp (in seconds) by casting rules to TimestampType.

Since
2.2.0
def to_utc_timestamp(ts: Column, tz: String): Column

Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in the given time zone, and renders that time as a timestamp in UTC.
Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in the given time zone, and renders that time as a timestamp in UTC. For example, 'GMT+1' would yield '2017-07-14 01:40:00.0'.

Since
1.5.0
def translate(src: Column, matchingString: String, replaceString: String): Column

Translate any character in the src by a character in replaceString.
Translate any character in the src by a character in replaceString. The characters in replaceString correspond to the characters in matchingString. The translate will happen when any character in the string matches the character in the matchingString.

Since
1.5.0
def trim(e: Column, trimString: String): Column

Trim the specified character from both ends for the specified string column.
Trim the specified character from both ends for the specified string column.

Since
2.3.0
def trim(e: Column): Column

Trim the spaces from both ends for the specified string column.
Trim the spaces from both ends for the specified string column.

Since
1.5.0
def trunc(date: Column, format: String): Column

Returns date truncated to the unit specified by the format.
Returns date truncated to the unit specified by the format.

Since
1.5.0
def typedLit[T](literal: T)(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[T]): Column

Creates a Column of literal value.
Creates a Column of literal value.
The passed in object is returned directly if it is already a Column. If the object is a Scala Symbol, it is converted into a Column also. Otherwise, a new Column is created to represent the literal value. The difference between this function and lit is that this function can handle parameterized scala types e.g.: List, Seq and Map.

Since
2.2.0
def udf(f: AnyRef, dataType: DataType): UserDefinedFunction

Defines a deterministic user-defined function (UDF) using a Scala closure.
Defines a deterministic user-defined function (UDF) using a Scala closure. For this variant, the caller must specify the output data type, and there is no automatic input type coercion. By default the returned UDF is deterministic. To change it to nondeterministic, call the API UserDefinedFunction.asNondeterministic().
f
A closure in Scala
dataType
The output data type of the UDF

Since
2.0.0
def udf(f: UDF10[_, _, _, _, _, _, _, _, _, _, _], returnType: DataType): UserDefinedFunction

Defines a Java UDF10 instance as user-defined function (UDF).
Defines a Java UDF10 instance as user-defined function (UDF). The caller must specify the output data type, and there is no automatic input type coercion. By default the returned UDF is deterministic. To change it to nondeterministic, call the API UserDefinedFunction.asNondeterministic().

Since
2.3.0
def udf(f: UDF9[_, _, _, _, _, _, _, _, _, _], returnType: DataType): UserDefinedFunction

Defines a Java UDF9 instance as user-defined function (UDF).
Defines a Java UDF9 instance as user-defined function (UDF). The caller must specify the output data type, and there is no automatic input type coercion. By default the returned UDF is deterministic. To change it to nondeterministic, call the API UserDefinedFunction.asNondeterministic().

Since
2.3.0
def udf(f: UDF8[_, _, _, _, _, _, _, _, _], returnType: DataType): UserDefinedFunction

Defines a Java UDF8 instance as user-defined function (UDF).
Defines a Java UDF8 instance as user-defined function (UDF). The caller must specify the output data type, and there is no automatic input type coercion. By default the returned UDF is deterministic. To change it to nondeterministic, call the API UserDefinedFunction.asNondeterministic().

Since
2.3.0
def udf(f: UDF7[_, _, _, _, _, _, _, _], returnType: DataType): UserDefinedFunction

Defines a Java UDF7 instance as user-defined function (UDF).
Defines a Java UDF7 instance as user-defined function (UDF). The caller must specify the output data type, and there is no automatic input type coercion. By default the returned UDF is deterministic. To change it to nondeterministic, call the API UserDefinedFunction.asNondeterministic().

Since
2.3.0
def udf(f: UDF6[_, _, _, _, _, _, _], returnType: DataType): UserDefinedFunction

Defines a Java UDF6 instance as user-defined function (UDF).
Defines a Java UDF6 instance as user-defined function (UDF). The caller must specify the output data type, and there is no automatic input type coercion. By default the returned UDF is deterministic. To change it to nondeterministic, call the API UserDefinedFunction.asNondeterministic().

Since
2.3.0
def udf(f: UDF5[_, _, _, _, _, _], returnType: DataType): UserDefinedFunction

Defines a Java UDF5 instance as user-defined function (UDF).
Defines a Java UDF5 instance as user-defined function (UDF). The caller must specify the output data type, and there is no automatic input type coercion. By default the returned UDF is deterministic. To change it to nondeterministic, call the API UserDefinedFunction.asNondeterministic().

Since
2.3.0
def udf(f: UDF4[_, _, _, _, _], returnType: DataType): UserDefinedFunction

Defines a Java UDF4 instance as user-defined function (UDF).
Defines a Java UDF4 instance as user-defined function (UDF). The caller must specify the output data type, and there is no automatic input type coercion. By default the returned UDF is deterministic. To change it to nondeterministic, call the API UserDefinedFunction.asNondeterministic().

Since
2.3.0
def udf(f: UDF3[_, _, _, _], returnType: DataType): UserDefinedFunction

Defines a Java UDF3 instance as user-defined function (UDF).
Defines a Java UDF3 instance as user-defined function (UDF). The caller must specify the output data type, and there is no automatic input type coercion. By default the returned UDF is deterministic. To change it to nondeterministic, call the API UserDefinedFunction.asNondeterministic().

Since
2.3.0
def udf(f: UDF2[_, _, _], returnType: DataType): UserDefinedFunction

Defines a Java UDF2 instance as user-defined function (UDF).
Defines a Java UDF2 instance as user-defined function (UDF). The caller must specify the output data type, and there is no automatic input type coercion. By default the returned UDF is deterministic. To change it to nondeterministic, call the API UserDefinedFunction.asNondeterministic().

Since
2.3.0
def udf(f: UDF1[_, _], returnType: DataType): UserDefinedFunction

Defines a Java UDF1 instance as user-defined function (UDF).
Defines a Java UDF1 instance as user-defined function (UDF). The caller must specify the output data type, and there is no automatic input type coercion. By default the returned UDF is deterministic. To change it to nondeterministic, call the API UserDefinedFunction.asNondeterministic().

Since
2.3.0
def udf(f: UDF0[_], returnType: DataType): UserDefinedFunction

Defines a Java UDF0 instance as user-defined function (UDF).
Defines a Java UDF0 instance as user-defined function (UDF). The caller must specify the output data type, and there is no automatic input type coercion. By default the returned UDF is deterministic. To change it to nondeterministic, call the API UserDefinedFunction.asNondeterministic().

Since
2.3.0
def udf[RT, A1, A2, A3, A4, A5, A6, A7, A8, A9, A10](f: (A1, A2, A3, A4, A5, A6, A7, A8, A9, A10) ⇒ RT)(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[RT], arg1: scala.reflect.api.JavaUniverse.TypeTag[A1], arg2: scala.reflect.api.JavaUniverse.TypeTag[A2], arg3: scala.reflect.api.JavaUniverse.TypeTag[A3], arg4: scala.reflect.api.JavaUniverse.TypeTag[A4], arg5: scala.reflect.api.JavaUniverse.TypeTag[A5], arg6: scala.reflect.api.JavaUniverse.TypeTag[A6], arg7: scala.reflect.api.JavaUniverse.TypeTag[A7], arg8: scala.reflect.api.JavaUniverse.TypeTag[A8], arg9: scala.reflect.api.JavaUniverse.TypeTag[A9], arg10: scala.reflect.api.JavaUniverse.TypeTag[A10]): UserDefinedFunction

Defines a Scala closure of 10 arguments as user-defined function (UDF).
Defines a Scala closure of 10 arguments as user-defined function (UDF). The data types are automatically inferred based on the Scala closure's signature. By default the returned UDF is deterministic. To change it to nondeterministic, call the API UserDefinedFunction.asNondeterministic().

Since
1.3.0
def udf[RT, A1, A2, A3, A4, A5, A6, A7, A8, A9](f: (A1, A2, A3, A4, A5, A6, A7, A8, A9) ⇒ RT)(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[RT], arg1: scala.reflect.api.JavaUniverse.TypeTag[A1], arg2: scala.reflect.api.JavaUniverse.TypeTag[A2], arg3: scala.reflect.api.JavaUniverse.TypeTag[A3], arg4: scala.reflect.api.JavaUniverse.TypeTag[A4], arg5: scala.reflect.api.JavaUniverse.TypeTag[A5], arg6: scala.reflect.api.JavaUniverse.TypeTag[A6], arg7: scala.reflect.api.JavaUniverse.TypeTag[A7], arg8: scala.reflect.api.JavaUniverse.TypeTag[A8], arg9: scala.reflect.api.JavaUniverse.TypeTag[A9]): UserDefinedFunction

Defines a Scala closure of 9 arguments as user-defined function (UDF).
Defines a Scala closure of 9 arguments as user-defined function (UDF). The data types are automatically inferred based on the Scala closure's signature. By default the returned UDF is deterministic. To change it to nondeterministic, call the API UserDefinedFunction.asNondeterministic().

Since
1.3.0
def udf[RT, A1, A2, A3, A4, A5, A6, A7, A8](f: (A1, A2, A3, A4, A5, A6, A7, A8) ⇒ RT)(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[RT], arg1: scala.reflect.api.JavaUniverse.TypeTag[A1], arg2: scala.reflect.api.JavaUniverse.TypeTag[A2], arg3: scala.reflect.api.JavaUniverse.TypeTag[A3], arg4: scala.reflect.api.JavaUniverse.TypeTag[A4], arg5: scala.reflect.api.JavaUniverse.TypeTag[A5], arg6: scala.reflect.api.JavaUniverse.TypeTag[A6], arg7: scala.reflect.api.JavaUniverse.TypeTag[A7], arg8: scala.reflect.api.JavaUniverse.TypeTag[A8]): UserDefinedFunction

Defines a Scala closure of 8 arguments as user-defined function (UDF).
Defines a Scala closure of 8 arguments as user-defined function (UDF). The data types are automatically inferred based on the Scala closure's signature. By default the returned UDF is deterministic. To change it to nondeterministic, call the API UserDefinedFunction.asNondeterministic().

Since
1.3.0
def udf[RT, A1, A2, A3, A4, A5, A6, A7](f: (A1, A2, A3, A4, A5, A6, A7) ⇒ RT)(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[RT], arg1: scala.reflect.api.JavaUniverse.TypeTag[A1], arg2: scala.reflect.api.JavaUniverse.TypeTag[A2], arg3: scala.reflect.api.JavaUniverse.TypeTag[A3], arg4: scala.reflect.api.JavaUniverse.TypeTag[A4], arg5: scala.reflect.api.JavaUniverse.TypeTag[A5], arg6: scala.reflect.api.JavaUniverse.TypeTag[A6], arg7: scala.reflect.api.JavaUniverse.TypeTag[A7]): UserDefinedFunction

Defines a Scala closure of 7 arguments as user-defined function (UDF).
Defines a Scala closure of 7 arguments as user-defined function (UDF). The data types are automatically inferred based on the Scala closure's signature. By default the returned UDF is deterministic. To change it to nondeterministic, call the API UserDefinedFunction.asNondeterministic().

Since
1.3.0
def udf[RT, A1, A2, A3, A4, A5, A6](f: (A1, A2, A3, A4, A5, A6) ⇒ RT)(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[RT], arg1: scala.reflect.api.JavaUniverse.TypeTag[A1], arg2: scala.reflect.api.JavaUniverse.TypeTag[A2], arg3: scala.reflect.api.JavaUniverse.TypeTag[A3], arg4: scala.reflect.api.JavaUniverse.TypeTag[A4], arg5: scala.reflect.api.JavaUniverse.TypeTag[A5], arg6: scala.reflect.api.JavaUniverse.TypeTag[A6]): UserDefinedFunction

Defines a Scala closure of 6 arguments as user-defined function (UDF).
Defines a Scala closure of 6 arguments as user-defined function (UDF). The data types are automatically inferred based on the Scala closure's signature. By default the returned UDF is deterministic. To change it to nondeterministic, call the API UserDefinedFunction.asNondeterministic().

Since
1.3.0
def udf[RT, A1, A2, A3, A4, A5](f: (A1, A2, A3, A4, A5) ⇒ RT)(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[RT], arg1: scala.reflect.api.JavaUniverse.TypeTag[A1], arg2: scala.reflect.api.JavaUniverse.TypeTag[A2], arg3: scala.reflect.api.JavaUniverse.TypeTag[A3], arg4: scala.reflect.api.JavaUniverse.TypeTag[A4], arg5: scala.reflect.api.JavaUniverse.TypeTag[A5]): UserDefinedFunction

Defines a Scala closure of 5 arguments as user-defined function (UDF).
Defines a Scala closure of 5 arguments as user-defined function (UDF). The data types are automatically inferred based on the Scala closure's signature. By default the returned UDF is deterministic. To change it to nondeterministic, call the API UserDefinedFunction.asNondeterministic().

Since
1.3.0
def udf[RT, A1, A2, A3, A4](f: (A1, A2, A3, A4) ⇒ RT)(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[RT], arg1: scala.reflect.api.JavaUniverse.TypeTag[A1], arg2: scala.reflect.api.JavaUniverse.TypeTag[A2], arg3: scala.reflect.api.JavaUniverse.TypeTag[A3], arg4: scala.reflect.api.JavaUniverse.TypeTag[A4]): UserDefinedFunction

Defines a Scala closure of 4 arguments as user-defined function (UDF).
Defines a Scala closure of 4 arguments as user-defined function (UDF). The data types are automatically inferred based on the Scala closure's signature. By default the returned UDF is deterministic. To change it to nondeterministic, call the API UserDefinedFunction.asNondeterministic().

Since
1.3.0
def udf[RT, A1, A2, A3](f: (A1, A2, A3) ⇒ RT)(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[RT], arg1: scala.reflect.api.JavaUniverse.TypeTag[A1], arg2: scala.reflect.api.JavaUniverse.TypeTag[A2], arg3: scala.reflect.api.JavaUniverse.TypeTag[A3]): UserDefinedFunction

Defines a Scala closure of 3 arguments as user-defined function (UDF).
Defines a Scala closure of 3 arguments as user-defined function (UDF). The data types are automatically inferred based on the Scala closure's signature. By default the returned UDF is deterministic. To change it to nondeterministic, call the API UserDefinedFunction.asNondeterministic().

Since
1.3.0
def udf[RT, A1, A2](f: (A1, A2) ⇒ RT)(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[RT], arg1: scala.reflect.api.JavaUniverse.TypeTag[A1], arg2: scala.reflect.api.JavaUniverse.TypeTag[A2]): UserDefinedFunction

Defines a Scala closure of 2 arguments as user-defined function (UDF).
Defines a Scala closure of 2 arguments as user-defined function (UDF). The data types are automatically inferred based on the Scala closure's signature. By default the returned UDF is deterministic. To change it to nondeterministic, call the API UserDefinedFunction.asNondeterministic().

Since
1.3.0
def udf[RT, A1](f: (A1) ⇒ RT)(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[RT], arg1: scala.reflect.api.JavaUniverse.TypeTag[A1]): UserDefinedFunction

Defines a Scala closure of 1 arguments as user-defined function (UDF).
Defines a Scala closure of 1 arguments as user-defined function (UDF). The data types are automatically inferred based on the Scala closure's signature. By default the returned UDF is deterministic. To change it to nondeterministic, call the API UserDefinedFunction.asNondeterministic().

Since
1.3.0
def udf[RT](f: () ⇒ RT)(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[RT]): UserDefinedFunction

Defines a Scala closure of 0 arguments as user-defined function (UDF).
Defines a Scala closure of 0 arguments as user-defined function (UDF). The data types are automatically inferred based on the Scala closure's signature. By default the returned UDF is deterministic. To change it to nondeterministic, call the API UserDefinedFunction.asNondeterministic().

Since
1.3.0
def unbase64(e: Column): Column

Decodes a BASE64 encoded string column and returns it as a binary column.
Decodes a BASE64 encoded string column and returns it as a binary column. This is the reverse of base64.

Since
1.5.0
def unboundedFollowing(): Column

Window function: returns the special frame boundary that represents the last row in the window partition.
Window function: returns the special frame boundary that represents the last row in the window partition.

Since
2.3.0
def unboundedPreceding(): Column

Window function: returns the special frame boundary that represents the first row in the window partition.
Window function: returns the special frame boundary that represents the first row in the window partition.

Since
2.3.0
def unhex(column: Column): Column

Inverse of hex.
Inverse of hex. Interprets each pair of characters as a hexadecimal number and converts to the byte representation of number.

Since
1.5.0
def unix_timestamp(s: Column, p: String): Column

Converts time string with given pattern to Unix timestamp (in seconds).
Converts time string with given pattern to Unix timestamp (in seconds). Returns null if fails.

Since
1.5.0
See also
Customizing Formats
def unix_timestamp(s: Column): Column

Converts time string in format yyyy-MM-dd HH:mm:ss to Unix timestamp (in seconds), using the default timezone and the default locale.
Converts time string in format yyyy-MM-dd HH:mm:ss to Unix timestamp (in seconds), using the default timezone and the default locale. Returns null if fails.

Since
1.5.0
def unix_timestamp(): Column

Returns the current Unix timestamp (in seconds).
Returns the current Unix timestamp (in seconds).

Since
1.5.0
Note
All calls of unix_timestamp within the same query return the same value (i.e. the current timestamp is calculated at the start of query evaluation).
def upper(e: Column): Column

Converts a string column to upper case.
Converts a string column to upper case.

Since
1.3.0
def var_pop(columnName: String): Column

Aggregate function: returns the population variance of the values in a group.
Aggregate function: returns the population variance of the values in a group.

Since
1.6.0
def var_pop(e: Column): Column

Aggregate function: returns the population variance of the values in a group.
Aggregate function: returns the population variance of the values in a group.

Since
1.6.0
def var_samp(columnName: String): Column

Aggregate function: returns the unbiased variance of the values in a group.
Aggregate function: returns the unbiased variance of the values in a group.

Since
1.6.0
def var_samp(e: Column): Column

Aggregate function: returns the unbiased variance of the values in a group.
Aggregate function: returns the unbiased variance of the values in a group.

Since
1.6.0
def variance(columnName: String): Column

Aggregate function: alias for var_samp.
Aggregate function: alias for var_samp.

Since
1.6.0
def variance(e: Column): Column

Aggregate function: alias for var_samp.
Aggregate function: alias for var_samp.

Since
1.6.0
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
def weekofyear(e: Column): Column

Extracts the week number as an integer from a given date/timestamp/string.
Extracts the week number as an integer from a given date/timestamp/string.

Since
1.5.0

def when(condition: Column, value: Any): Column

Evaluates a list of conditions and returns one of multiple possible result expressions.

Evaluates a list of conditions and returns one of multiple possible result expressions. If otherwise is not defined at the end, null is returned for unmatched conditions.

// Example: encoding gender string column into integer.

// Scala:
people.select(when(people("gender") === "male", 0)
  .when(people("gender") === "female", 1)
  .otherwise(2))

// Java:
people.select(when(col("gender").equalTo("male"), 0)
  .when(col("gender").equalTo("female"), 1)
  .otherwise(2))

Since: 1.4.0

def window(timeColumn: Column, windowDuration: String): Column

Generates tumbling time windows given a timestamp specifying column.
Generates tumbling time windows given a timestamp specifying column. Window starts are inclusive but the window ends are exclusive, e.g. 12:05 will be in the window [12:05,12:10) but not in [12:00,12:05). Windows can support microsecond precision. Windows in the order of months are not supported. The windows start beginning at 1970-01-01 00:00:00 UTC. The following example takes the average stock price for a one minute tumbling window:
```
val df = ... // schema => timestamp: TimestampType, stockId: StringType, price: DoubleType
df.groupBy(window($"time", "1 minute"), $"stockId")
  .agg(mean("price"))
```
The windows will look like:
```
09:00:00-09:01:00
09:01:00-09:02:00
09:02:00-09:03:00 ...
```
For a streaming query, you may use the function current_timestamp to generate windows on processing time.
timeColumn
The column or the expression to use as the timestamp for windowing by time. The time column must be of TimestampType.
windowDuration
A string specifying the width of the window, e.g. 10 minutes, 1 second. Check org.apache.spark.unsafe.types.CalendarInterval for valid duration identifiers.

Since
2.0.0
def window(timeColumn: Column, windowDuration: String, slideDuration: String): Column

Bucketize rows into one or more time windows given a timestamp specifying column.
Bucketize rows into one or more time windows given a timestamp specifying column. Window starts are inclusive but the window ends are exclusive, e.g. 12:05 will be in the window [12:05,12:10) but not in [12:00,12:05). Windows can support microsecond precision. Windows in the order of months are not supported. The windows start beginning at 1970-01-01 00:00:00 UTC. The following example takes the average stock price for a one minute window every 10 seconds:
```
val df = ... // schema => timestamp: TimestampType, stockId: StringType, price: DoubleType
df.groupBy(window($"time", "1 minute", "10 seconds"), $"stockId")
  .agg(mean("price"))
```
The windows will look like:
```
09:00:00-09:01:00
09:00:10-09:01:10
09:00:20-09:01:20 ...
```
For a streaming query, you may use the function current_timestamp to generate windows on processing time.
timeColumn
The column or the expression to use as the timestamp for windowing by time. The time column must be of TimestampType.
windowDuration
A string specifying the width of the window, e.g. 10 minutes, 1 second. Check org.apache.spark.unsafe.types.CalendarInterval for valid duration identifiers. Note that the duration is a fixed length of time, and does not vary over time according to a calendar. For example, 1 day always means 86,400,000 milliseconds, not a calendar day.
slideDuration
A string specifying the sliding interval of the window, e.g. 1 minute. A new window will be generated every slideDuration. Must be less than or equal to the windowDuration. Check org.apache.spark.unsafe.types.CalendarInterval for valid duration identifiers. This duration is likewise absolute, and does not vary according to a calendar.

Since
2.0.0
def window(timeColumn: Column, windowDuration: String, slideDuration: String, startTime: String): Column

Bucketize rows into one or more time windows given a timestamp specifying column.
Bucketize rows into one or more time windows given a timestamp specifying column. Window starts are inclusive but the window ends are exclusive, e.g. 12:05 will be in the window [12:05,12:10) but not in [12:00,12:05). Windows can support microsecond precision. Windows in the order of months are not supported. The following example takes the average stock price for a one minute window every 10 seconds starting 5 seconds after the hour:
```
val df = ... // schema => timestamp: TimestampType, stockId: StringType, price: DoubleType
df.groupBy(window($"time", "1 minute", "10 seconds", "5 seconds"), $"stockId")
  .agg(mean("price"))
```
The windows will look like:
```
09:00:05-09:01:05
09:00:15-09:01:15
09:00:25-09:01:25 ...
```
For a streaming query, you may use the function current_timestamp to generate windows on processing time.
timeColumn
The column or the expression to use as the timestamp for windowing by time. The time column must be of TimestampType.
windowDuration
A string specifying the width of the window, e.g. 10 minutes, 1 second. Check org.apache.spark.unsafe.types.CalendarInterval for valid duration identifiers. Note that the duration is a fixed length of time, and does not vary over time according to a calendar. For example, 1 day always means 86,400,000 milliseconds, not a calendar day.
slideDuration
A string specifying the sliding interval of the window, e.g. 1 minute. A new window will be generated every slideDuration. Must be less than or equal to the windowDuration. Check org.apache.spark.unsafe.types.CalendarInterval for valid duration identifiers. This duration is likewise absolute, and does not vary according to a calendar.
startTime
The offset with respect to 1970-01-01 00:00:00 UTC with which to start window intervals. For example, in order to have hourly tumbling windows that start 15 minutes past the hour, e.g. 12:15-13:15, 13:15-14:15... provide startTime as 15 minutes.

Since
2.0.0
def year(e: Column): Column

Extracts the year as an integer from a given date/timestamp/string.
Extracts the year as an integer from a given date/timestamp/string.

Since
1.5.0

Deprecated Value Members

def approxCountDistinct(columnName: String, rsd: Double): Column

Annotations
@deprecated
Deprecated
(Since version 2.1.0) Use approx_count_distinct
Since
1.3.0
def approxCountDistinct(e: Column, rsd: Double): Column

Annotations
@deprecated
Deprecated
(Since version 2.1.0) Use approx_count_distinct
Since
1.3.0
def approxCountDistinct(columnName: String): Column

Annotations
@deprecated
Deprecated
(Since version 2.1.0) Use approx_count_distinct
Since
1.3.0
def approxCountDistinct(e: Column): Column

Annotations
@deprecated
Deprecated
(Since version 2.1.0) Use approx_count_distinct
Since
1.3.0
def monotonicallyIncreasingId(): Column

A column expression that generates monotonically increasing 64-bit integers.
A column expression that generates monotonically increasing 64-bit integers.
The generated ID is guaranteed to be monotonically increasing and unique, but not consecutive. The current implementation puts the partition ID in the upper 31 bits, and the record number within each partition in the lower 33 bits. The assumption is that the data frame has less than 1 billion partitions, and each partition has less than 8 billion records.
As an example, consider a DataFrame with two partitions, each with 3 records. This expression would return the following IDs:
```
0, 1, 2, 8589934592 (1L << 33), 8589934593, 8589934594.
```
Annotations
@deprecated
Deprecated
(Since version 2.0.0) Use monotonically_increasing_id()
Since
1.4.0
def toDegrees(columnName: String): Column

Annotations
@deprecated
Deprecated
(Since version 2.1.0) Use degrees
Since
1.4.0
def toDegrees(e: Column): Column

Annotations
@deprecated
Deprecated
(Since version 2.1.0) Use degrees
Since
1.4.0
def toRadians(columnName: String): Column

Annotations
@deprecated
Deprecated
(Since version 2.1.0) Use radians
Since
1.4.0
def toRadians(e: Column): Column

Annotations
@deprecated
Deprecated
(Since version 2.1.0) Use radians
Since
1.4.0

Related Doc: package sql

object functions

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

def abs(e: Column): Column

def acos(columnName: String): Column

def acos(e: Column): Column

def add_months(startDate: Column, numMonths: Int): Column

def approx_count_distinct(columnName: String, rsd: Double): Column

def approx_count_distinct(e: Column, rsd: Double): Column

def approx_count_distinct(columnName: String): Column

def approx_count_distinct(e: Column): Column

def array(colName: String, colNames: String*): Column

def array(cols: Column*): Column

def array_contains(column: Column, value: Any): Column

final def asInstanceOf[T0]: T0

def asc(columnName: String): Column

def asc_nulls_first(columnName: String): Column

def asc_nulls_last(columnName: String): Column

def ascii(e: Column): Column

def asin(columnName: String): Column

def asin(e: Column): Column

def atan(columnName: String): Column

def atan(e: Column): Column

def atan2(l: Double, rightName: String): Column

def atan2(l: Double, r: Column): Column

def atan2(leftName: String, r: Double): Column

def atan2(l: Column, r: Double): Column

def atan2(leftName: String, rightName: String): Column

def atan2(leftName: String, r: Column): Column

def atan2(l: Column, rightName: String): Column

def atan2(l: Column, r: Column): Column

def avg(columnName: String): Column

def avg(e: Column): Column

def base64(e: Column): Column

def bin(columnName: String): Column

def bin(e: Column): Column

def bitwiseNOT(e: Column): Column

def broadcast[T](df: Dataset[T]): Dataset[T]

def bround(e: Column, scale: Int): Column

def bround(e: Column): Column

def callUDF(udfName: String, cols: Column*): Column

def cbrt(columnName: String): Column

def cbrt(e: Column): Column

def ceil(columnName: String): Column

def ceil(e: Column): Column

def clone(): AnyRef

def coalesce(e: Column*): Column

def col(colName: String): Column

def collect_list(columnName: String): Column

def collect_list(e: Column): Column

def collect_set(columnName: String): Column

def collect_set(e: Column): Column

def column(colName: String): Column

def concat(exprs: Column*): Column

def concat_ws(sep: String, exprs: Column*): Column

def conv(num: Column, fromBase: Int, toBase: Int): Column

def corr(columnName1: String, columnName2: String): Column

def corr(column1: Column, column2: Column): Column

def cos(columnName: String): Column

def cos(e: Column): Column

def cosh(columnName: String): Column

def cosh(e: Column): Column

def count(columnName: String): TypedColumn[Any, Long]

def count(e: Column): Column

def countDistinct(columnName: String, columnNames: String*): Column

def countDistinct(expr: Column, exprs: Column*): Column

def covar_pop(columnName1: String, columnName2: String): Column

def covar_pop(column1: Column, column2: Column): Column

def covar_samp(columnName1: String, columnName2: String): Column

def covar_samp(column1: Column, column2: Column): Column

def crc32(e: Column): Column

def cume_dist(): Column

def currentRow(): Column

def current_date(): Column

def current_timestamp(): Column

def date_add(start: Column, days: Int): Column