StateSpec

:: Experimental :: Builder object for creating instances of org.apache.spark.streaming.StateSpec that is used for specifying the parameters of the DStream transformation mapWithState that is used for specifying the parameters of the DStream transformation mapWithState operation of a pair DStream (Scala) or a JavaPairDStream (Java).

Example in Scala:

// A mapping function that maintains an integer state and return a String
def mappingFunction(key: String, value: Option[Int], state: State[Int]): Option[String] = {
  // Use state.exists(), state.get(), state.update() and state.remove()
  // to manage state, and return the necessary string
}

val spec = StateSpec.function(mappingFunction).numPartitions(10)

val mapWithStateDStream = keyValueDStream.mapWithState[StateType, MappedType](spec)

Example in Java:

// A mapping function that maintains an integer state and return a string
Function3<String, Optional<Integer>, State<Integer>, String> mappingFunction =
    new Function3<String, Optional<Integer>, State<Integer>, String>() {
        @Override
        public Optional<String> call(Optional<Integer> value, State<Integer> state) {
            // Use state.exists(), state.get(), state.update() and state.remove()
            // to manage state, and return the necessary string
        }
    };

 JavaMapWithStateDStream<String, Integer, Integer, String> mapWithStateDStream =
     keyValueDStream.mapWithState(StateSpec.function(mappingFunc));

Annotations: @Experimental()
Source: StateSpec.scala

Linear Supertypes

Serializable, Serializable, AnyRef, Any

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
def function[KeyType, ValueType, StateType, MappedType](mappingFunction: Function3[KeyType, Optional[ValueType], State[StateType], MappedType]): StateSpec[KeyType, ValueType, StateType, MappedType]

Create a StateSpec for setting all the specifications of the mapWithState operation on a JavaPairDStream.
Create a StateSpec for setting all the specifications of the mapWithState operation on a JavaPairDStream.
ValueType
Class of the values
StateType
Class of the states data
MappedType
Class of the mapped data
mappingFunction
The function applied on every data item to manage the associated state and generate the mapped data
def function[KeyType, ValueType, StateType, MappedType](mappingFunction: Function4[Time, KeyType, Optional[ValueType], State[StateType], Optional[MappedType]]): StateSpec[KeyType, ValueType, StateType, MappedType]

Create a StateSpec for setting all the specifications of the mapWithState operation on a JavaPairDStream.
Create a StateSpec for setting all the specifications of the mapWithState operation on a JavaPairDStream.
KeyType
Class of the keys
ValueType
Class of the values
StateType
Class of the states data
MappedType
Class of the mapped data
mappingFunction
The function applied on every data item to manage the associated state and generate the mapped data
def function[KeyType, ValueType, StateType, MappedType](mappingFunction: (KeyType, Option[ValueType], State[StateType]) ⇒ MappedType): StateSpec[KeyType, ValueType, StateType, MappedType]

Create a StateSpec for setting all the specifications of the mapWithState operation on a pair DStream.
Create a StateSpec for setting all the specifications of the mapWithState operation on a pair DStream.
ValueType
Class of the values
StateType
Class of the states data
MappedType
Class of the mapped data
mappingFunction
The function applied on every data item to manage the associated state and generate the mapped data
def function[KeyType, ValueType, StateType, MappedType](mappingFunction: (Time, KeyType, Option[ValueType], State[StateType]) ⇒ Option[MappedType]): StateSpec[KeyType, ValueType, StateType, MappedType]

Create a StateSpec for setting all the specifications of the mapWithState operation on a pair DStream.
Create a StateSpec for setting all the specifications of the mapWithState operation on a pair DStream.
KeyType
Class of the keys
ValueType
Class of the values
StateType
Class of the states data
MappedType
Class of the mapped data
mappingFunction
The function applied on every data item to manage the associated state and generate the mapped data
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def hashCode(): Int

Definition Classes
AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Docs: class StateSpec | package streaming

object StateSpec extends Serializable

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

def clone(): AnyRef

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

def function[KeyType, ValueType, StateType, MappedType](mappingFunction: Function3[KeyType, Optional[ValueType], State[StateType], MappedType]): StateSpec[KeyType, ValueType, StateType, MappedType]

def function[KeyType, ValueType, StateType, MappedType](mappingFunction: Function4[Time, KeyType, Optional[ValueType], State[StateType], Optional[MappedType]]): StateSpec[KeyType, ValueType, StateType, MappedType]

def function[KeyType, ValueType, StateType, MappedType](mappingFunction: (KeyType, Option[ValueType], State[StateType]) ⇒ MappedType): StateSpec[KeyType, ValueType, StateType, MappedType]

def function[KeyType, ValueType, StateType, MappedType](mappingFunction: (Time, KeyType, Option[ValueType], State[StateType]) ⇒ Option[MappedType]): StateSpec[KeyType, ValueType, StateType, MappedType]

final def getClass(): Class[_]

def hashCode(): Int

final def isInstanceOf[T0]: Boolean

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from Serializable

Inherited from Serializable

Inherited from AnyRef

Inherited from Any

Ungrouped