org.apache.hadoop.mapred.SparkHadoopWriter

Internal helper class that saves an RDD using a Hadoop OutputFormat. This is only public because we need to access this class from the spark package to use some package-private Hadoop functions, but this class should not be used directly by users.

Saves the RDD using a JobConf, which should contain an output key class, an output value class, a filename to write to, etc, exactly like in a Hadoop MapReduce job.

Linear Supertypes

Serializable, Serializable, SparkHadoopMapRedUtil, Logging, AnyRef, Any

Instance Constructors

new SparkHadoopWriter(jobConf: JobConf)

Value Members

final def !=(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def !=(arg0: Any): Boolean

Definition Classes
Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def ==(arg0: Any): Boolean

Definition Classes
Any
final def asInstanceOf[T0]: T0

Definition Classes
Any
def cleanup(): Unit
def clone(): AnyRef

Attributes
protected[lang]
Definition Classes
AnyRef
Annotations
@throws()
def close(): Unit
def commit(): Unit
def commitJob(): Unit
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[lang]
Definition Classes
AnyRef
Annotations
@throws()
final def getClass(): java.lang.Class[_]

Definition Classes
AnyRef → Any
def hashCode(): Int

Definition Classes
AnyRef → Any
def initLogging(): Unit

Attributes
protected
Definition Classes
Logging
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def isTraceEnabled(): Boolean

Attributes
protected
Definition Classes
Logging
def log: Logger

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def newJobContext(conf: JobConf, jobId: JobID): JobContext

Definition Classes
SparkHadoopMapRedUtil
def newTaskAttemptContext(conf: JobConf, attemptId: TaskAttemptID): TaskAttemptContext

Definition Classes
SparkHadoopMapRedUtil
def newTaskAttemptID(jtIdentifier: String, jobId: Int, isMap: Boolean, taskId: Int, attemptId: Int): TaskAttemptID

Definition Classes
SparkHadoopMapRedUtil
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def open(): Unit
def preSetup(): Unit
def setup(jobid: Int, splitid: Int, attemptid: Int): Unit
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws()
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws()
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws()
def write(key: AnyRef, value: AnyRef): Unit

SparkHadoopWriter

class SparkHadoopWriter extends Logging with SparkHadoopMapRedUtil with Serializable

Instance Constructors

new SparkHadoopWriter(jobConf: JobConf)

Value Members

final def !=(arg0: AnyRef): Boolean

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: AnyRef): Boolean

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

def cleanup(): Unit

def clone(): AnyRef

def close(): Unit

def commit(): Unit

def commitJob(): Unit

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

final def getClass(): java.lang.Class[_]

def hashCode(): Int

def initLogging(): Unit

final def isInstanceOf[T0]: Boolean

def isTraceEnabled(): Boolean

def log: Logger

def logDebug(msg: ⇒ String, throwable: Throwable): Unit

def logDebug(msg: ⇒ String): Unit

def logError(msg: ⇒ String, throwable: Throwable): Unit

def logError(msg: ⇒ String): Unit

def logInfo(msg: ⇒ String, throwable: Throwable): Unit

def logInfo(msg: ⇒ String): Unit

def logTrace(msg: ⇒ String, throwable: Throwable): Unit

def logTrace(msg: ⇒ String): Unit

def logWarning(msg: ⇒ String, throwable: Throwable): Unit

def logWarning(msg: ⇒ String): Unit

final def ne(arg0: AnyRef): Boolean

def newJobContext(conf: JobConf, jobId: JobID): JobContext

def newTaskAttemptContext(conf: JobConf, attemptId: TaskAttemptID): TaskAttemptContext

def newTaskAttemptID(jtIdentifier: String, jobId: Int, isMap: Boolean, taskId: Int, attemptId: Int): TaskAttemptID

final def notify(): Unit

final def notifyAll(): Unit

def open(): Unit

def preSetup(): Unit

def setup(jobid: Int, splitid: Int, attemptid: Int): Unit

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

def write(key: AnyRef, value: AnyRef): Unit

Inherited from Serializable

Inherited from Serializable

Inherited from SparkHadoopMapRedUtil

Inherited from Logging

Inherited from AnyRef

Inherited from Any