JobLogger

Instance Constructors

new JobLogger()
new JobLogger(user: String, logDirName: String)

logDirName
The base directory for the log files.

Value Members

final def !=(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def !=(arg0: Any): Boolean

Definition Classes
Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def ==(arg0: Any): Boolean

Definition Classes
Any
final def asInstanceOf[T0]: T0

Definition Classes
Any
def buildJobDep(jobID: Int, stage: Stage): Unit

Build stage dependency for a job
Build stage dependency for a job
jobID
ID of the job
stage
Root stage of the job

Attributes
protected
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def closeLogWriter(jobID: Int): Unit

Close log file, and clean the stage relationship in stageIDToJobID
Close log file, and clean the stage relationship in stageIDToJobID
jobID
ID of the job

Attributes
protected
def createLogDir(): Unit

Create a folder for log files, the folder's name is the creation time of jobLogger
Create a folder for log files, the folder's name is the creation time of jobLogger

Attributes
protected
def createLogWriter(jobID: Int): Unit

Create a log file for one job
Create a log file for one job
jobID
ID of the job

Attributes
protected
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def getRddName(rdd: RDD[_]): String

Get RDD's name
Get RDD's name
rdd
Input RDD
returns
String of RDD's name

Attributes
protected
def hashCode(): Int

Definition Classes
AnyRef → Any
def indentString(indent: Int): String

Generate indents and convert to String
Generate indents and convert to String
indent
Number of indents
returns
string of indents

Attributes
protected
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def isTraceEnabled(): Boolean

Attributes
protected
Definition Classes
Logging
def jobLogInfo(jobID: Int, info: String, withTime: Boolean = true): Unit

Write info into log file
Write info into log file
jobID
ID of the job
info
Info to be recorded
withTime
Controls whether to record time stamp before the info, default is true

Attributes
protected
def log: Logger

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
val logDirName: String

The base directory for the log files.
def logError(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def onJobEnd(jobEnd: SparkListenerJobEnd): Unit

When job ends, recording job completion status and close log file
When job ends, recording job completion status and close log file
jobEnd
Job end event

Definition Classes
JobLogger → SparkListener
def onJobStart(jobStart: SparkListenerJobStart): Unit

When job starts, record job property and stage graph
When job starts, record job property and stage graph
jobStart
Job start event

Definition Classes
JobLogger → SparkListener
def onStageCompleted(stageCompleted: SparkListenerStageCompleted): Unit

When stage is completed, record stage completion status
When stage is completed, record stage completion status
stageCompleted
Stage completed event

Definition Classes
JobLogger → SparkListener
def onStageSubmitted(stageSubmitted: SparkListenerStageSubmitted): Unit

When stage is submitted, record stage submit info
When stage is submitted, record stage submit info
stageSubmitted
Stage submitted event

Definition Classes
JobLogger → SparkListener
def onTaskEnd(taskEnd: SparkListenerTaskEnd): Unit

When task ends, record task completion status and metrics
When task ends, record task completion status and metrics
taskEnd
Task end event

Definition Classes
JobLogger → SparkListener
def onTaskGettingResult(taskGettingResult: SparkListenerTaskGettingResult): Unit

Called when a task begins remotely fetching its result (will not be called for tasks that do not need to fetch the result remotely).
Called when a task begins remotely fetching its result (will not be called for tasks that do not need to fetch the result remotely).

Definition Classes
SparkListener
def onTaskStart(taskStart: SparkListenerTaskStart): Unit

Called when a task starts
Called when a task starts

Definition Classes
JobLogger → SparkListener
def recordJobProperties(jobID: Int, properties: Properties): Unit

Record job properties into job log file
Record job properties into job log file
jobID
ID of the job
properties
Properties of the job

Attributes
protected
def recordRddInStageGraph(jobID: Int, rdd: RDD[_], indent: Int): Unit

Record RDD dependency graph in a stage
Record RDD dependency graph in a stage
jobID
Job ID of the stage
rdd
Root RDD of the stage
indent
Indent number before info

Attributes
protected
def recordStageDep(jobID: Int): Unit

Record stage dependency and RDD dependency for a stage
Record stage dependency and RDD dependency for a stage
jobID
Job ID of the stage

Attributes
protected
def recordStageDepGraph(jobID: Int, stage: Stage, idSet: HashSet[Int], indent: Int = 0): Unit

Record stage dependency graph of a job
Record stage dependency graph of a job
jobID
Job ID of the stage
stage
Root stage of the job
indent
Indent number before info, default is 0

Attributes
protected
def recordTaskMetrics(stageID: Int, status: String, taskInfo: TaskInfo, taskMetrics: TaskMetrics): Unit

Record task metrics into job log files, including execution info and shuffle metrics
Record task metrics into job log files, including execution info and shuffle metrics
stageID
Stage ID of the task
status
Status info of the task
taskInfo
Task description info
taskMetrics
Task running metrics

Attributes
protected
def stageLogInfo(stageID: Int, info: String, withTime: Boolean = true): Unit

Write info into log file
Write info into log file
stageID
ID of the stage
info
Info to be recorded
withTime
Controls whether to record time stamp before the info, default is true

Attributes
protected
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
val user: String
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

class JobLogger extends SparkListener with Logging

Instance Constructors

new JobLogger()

new JobLogger(user: String, logDirName: String)

Value Members

final def !=(arg0: AnyRef): Boolean

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: AnyRef): Boolean

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

def buildJobDep(jobID: Int, stage: Stage): Unit

def clone(): AnyRef

def closeLogWriter(jobID: Int): Unit

def createLogDir(): Unit

def createLogWriter(jobID: Int): Unit

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

final def getClass(): Class[_]

def getRddName(rdd: RDD[_]): String

def hashCode(): Int

def indentString(indent: Int): String

final def isInstanceOf[T0]: Boolean

def isTraceEnabled(): Boolean

def jobLogInfo(jobID: Int, info: String, withTime: Boolean = true): Unit

def log: Logger

def logDebug(msg: ⇒ String, throwable: Throwable): Unit

def logDebug(msg: ⇒ String): Unit

val logDirName: String

def logError(msg: ⇒ String, throwable: Throwable): Unit

def logError(msg: ⇒ String): Unit

def logInfo(msg: ⇒ String, throwable: Throwable): Unit

def logInfo(msg: ⇒ String): Unit

def logTrace(msg: ⇒ String, throwable: Throwable): Unit

def logTrace(msg: ⇒ String): Unit

def logWarning(msg: ⇒ String, throwable: Throwable): Unit

def logWarning(msg: ⇒ String): Unit

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

def onJobEnd(jobEnd: SparkListenerJobEnd): Unit

def onJobStart(jobStart: SparkListenerJobStart): Unit

def onStageCompleted(stageCompleted: SparkListenerStageCompleted): Unit

def onStageSubmitted(stageSubmitted: SparkListenerStageSubmitted): Unit

def onTaskEnd(taskEnd: SparkListenerTaskEnd): Unit

def onTaskGettingResult(taskGettingResult: SparkListenerTaskGettingResult): Unit

def onTaskStart(taskStart: SparkListenerTaskStart): Unit

def recordJobProperties(jobID: Int, properties: Properties): Unit

def recordRddInStageGraph(jobID: Int, rdd: RDD[_], indent: Int): Unit

def recordStageDep(jobID: Int): Unit

def recordStageDepGraph(jobID: Int, stage: Stage, idSet: HashSet[Int], indent: Int = 0): Unit

def recordTaskMetrics(stageID: Int, status: String, taskInfo: TaskInfo, taskMetrics: TaskMetrics): Unit

def stageLogInfo(stageID: Int, info: String, withTime: Boolean = true): Unit

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

val user: String

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from Logging

Inherited from SparkListener

Inherited from AnyRef

Inherited from Any

Ungrouped