DefaultSource

libsvm package implements Spark SQL data source API for loading LIBSVM data as DataFrame. The loaded DataFrame has two columns: label containing labels stored as doubles and features containing feature vectors stored as Vectors.

To use LIBSVM data source, you need to set "libsvm" as the format in DataFrameReader and optionally specify options, for example:

// Scala
val df = spark.read.format("libsvm")
  .option("numFeatures", "780")
  .load("data/mllib/sample_libsvm_data.txt")

// Java
DataFrame df = spark.read().format("libsvm")
  .option("numFeatures, "780")
  .load("data/mllib/sample_libsvm_data.txt");

LIBSVM data source supports the following options:

"numFeatures": number of features. If unspecified or nonpositive, the number of features will be determined automatically at the cost of one additional pass. This is also useful when the dataset is already split into multiple files and you want to load them separately, because some features may not present in certain files, which leads to inconsistent feature dimensions.
"vectorType": feature vector type, "sparse" (default) or "dense".

Annotations: @Since( "1.6.0" )
Source: LibSVMRelation.scala
See also: LIBSVM datasets

Linear Supertypes

DataSourceRegister, FileFormat, AnyRef, Any

Instance Constructors

new DefaultSource()

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def asInstanceOf[T0]: T0

Definition Classes
Any
def buildReader(sparkSession: SparkSession, dataSchema: StructType, partitionSchema: StructType, requiredSchema: StructType, filters: Seq[Filter], options: Map[String, String], hadoopConf: Configuration): (PartitionedFile) ⇒ Iterator[InternalRow]

Definition Classes
DefaultSource → FileFormat
def buildWriter(sqlContext: SQLContext, dataSchema: StructType, options: Map[String, String]): OutputWriterFactory

Definition Classes
FileFormat
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def hashCode(): Int

Definition Classes
AnyRef → Any
def inferSchema(sparkSession: SparkSession, options: Map[String, String], files: Seq[FileStatus]): Option[StructType]

Definition Classes
DefaultSource → FileFormat
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def prepareRead(sparkSession: SparkSession, options: Map[String, String], files: Seq[FileStatus]): Map[String, String]

Definition Classes
DefaultSource → FileFormat
def prepareWrite(sparkSession: SparkSession, job: Job, options: Map[String, String], dataSchema: StructType): OutputWriterFactory

Definition Classes
DefaultSource → FileFormat
def shortName(): String

The string that represents the format that this data source provider uses.
The string that represents the format that this data source provider uses. This is overridden by children to provide a nice alias for the data source. For example:
```
override def shortName(): String = "parquet"
```
Definition Classes
DefaultSource → DataSourceRegister
Annotations
@Since( "1.6.0" )
Since
1.5.0
def supportBatch(sparkSession: SparkSession, dataSchema: StructType): Boolean

Definition Classes
FileFormat
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
DefaultSource → AnyRef → Any
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Doc: package libsvm

class DefaultSource extends FileFormat with DataSourceRegister

Instance Constructors

new DefaultSource()

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

def buildReader(sparkSession: SparkSession, dataSchema: StructType, partitionSchema: StructType, requiredSchema: StructType, filters: Seq[Filter], options: Map[String, String], hadoopConf: Configuration): (PartitionedFile) ⇒ Iterator[InternalRow]

def buildWriter(sqlContext: SQLContext, dataSchema: StructType, options: Map[String, String]): OutputWriterFactory

def clone(): AnyRef

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

final def getClass(): Class[_]

def hashCode(): Int

def inferSchema(sparkSession: SparkSession, options: Map[String, String], files: Seq[FileStatus]): Option[StructType]

final def isInstanceOf[T0]: Boolean

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

def prepareRead(sparkSession: SparkSession, options: Map[String, String], files: Seq[FileStatus]): Map[String, String]

def prepareWrite(sparkSession: SparkSession, job: Job, options: Map[String, String], dataSchema: StructType): OutputWriterFactory

def shortName(): String

def supportBatch(sparkSession: SparkSession, dataSchema: StructType): Boolean

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from DataSourceRegister

Inherited from FileFormat

Inherited from AnyRef

Inherited from Any

Members