org.apache.spark.streaming.util

Class HdfsUtils

Object

org.apache.spark.streaming.util.HdfsUtils

public class HdfsUtils
extends Object

Constructor Summary

Constructors

Constructor and Description
HdfsUtils()

Method Summary

Methods

Modifier and Type	Method and Description
static void	checkState(boolean state, scala.Function0<String> errorMsg)
static String[]	getFileSegmentLocations(String path, long offset, long length, org.apache.hadoop.conf.Configuration conf) Get the locations of the HDFS blocks containing the given file segment.
static org.apache.hadoop.fs.FileSystem	getFileSystemForPath(org.apache.hadoop.fs.Path path, org.apache.hadoop.conf.Configuration conf)
static org.apache.hadoop.fs.FSDataInputStream	getInputStream(String path, org.apache.hadoop.conf.Configuration conf)
static org.apache.hadoop.fs.FSDataOutputStream	getOutputStream(String path, org.apache.hadoop.conf.Configuration conf)

Methods inherited from class Object

equals, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

HdfsUtils

public HdfsUtils()

Method Detail

getOutputStream

public static org.apache.hadoop.fs.FSDataOutputStream getOutputStream(String path,
                                                      org.apache.hadoop.conf.Configuration conf)

getInputStream

public static org.apache.hadoop.fs.FSDataInputStream getInputStream(String path,
                                                    org.apache.hadoop.conf.Configuration conf)

checkState

public static void checkState(boolean state,
              scala.Function0<String> errorMsg)

getFileSegmentLocations

public static String[] getFileSegmentLocations(String path,
                               long offset,
                               long length,
                               org.apache.hadoop.conf.Configuration conf)

Get the locations of the HDFS blocks containing the given file segment.

getFileSystemForPath

public static org.apache.hadoop.fs.FileSystem getFileSystemForPath(org.apache.hadoop.fs.Path path,
                                                   org.apache.hadoop.conf.Configuration conf)