org.apache.spark.sql.hive.execution

Class InsertIntoHiveDirCommand

Object

org.apache.spark.sql.catalyst.trees.TreeNode<PlanType>

org.apache.spark.sql.catalyst.plans.QueryPlan<org.apache.spark.sql.catalyst.plans.logical.LogicalPlan>

org.apache.spark.sql.catalyst.plans.logical.LogicalPlan

org.apache.spark.sql.hive.execution.InsertIntoHiveDirCommand

All Implemented Interfaces:

java.io.Serializable, Logging, org.apache.spark.sql.catalyst.plans.logical.QueryPlanConstraints, org.apache.spark.sql.catalyst.plans.logical.statsEstimation.LogicalPlanStats, scala.Equals, scala.Product

public class InsertIntoHiveDirCommand
extends org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
implements scala.Product, scala.Serializable

Command for writing the results of query to file system.

The syntax of using this command in SQL is:

   INSERT OVERWRITE [LOCAL] DIRECTORY
   path
   [ROW FORMAT row_format]
   [STORED AS file_format]
   SELECT ...
 

param: isLocal whether the path specified in storage is a local directory param: storage storage format used to describe how the query result is stored. param: query the logical plan representing data to write to param: overwrite whether overwrites existing directory

See Also:

Serialized Form

Constructor Summary

Constructors

Constructor and Description
InsertIntoHiveDirCommand(boolean isLocal, org.apache.spark.sql.catalyst.catalog.CatalogStorageFormat storage, org.apache.spark.sql.catalyst.plans.logical.LogicalPlan query, boolean overwrite, scala.collection.Seq<org.apache.spark.sql.catalyst.expressions.Attribute> outputColumns)

Method Summary

Modifier and Type	Method and Description
static org.apache.spark.sql.catalyst.expressions.AttributeSeq	allAttributes()
static org.apache.spark.sql.catalyst.trees.TreeNode<?>	apply(int number)
static String	argString()
static String	asCode()
static org.apache.spark.sql.execution.datasources.BasicWriteJobStatsTracker	basicWriteJobStatsTracker(org.apache.hadoop.conf.Configuration hadoopConf)
abstract static boolean	canEqual(Object that)
static PlanType	canonicalized()
static scala.collection.Seq<org.apache.spark.sql.catalyst.plans.logical.LogicalPlan>	children()
static boolean	childrenResolved()
static <B> scala.collection.Seq<B>	collect(scala.PartialFunction<BaseType,B> pf)
static <B> scala.Option<B>	collectFirst(scala.PartialFunction<BaseType,B> pf)
static scala.collection.Seq<BaseType>	collectLeaves()
static org.apache.spark.sql.internal.SQLConf	conf()
static org.apache.spark.sql.catalyst.expressions.ExpressionSet	constraints()
static scala.collection.immutable.Set<org.apache.spark.sql.catalyst.trees.TreeNode<?>>	containsChild()
static void	createdTempDir_$eq(scala.Option<org.apache.hadoop.fs.Path> x$1)
static scala.Option<org.apache.hadoop.fs.Path>	createdTempDir()
scala.Option<org.apache.hadoop.fs.Path>	createdTempDir()
void	deleteExternalTmpPath(org.apache.hadoop.conf.Configuration hadoopConf)
abstract static boolean	equals(Object that)
String	executionId()
static scala.collection.Seq<org.apache.spark.sql.catalyst.expressions.Expression>	expressions()
static boolean	fastEquals(org.apache.spark.sql.catalyst.trees.TreeNode<?> other)
static scala.Option<BaseType>	find(scala.Function1<BaseType,Object> f)
static <A> scala.collection.Seq<A>	flatMap(scala.Function1<BaseType,scala.collection.TraversableOnce<A>> f)
static void	foreach(scala.Function1<BaseType,scala.runtime.BoxedUnit> f)
static void	foreachUp(scala.Function1<BaseType,scala.runtime.BoxedUnit> f)
static scala.collection.mutable.StringBuilder	generateTreeString(int depth, scala.collection.Seq<Object> lastChildren, scala.collection.mutable.StringBuilder builder, boolean verbose, String prefix, boolean addSuffix)
static String	generateTreeString$default$5()
static boolean	generateTreeString$default$6()
org.apache.hadoop.fs.Path	getExternalScratchDir(java.net.URI extURI, org.apache.hadoop.conf.Configuration hadoopConf, String stagingDir)
org.apache.hadoop.fs.Path	getExternalTmpPath(SparkSession sparkSession, org.apache.hadoop.conf.Configuration hadoopConf, org.apache.hadoop.fs.Path path)
org.apache.hadoop.fs.Path	getExtTmpPathRelTo(org.apache.hadoop.fs.Path path, org.apache.hadoop.conf.Configuration hadoopConf, String stagingDir)
org.apache.hadoop.fs.Path	getStagingDir(org.apache.hadoop.fs.Path inputPath, org.apache.hadoop.conf.Configuration hadoopConf, String stagingDir)
static int	hashCode()
static org.apache.spark.sql.catalyst.expressions.AttributeSet	inputSet()
static void	invalidateStatsCache()
boolean	isLocal()
static boolean	isStreaming()
static BaseType	makeCopy(Object[] newArgs)
static <A> scala.collection.Seq<A>	map(scala.Function1<BaseType,A> f)
static BaseType	mapChildren(scala.Function1<BaseType,BaseType> f)
static org.apache.spark.sql.catalyst.plans.QueryPlan<PlanType>	mapExpressions(scala.Function1<org.apache.spark.sql.catalyst.expressions.Expression,org.apache.spark.sql.catalyst.expressions.Expression> f)
static scala.Option<Object>	maxRows()
static scala.Option<Object>	maxRowsPerPartition()
static scala.collection.immutable.Map<String,org.apache.spark.sql.execution.metric.SQLMetric>	metrics()
static org.apache.spark.sql.catalyst.expressions.AttributeSet	missingInput()
org.apache.hadoop.fs.Path	newVersionExternalTempPath(org.apache.hadoop.fs.Path path, org.apache.hadoop.conf.Configuration hadoopConf, String stagingDir)
static String	nodeName()
static String	numberedTreeString()
org.apache.hadoop.fs.Path	oldVersionExternalTempPath(org.apache.hadoop.fs.Path path, org.apache.hadoop.conf.Configuration hadoopConf, String scratchDir)
static org.apache.spark.sql.catalyst.trees.Origin	origin()
static scala.collection.Seq<org.apache.spark.sql.catalyst.expressions.Attribute>	output()
scala.collection.Seq<org.apache.spark.sql.catalyst.expressions.Attribute>	outputColumns()
static org.apache.spark.sql.catalyst.expressions.AttributeSet	outputSet()
boolean	overwrite()
static BaseType	p(int number)
static String	prettyJson()
static void	printSchema()
static org.apache.spark.sql.catalyst.expressions.AttributeSet	producedAttributes()
abstract static int	productArity()
abstract static Object	productElement(int n)
static scala.collection.Iterator<Object>	productIterator()
static String	productPrefix()
org.apache.spark.sql.catalyst.plans.logical.LogicalPlan	query()
static org.apache.spark.sql.catalyst.expressions.AttributeSet	references()
static void	refresh()
static scala.Option<org.apache.spark.sql.catalyst.expressions.NamedExpression>	resolve(scala.collection.Seq<String> nameParts, scala.Function2<String,String,Object> resolver)
static scala.collection.Seq<org.apache.spark.sql.catalyst.expressions.Attribute>	resolve(StructType schema, scala.Function2<String,String,Object> resolver)
static scala.Option<org.apache.spark.sql.catalyst.expressions.NamedExpression>	resolveChildren(scala.collection.Seq<String> nameParts, scala.Function2<String,String,Object> resolver)
static boolean	resolved()
static scala.Option<org.apache.spark.sql.catalyst.expressions.NamedExpression>	resolveQuoted(String name, scala.Function2<String,String,Object> resolver)
scala.collection.Seq<Row>	run(SparkSession sparkSession, org.apache.spark.sql.execution.SparkPlan child)
static boolean	sameResult(PlanType other)
scala.collection.immutable.Set<String>	saveAsHiveFile(SparkSession sparkSession, org.apache.spark.sql.execution.SparkPlan plan, org.apache.hadoop.conf.Configuration hadoopConf, org.apache.spark.sql.hive.HiveShim.ShimFileSinkDesc fileSinkConf, String outputLocation, scala.collection.Seq<org.apache.spark.sql.catalyst.expressions.Attribute> allColumns, scala.collection.immutable.Map<scala.collection.immutable.Map<String,String>,String> customPartitionLocations, scala.collection.Seq<org.apache.spark.sql.catalyst.expressions.Attribute> partitionAttributes)
static StructType	schema()
static String	schemaString()
static int	semanticHash()
static String	simpleString()
static org.apache.spark.sql.catalyst.plans.logical.Statistics	stats()
org.apache.spark.sql.catalyst.catalog.CatalogStorageFormat	storage()
static scala.collection.Seq<PlanType>	subqueries()
static String	toJSON()
static String	toString()
static BaseType	transform(scala.PartialFunction<BaseType,BaseType> rule)
static org.apache.spark.sql.catalyst.plans.QueryPlan<PlanType>	transformAllExpressions(scala.PartialFunction<org.apache.spark.sql.catalyst.expressions.Expression,org.apache.spark.sql.catalyst.expressions.Expression> rule)
static BaseType	transformDown(scala.PartialFunction<BaseType,BaseType> rule)
static org.apache.spark.sql.catalyst.plans.QueryPlan<PlanType>	transformExpressions(scala.PartialFunction<org.apache.spark.sql.catalyst.expressions.Expression,org.apache.spark.sql.catalyst.expressions.Expression> rule)
static org.apache.spark.sql.catalyst.plans.QueryPlan<PlanType>	transformExpressionsDown(scala.PartialFunction<org.apache.spark.sql.catalyst.expressions.Expression,org.apache.spark.sql.catalyst.expressions.Expression> rule)
static org.apache.spark.sql.catalyst.plans.QueryPlan<PlanType>	transformExpressionsUp(scala.PartialFunction<org.apache.spark.sql.catalyst.expressions.Expression,org.apache.spark.sql.catalyst.expressions.Expression> rule)
static BaseType	transformUp(scala.PartialFunction<BaseType,BaseType> rule)
static String	treeString()
static String	treeString(boolean verbose, boolean addSuffix)
static boolean	treeString$default$2()
static String	verboseString()
static String	verboseStringWithSuffix()
static BaseType	withNewChildren(scala.collection.Seq<BaseType> newChildren)

Methods inherited from class org.apache.spark.sql.catalyst.plans.logical.LogicalPlan

childrenResolved, constraints, initializeLogIfNecessary, initializeLogIfNecessary, initializeLogIfNecessary$default$2, invalidateStatsCache, isStreaming, isTraceEnabled, log, logDebug, logDebug, logError, logError, logInfo, logInfo, logName, logTrace, logTrace, logWarning, logWarning, maxRows, maxRowsPerPartition, org$apache$spark$internal$Logging$$log__$eq, org$apache$spark$internal$Logging$$log_, org$apache$spark$sql$catalyst$plans$logical$LogicalPlan$$name$1, org$apache$spark$sql$catalyst$plans$logical$LogicalPlan$$resolveAsColumn, org$apache$spark$sql$catalyst$plans$logical$LogicalPlan$$resolveAsTableColumn, refresh, resolve, resolve, resolve, resolveChildren, resolved, resolveQuoted, statePrefix, stats, statsCache_$eq, statsCache, validConstraints, verboseStringWithSuffix

Methods inherited from class org.apache.spark.sql.catalyst.plans.QueryPlan

allAttributes, canEvaluate, canEvaluateWithinJoin, canonicalized, conf, doCanonicalize, expressions, innerChildren, inputSet, isCanonicalizedPlan, mapExpressions, missingInput, normalizeExprId, normalizePredicates, org$apache$spark$sql$catalyst$plans$QueryPlan$$recursiveTransform$1, org$apache$spark$sql$catalyst$plans$QueryPlan$$seqToExpressions$1, output, outputSet, printSchema, producedAttributes, references, replaceAlias, sameResult, schema, schemaString, semanticHash, simpleString, splitConjunctivePredicates, splitDisjunctivePredicates, subqueries, transformAllExpressions, transformExpressions, transformExpressionsDown, transformExpressionsUp, verboseString

Methods inherited from class org.apache.spark.sql.catalyst.trees.TreeNode

apply, argString, asCode, children, collect, collectFirst, collectLeaves, containsChild, fastEquals, find, flatMap, foreach, foreachUp, generateTreeString, generateTreeString$default$5, generateTreeString$default$6, hashCode, jsonFields, makeCopy, map, mapChildren, mapProductIterator, nodeName, numberedTreeString, org$apache$spark$sql$catalyst$trees$TreeNode$$allChildren, org$apache$spark$sql$catalyst$trees$TreeNode$$collectJsonValue$1, org$apache$spark$sql$catalyst$trees$TreeNode$$getNodeNumbered, org$apache$spark$sql$catalyst$trees$TreeNode$$parseToJson, origin, otherCopyArgs, p, prettyJson, productIterator, productPrefix, stringArgs, toJSON, toString, transform, transformDown, transformUp, treeString, treeString, treeString$default$2, withNewChildren

Methods inherited from class Object

equals, getClass, notify, notifyAll, wait, wait, wait

Methods inherited from interface org.apache.spark.sql.execution.command.DataWritingCommand

basicWriteJobStatsTracker, children, metrics

Methods inherited from interface org.apache.spark.sql.catalyst.plans.logical.Command

output

Methods inherited from interface scala.Product

productArity, productElement, productIterator, productPrefix

Methods inherited from interface scala.Equals

canEqual, equals

Methods inherited from interface org.apache.spark.internal.Logging

initializeLogging, log_

Constructor Detail

InsertIntoHiveDirCommand

public InsertIntoHiveDirCommand(boolean isLocal,
                                org.apache.spark.sql.catalyst.catalog.CatalogStorageFormat storage,
                                org.apache.spark.sql.catalyst.plans.logical.LogicalPlan query,
                                boolean overwrite,
                                scala.collection.Seq<org.apache.spark.sql.catalyst.expressions.Attribute> outputColumns)

Method Detail

canEqual

public abstract static boolean canEqual(Object that)

equals

public abstract static boolean equals(Object that)

productElement

public abstract static Object productElement(int n)

productArity

public abstract static int productArity()

productIterator

public static scala.collection.Iterator<Object> productIterator()

productPrefix

public static String productPrefix()

origin

public static org.apache.spark.sql.catalyst.trees.Origin origin()

containsChild

public static scala.collection.immutable.Set<org.apache.spark.sql.catalyst.trees.TreeNode<?>> containsChild()

hashCode

public static int hashCode()

fastEquals

public static boolean fastEquals(org.apache.spark.sql.catalyst.trees.TreeNode<?> other)

find

public static scala.Option<BaseType> find(scala.Function1<BaseType,Object> f)

foreach

public static void foreach(scala.Function1<BaseType,scala.runtime.BoxedUnit> f)

foreachUp

public static void foreachUp(scala.Function1<BaseType,scala.runtime.BoxedUnit> f)

map

public static <A> scala.collection.Seq<A> map(scala.Function1<BaseType,A> f)

flatMap

public static <A> scala.collection.Seq<A> flatMap(scala.Function1<BaseType,scala.collection.TraversableOnce<A>> f)

collect

public static <B> scala.collection.Seq<B> collect(scala.PartialFunction<BaseType,B> pf)

collectLeaves

public static scala.collection.Seq<BaseType> collectLeaves()

collectFirst

public static <B> scala.Option<B> collectFirst(scala.PartialFunction<BaseType,B> pf)

withNewChildren

public static BaseType withNewChildren(scala.collection.Seq<BaseType> newChildren)

transform

public static BaseType transform(scala.PartialFunction<BaseType,BaseType> rule)

transformDown

public static BaseType transformDown(scala.PartialFunction<BaseType,BaseType> rule)

transformUp

public static BaseType transformUp(scala.PartialFunction<BaseType,BaseType> rule)

mapChildren

public static BaseType mapChildren(scala.Function1<BaseType,BaseType> f)

makeCopy

public static BaseType makeCopy(Object[] newArgs)

nodeName

public static String nodeName()

argString

public static String argString()

toString

public static String toString()

treeString

public static String treeString()

treeString

public static String treeString(boolean verbose,
                                boolean addSuffix)

numberedTreeString

public static String numberedTreeString()

apply

public static org.apache.spark.sql.catalyst.trees.TreeNode<?> apply(int number)

p

public static BaseType p(int number)

generateTreeString

public static scala.collection.mutable.StringBuilder generateTreeString(int depth,
                                                                        scala.collection.Seq<Object> lastChildren,
                                                                        scala.collection.mutable.StringBuilder builder,
                                                                        boolean verbose,
                                                                        String prefix,
                                                                        boolean addSuffix)

asCode

public static String asCode()

toJSON

public static String toJSON()

prettyJson

public static String prettyJson()

treeString$default$2

public static boolean treeString$default$2()

generateTreeString$default$5

public static String generateTreeString$default$5()

generateTreeString$default$6

public static boolean generateTreeString$default$6()

conf

public static org.apache.spark.sql.internal.SQLConf conf()

outputSet

public static org.apache.spark.sql.catalyst.expressions.AttributeSet outputSet()

references

public static org.apache.spark.sql.catalyst.expressions.AttributeSet references()

inputSet

public static org.apache.spark.sql.catalyst.expressions.AttributeSet inputSet()

producedAttributes

public static org.apache.spark.sql.catalyst.expressions.AttributeSet producedAttributes()

missingInput

public static org.apache.spark.sql.catalyst.expressions.AttributeSet missingInput()

transformExpressions

public static org.apache.spark.sql.catalyst.plans.QueryPlan<PlanType> transformExpressions(scala.PartialFunction<org.apache.spark.sql.catalyst.expressions.Expression,org.apache.spark.sql.catalyst.expressions.Expression> rule)

transformExpressionsDown

public static org.apache.spark.sql.catalyst.plans.QueryPlan<PlanType> transformExpressionsDown(scala.PartialFunction<org.apache.spark.sql.catalyst.expressions.Expression,org.apache.spark.sql.catalyst.expressions.Expression> rule)

transformExpressionsUp

public static org.apache.spark.sql.catalyst.plans.QueryPlan<PlanType> transformExpressionsUp(scala.PartialFunction<org.apache.spark.sql.catalyst.expressions.Expression,org.apache.spark.sql.catalyst.expressions.Expression> rule)

mapExpressions

public static org.apache.spark.sql.catalyst.plans.QueryPlan<PlanType> mapExpressions(scala.Function1<org.apache.spark.sql.catalyst.expressions.Expression,org.apache.spark.sql.catalyst.expressions.Expression> f)

transformAllExpressions

public static org.apache.spark.sql.catalyst.plans.QueryPlan<PlanType> transformAllExpressions(scala.PartialFunction<org.apache.spark.sql.catalyst.expressions.Expression,org.apache.spark.sql.catalyst.expressions.Expression> rule)

expressions

public static final scala.collection.Seq<org.apache.spark.sql.catalyst.expressions.Expression> expressions()

schema

public static StructType schema()

schemaString

public static String schemaString()

printSchema

public static void printSchema()

simpleString

public static String simpleString()

verboseString

public static String verboseString()

subqueries

public static scala.collection.Seq<PlanType> subqueries()

canonicalized

public static final PlanType canonicalized()

sameResult

public static final boolean sameResult(PlanType other)

semanticHash

public static final int semanticHash()

allAttributes

public static org.apache.spark.sql.catalyst.expressions.AttributeSeq allAttributes()

stats

public static org.apache.spark.sql.catalyst.plans.logical.Statistics stats()

invalidateStatsCache

public static final void invalidateStatsCache()

constraints

public static org.apache.spark.sql.catalyst.expressions.ExpressionSet constraints()

isStreaming

public static boolean isStreaming()

verboseStringWithSuffix

public static String verboseStringWithSuffix()

maxRows

public static scala.Option<Object> maxRows()

maxRowsPerPartition

public static scala.Option<Object> maxRowsPerPartition()

resolved

public static boolean resolved()

childrenResolved

public static boolean childrenResolved()

resolve

public static scala.collection.Seq<org.apache.spark.sql.catalyst.expressions.Attribute> resolve(StructType schema,
                                                                                                scala.Function2<String,String,Object> resolver)

resolveChildren

public static scala.Option<org.apache.spark.sql.catalyst.expressions.NamedExpression> resolveChildren(scala.collection.Seq<String> nameParts,
                                                                                                      scala.Function2<String,String,Object> resolver)

resolve

public static scala.Option<org.apache.spark.sql.catalyst.expressions.NamedExpression> resolve(scala.collection.Seq<String> nameParts,
                                                                                              scala.Function2<String,String,Object> resolver)

resolveQuoted

public static scala.Option<org.apache.spark.sql.catalyst.expressions.NamedExpression> resolveQuoted(String name,
                                                                                                    scala.Function2<String,String,Object> resolver)

refresh

public static void refresh()

output

public static scala.collection.Seq<org.apache.spark.sql.catalyst.expressions.Attribute> output()

children

public static final scala.collection.Seq<org.apache.spark.sql.catalyst.plans.logical.LogicalPlan> children()

metrics

public static scala.collection.immutable.Map<String,org.apache.spark.sql.execution.metric.SQLMetric> metrics()

basicWriteJobStatsTracker

public static org.apache.spark.sql.execution.datasources.BasicWriteJobStatsTracker basicWriteJobStatsTracker(org.apache.hadoop.conf.Configuration hadoopConf)

createdTempDir

public static scala.Option<org.apache.hadoop.fs.Path> createdTempDir()

createdTempDir_$eq

public static void createdTempDir_$eq(scala.Option<org.apache.hadoop.fs.Path> x$1)

isLocal

public boolean isLocal()

storage

public org.apache.spark.sql.catalyst.catalog.CatalogStorageFormat storage()

query

public org.apache.spark.sql.catalyst.plans.logical.LogicalPlan query()

overwrite

public boolean overwrite()

outputColumns

public scala.collection.Seq<org.apache.spark.sql.catalyst.expressions.Attribute> outputColumns()

run

public scala.collection.Seq<Row> run(SparkSession sparkSession,
                                     org.apache.spark.sql.execution.SparkPlan child)

createdTempDir

public scala.Option<org.apache.hadoop.fs.Path> createdTempDir()

saveAsHiveFile

public scala.collection.immutable.Set<String> saveAsHiveFile(SparkSession sparkSession,
                                                             org.apache.spark.sql.execution.SparkPlan plan,
                                                             org.apache.hadoop.conf.Configuration hadoopConf,
                                                             org.apache.spark.sql.hive.HiveShim.ShimFileSinkDesc fileSinkConf,
                                                             String outputLocation,
                                                             scala.collection.Seq<org.apache.spark.sql.catalyst.expressions.Attribute> allColumns,
                                                             scala.collection.immutable.Map<scala.collection.immutable.Map<String,String>,String> customPartitionLocations,
                                                             scala.collection.Seq<org.apache.spark.sql.catalyst.expressions.Attribute> partitionAttributes)

getExternalTmpPath

public org.apache.hadoop.fs.Path getExternalTmpPath(SparkSession sparkSession,
                                                    org.apache.hadoop.conf.Configuration hadoopConf,
                                                    org.apache.hadoop.fs.Path path)

deleteExternalTmpPath

public void deleteExternalTmpPath(org.apache.hadoop.conf.Configuration hadoopConf)

oldVersionExternalTempPath

public org.apache.hadoop.fs.Path oldVersionExternalTempPath(org.apache.hadoop.fs.Path path,
                                                            org.apache.hadoop.conf.Configuration hadoopConf,
                                                            String scratchDir)

newVersionExternalTempPath

public org.apache.hadoop.fs.Path newVersionExternalTempPath(org.apache.hadoop.fs.Path path,
                                                            org.apache.hadoop.conf.Configuration hadoopConf,
                                                            String stagingDir)

getExtTmpPathRelTo

public org.apache.hadoop.fs.Path getExtTmpPathRelTo(org.apache.hadoop.fs.Path path,
                                                    org.apache.hadoop.conf.Configuration hadoopConf,
                                                    String stagingDir)

getExternalScratchDir

public org.apache.hadoop.fs.Path getExternalScratchDir(java.net.URI extURI,
                                                       org.apache.hadoop.conf.Configuration hadoopConf,
                                                       String stagingDir)

getStagingDir

public org.apache.hadoop.fs.Path getStagingDir(org.apache.hadoop.fs.Path inputPath,
                                               org.apache.hadoop.conf.Configuration hadoopConf,
                                               String stagingDir)

executionId

public String executionId()