org.apache.spark.sql.hive.execution

Class ScriptTransformation

Object

org.apache.spark.sql.catalyst.trees.TreeNode<PlanType>

org.apache.spark.sql.catalyst.plans.QueryPlan<org.apache.spark.sql.execution.SparkPlan>

org.apache.spark.sql.execution.SparkPlan

org.apache.spark.sql.hive.execution.ScriptTransformation

All Implemented Interfaces:

java.io.Serializable, org.apache.spark.sql.execution.UnaryExecNode, scala.Equals, scala.Product

public class ScriptTransformation
extends org.apache.spark.sql.execution.SparkPlan
implements org.apache.spark.sql.execution.UnaryExecNode, scala.Product, scala.Serializable

Transforms the input by forking and running the specified script.

param: input the set of expression that should be passed to the script. param: script the command that should be executed. param: output the attributes that are produced by the script.

See Also:

Serialized Form

Constructor Summary

Constructors

Constructor and Description
ScriptTransformation(scala.collection.Seq<org.apache.spark.sql.catalyst.expressions.Expression> input, String script, scala.collection.Seq<org.apache.spark.sql.catalyst.expressions.Attribute> output, org.apache.spark.sql.execution.SparkPlan child, HiveScriptIOSchema ioschema)

Method Summary

Methods

Modifier and Type	Method and Description
static org.apache.spark.sql.catalyst.expressions.AttributeSeq	allAttributes()
static org.apache.spark.sql.catalyst.trees.TreeNode<?>	apply(int number)
static String	argString()
static String	asCode()
abstract static boolean	canEqual(Object that)
org.apache.spark.sql.execution.SparkPlan	child()
static scala.collection.Seq<org.apache.spark.sql.execution.SparkPlan>	children()
static <B> scala.collection.Seq<B>	collect(scala.PartialFunction<BaseType,B> pf)
static <B> scala.Option<B>	collectFirst(scala.PartialFunction<BaseType,B> pf)
static scala.collection.Seq<BaseType>	collectLeaves()
static org.apache.spark.sql.catalyst.expressions.ExpressionSet	constraints()
static scala.collection.immutable.Set<org.apache.spark.sql.catalyst.trees.TreeNode<?>>	containsChild()
abstract static boolean	equals(Object that)
static RDD<org.apache.spark.sql.catalyst.InternalRow>	execute()
static <T> Broadcast<T>	executeBroadcast()
static org.apache.spark.sql.catalyst.InternalRow[]	executeCollect()
static Row[]	executeCollectPublic()
static org.apache.spark.sql.catalyst.InternalRow[]	executeTake(int n)
static scala.collection.Iterator<org.apache.spark.sql.catalyst.InternalRow>	executeToIterator()
static scala.collection.Seq<org.apache.spark.sql.catalyst.expressions.Expression>	expressions()
static boolean	fastEquals(org.apache.spark.sql.catalyst.trees.TreeNode<?> other)
static scala.Option<BaseType>	find(scala.Function1<BaseType,Object> f)
static <A> scala.collection.Seq<A>	flatMap(scala.Function1<BaseType,scala.collection.TraversableOnce<A>> f)
static void	foreach(scala.Function1<BaseType,scala.runtime.BoxedUnit> f)
static void	foreachUp(scala.Function1<BaseType,scala.runtime.BoxedUnit> f)
static scala.collection.mutable.StringBuilder	generateTreeString(int depth, scala.collection.Seq<Object> lastChildren, scala.collection.mutable.StringBuilder builder, boolean verbose, String prefix)
static String	generateTreeString$default$5()
static int	hashCode()
void	initializeLogging(boolean isInterpreter)
scala.collection.Seq<org.apache.spark.sql.catalyst.expressions.Expression>	input()
static org.apache.spark.sql.catalyst.expressions.AttributeSet	inputSet()
HiveScriptIOSchema	ioschema()
org.slf4j.Logger	log_()
static org.apache.spark.sql.execution.metric.SQLMetric	longMetric(String name)
static org.apache.spark.sql.execution.SparkPlan	makeCopy(Object[] newArgs)
static <A> scala.collection.Seq<A>	map(scala.Function1<BaseType,A> f)
static BaseType	mapChildren(scala.Function1<BaseType,BaseType> f)
static org.apache.spark.sql.catalyst.plans.QueryPlan<PlanType>	mapExpressions(scala.Function1<org.apache.spark.sql.catalyst.expressions.Expression,org.apache.spark.sql.catalyst.expressions.Expression> f)
static scala.collection.immutable.Map<String,String>	metadata()
static scala.collection.immutable.Map<String,org.apache.spark.sql.execution.metric.SQLMetric>	metrics()
static org.apache.spark.sql.catalyst.expressions.AttributeSet	missingInput()
static String	nodeName()
static String	numberedTreeString()
static org.apache.spark.sql.catalyst.trees.Origin	origin()
scala.collection.Seq<org.apache.spark.sql.catalyst.expressions.Attribute>	output()
static scala.collection.Seq<org.apache.spark.sql.catalyst.expressions.SortOrder>	outputOrdering()
org.apache.spark.sql.catalyst.plans.physical.Partitioning	outputPartitioning()
static org.apache.spark.sql.catalyst.expressions.AttributeSet	outputSet()
static BaseType	p(int number)
static void	prepare()
static String	prettyJson()
static void	printSchema()
org.apache.spark.sql.catalyst.expressions.AttributeSet	producedAttributes()
abstract static int	productArity()
abstract static Object	productElement(int n)
static scala.collection.Iterator<Object>	productIterator()
static String	productPrefix()
static org.apache.spark.sql.catalyst.expressions.AttributeSet	references()
static scala.collection.Seq<org.apache.spark.sql.catalyst.plans.physical.Distribution>	requiredChildDistribution()
static scala.collection.Seq<scala.collection.Seq<org.apache.spark.sql.catalyst.expressions.SortOrder>>	requiredChildOrdering()
static void	resetMetrics()
static boolean	sameResult(PlanType plan)
static StructType	schema()
static String	schemaString()
String	script()
static String	simpleString()
static SQLContext	sqlContext()
static boolean	subexpressionEliminationEnabled()
static scala.collection.Seq<PlanType>	subqueries()
static String	toJSON()
static String	toString()
static BaseType	transform(scala.PartialFunction<BaseType,BaseType> rule)
static org.apache.spark.sql.catalyst.plans.QueryPlan<PlanType>	transformAllExpressions(scala.PartialFunction<org.apache.spark.sql.catalyst.expressions.Expression,org.apache.spark.sql.catalyst.expressions.Expression> rule)
static BaseType	transformDown(scala.PartialFunction<BaseType,BaseType> rule)
static org.apache.spark.sql.catalyst.plans.QueryPlan<PlanType>	transformExpressions(scala.PartialFunction<org.apache.spark.sql.catalyst.expressions.Expression,org.apache.spark.sql.catalyst.expressions.Expression> rule)
static org.apache.spark.sql.catalyst.plans.QueryPlan<PlanType>	transformExpressionsDown(scala.PartialFunction<org.apache.spark.sql.catalyst.expressions.Expression,org.apache.spark.sql.catalyst.expressions.Expression> rule)
static org.apache.spark.sql.catalyst.plans.QueryPlan<PlanType>	transformExpressionsUp(scala.PartialFunction<org.apache.spark.sql.catalyst.expressions.Expression,org.apache.spark.sql.catalyst.expressions.Expression> rule)
static BaseType	transformUp(scala.PartialFunction<BaseType,BaseType> rule)
static String	treeString()
static String	treeString(boolean verbose)
static String	verboseString()
static BaseType	withNewChildren(scala.collection.Seq<BaseType> newChildren)

Methods inherited from class org.apache.spark.sql.execution.SparkPlan

doExecuteBroadcast, doPrepare, execute, executeBroadcast, executeCollect, executeCollectPublic, executeQuery, executeTake, executeToIterator, initializeLogIfNecessary, isTraceEnabled, log, logDebug, logDebug, logError, logError, logInfo, logInfo, logName, logTrace, logTrace, logWarning, logWarning, longMetric, makeCopy, metadata, metrics, newMutableProjection, newMutableProjection$default$3, newNaturalAscendingOrdering, newOrdering, newPredicate, org$apache$spark$internal$Logging$$log__$eq, org$apache$spark$internal$Logging$$log_, org$apache$spark$sql$execution$SparkPlan$$decodeUnsafeRows, org$apache$spark$sql$execution$SparkPlan$$runningSubqueries, outputOrdering, prepare, prepareSubqueries, requiredChildDistribution, requiredChildOrdering, resetMetrics, sparkContext, sqlContext, subexpressionEliminationEnabled, waitForSubqueries

Methods inherited from class org.apache.spark.sql.catalyst.plans.QueryPlan

allAttributes, canonicalized, cleanArgs, cleanExpression, constraints, expressions, getRelevantConstraints, innerChildren, inputSet, mapExpressions, missingInput, org$apache$spark$sql$catalyst$plans$QueryPlan$$aliasMap, org$apache$spark$sql$catalyst$plans$QueryPlan$$cleanArg$1, org$apache$spark$sql$catalyst$plans$QueryPlan$$getConstraintClass, org$apache$spark$sql$catalyst$plans$QueryPlan$$inferIsNotNullConstraints, org$apache$spark$sql$catalyst$plans$QueryPlan$$isRecursiveDeduction, org$apache$spark$sql$catalyst$plans$QueryPlan$$recursiveTransform$1, org$apache$spark$sql$catalyst$plans$QueryPlan$$scanNullIntolerantAttribute, org$apache$spark$sql$catalyst$plans$QueryPlan$$seqToExpressions$1, outputSet, printSchema, references, sameResult, schema, schemaString, simpleString, statePrefix, subqueries, transformAllExpressions, transformExpressions, transformExpressionsDown, transformExpressionsUp, validConstraints, verboseString

Methods inherited from class org.apache.spark.sql.catalyst.trees.TreeNode

apply, argString, asCode, children, collect, collectFirst, collectLeaves, containsChild, fastEquals, find, flatMap, foreach, foreachUp, generateTreeString, generateTreeString$default$5, hashCode, jsonFields, map, mapChildren, mapProductIterator, nodeName, numberedTreeString, org$apache$spark$sql$catalyst$trees$TreeNode$$allChildren, org$apache$spark$sql$catalyst$trees$TreeNode$$collectJsonValue$1, org$apache$spark$sql$catalyst$trees$TreeNode$$getNodeNumbered, org$apache$spark$sql$catalyst$trees$TreeNode$$parseToJson, origin, otherCopyArgs, p, prettyJson, productIterator, productPrefix, stringArgs, toJSON, toString, transform, transformDown, transformUp, treeString, treeString, withNewChildren

Methods inherited from class Object

equals, getClass, notify, notifyAll, wait, wait, wait

Methods inherited from interface org.apache.spark.sql.execution.UnaryExecNode

children

Methods inherited from interface scala.Product

productArity, productElement, productIterator, productPrefix

Methods inherited from interface scala.Equals

canEqual, equals

Constructor Detail

ScriptTransformation

public ScriptTransformation(scala.collection.Seq<org.apache.spark.sql.catalyst.expressions.Expression> input,
                    String script,
                    scala.collection.Seq<org.apache.spark.sql.catalyst.expressions.Attribute> output,
                    org.apache.spark.sql.execution.SparkPlan child,
                    HiveScriptIOSchema ioschema)

Method Detail

canEqual

public abstract static boolean canEqual(Object that)

equals

public abstract static boolean equals(Object that)

productElement

public abstract static Object productElement(int n)

productArity

public abstract static int productArity()

productIterator

public static scala.collection.Iterator<Object> productIterator()

productPrefix

public static String productPrefix()

origin

public static org.apache.spark.sql.catalyst.trees.Origin origin()

containsChild

public static scala.collection.immutable.Set<org.apache.spark.sql.catalyst.trees.TreeNode<?>> containsChild()

hashCode

public static int hashCode()

fastEquals

public static boolean fastEquals(org.apache.spark.sql.catalyst.trees.TreeNode<?> other)

find

public static scala.Option<BaseType> find(scala.Function1<BaseType,Object> f)

foreach

public static void foreach(scala.Function1<BaseType,scala.runtime.BoxedUnit> f)

foreachUp

public static void foreachUp(scala.Function1<BaseType,scala.runtime.BoxedUnit> f)

map

public static <A> scala.collection.Seq<A> map(scala.Function1<BaseType,A> f)

flatMap

public static <A> scala.collection.Seq<A> flatMap(scala.Function1<BaseType,scala.collection.TraversableOnce<A>> f)

collect

public static <B> scala.collection.Seq<B> collect(scala.PartialFunction<BaseType,B> pf)

collectLeaves

public static scala.collection.Seq<BaseType> collectLeaves()

collectFirst

public static <B> scala.Option<B> collectFirst(scala.PartialFunction<BaseType,B> pf)

withNewChildren

public static BaseType withNewChildren(scala.collection.Seq<BaseType> newChildren)

transform

public static BaseType transform(scala.PartialFunction<BaseType,BaseType> rule)

transformDown

public static BaseType transformDown(scala.PartialFunction<BaseType,BaseType> rule)

transformUp

public static BaseType transformUp(scala.PartialFunction<BaseType,BaseType> rule)

mapChildren

public static BaseType mapChildren(scala.Function1<BaseType,BaseType> f)

nodeName

public static String nodeName()

argString

public static String argString()

toString

public static String toString()

treeString

public static String treeString()

treeString

public static String treeString(boolean verbose)

numberedTreeString

public static String numberedTreeString()

apply

public static org.apache.spark.sql.catalyst.trees.TreeNode<?> apply(int number)

p

public static BaseType p(int number)

generateTreeString

public static scala.collection.mutable.StringBuilder generateTreeString(int depth,
                                                        scala.collection.Seq<Object> lastChildren,
                                                        scala.collection.mutable.StringBuilder builder,
                                                        boolean verbose,
                                                        String prefix)

asCode

public static String asCode()

toJSON

public static String toJSON()

prettyJson

public static String prettyJson()

generateTreeString$default$5

public static String generateTreeString$default$5()

constraints

public static org.apache.spark.sql.catalyst.expressions.ExpressionSet constraints()

outputSet

public static org.apache.spark.sql.catalyst.expressions.AttributeSet outputSet()

references

public static org.apache.spark.sql.catalyst.expressions.AttributeSet references()

inputSet

public static org.apache.spark.sql.catalyst.expressions.AttributeSet inputSet()

missingInput

public static org.apache.spark.sql.catalyst.expressions.AttributeSet missingInput()

transformExpressions

public static org.apache.spark.sql.catalyst.plans.QueryPlan<PlanType> transformExpressions(scala.PartialFunction<org.apache.spark.sql.catalyst.expressions.Expression,org.apache.spark.sql.catalyst.expressions.Expression> rule)

transformExpressionsDown

public static org.apache.spark.sql.catalyst.plans.QueryPlan<PlanType> transformExpressionsDown(scala.PartialFunction<org.apache.spark.sql.catalyst.expressions.Expression,org.apache.spark.sql.catalyst.expressions.Expression> rule)

transformExpressionsUp

public static org.apache.spark.sql.catalyst.plans.QueryPlan<PlanType> transformExpressionsUp(scala.PartialFunction<org.apache.spark.sql.catalyst.expressions.Expression,org.apache.spark.sql.catalyst.expressions.Expression> rule)

mapExpressions

public static org.apache.spark.sql.catalyst.plans.QueryPlan<PlanType> mapExpressions(scala.Function1<org.apache.spark.sql.catalyst.expressions.Expression,org.apache.spark.sql.catalyst.expressions.Expression> f)

transformAllExpressions

public static org.apache.spark.sql.catalyst.plans.QueryPlan<PlanType> transformAllExpressions(scala.PartialFunction<org.apache.spark.sql.catalyst.expressions.Expression,org.apache.spark.sql.catalyst.expressions.Expression> rule)

expressions

public static final scala.collection.Seq<org.apache.spark.sql.catalyst.expressions.Expression> expressions()

schema

public static StructType schema()

schemaString

public static String schemaString()

printSchema

public static void printSchema()

simpleString

public static String simpleString()

verboseString

public static String verboseString()

subqueries

public static scala.collection.Seq<PlanType> subqueries()

sameResult

public static boolean sameResult(PlanType plan)

allAttributes

public static org.apache.spark.sql.catalyst.expressions.AttributeSeq allAttributes()

sqlContext

public static final SQLContext sqlContext()

subexpressionEliminationEnabled

public static boolean subexpressionEliminationEnabled()

makeCopy

public static org.apache.spark.sql.execution.SparkPlan makeCopy(Object[] newArgs)

metadata

public static scala.collection.immutable.Map<String,String> metadata()

metrics

public static scala.collection.immutable.Map<String,org.apache.spark.sql.execution.metric.SQLMetric> metrics()

resetMetrics

public static void resetMetrics()

longMetric

public static org.apache.spark.sql.execution.metric.SQLMetric longMetric(String name)

requiredChildDistribution

public static scala.collection.Seq<org.apache.spark.sql.catalyst.plans.physical.Distribution> requiredChildDistribution()

outputOrdering

public static scala.collection.Seq<org.apache.spark.sql.catalyst.expressions.SortOrder> outputOrdering()

requiredChildOrdering

public static scala.collection.Seq<scala.collection.Seq<org.apache.spark.sql.catalyst.expressions.SortOrder>> requiredChildOrdering()

execute

public static final RDD<org.apache.spark.sql.catalyst.InternalRow> execute()

executeBroadcast

public static final <T> Broadcast<T> executeBroadcast()

prepare

public static final void prepare()

executeCollect

public static org.apache.spark.sql.catalyst.InternalRow[] executeCollect()

executeToIterator

public static scala.collection.Iterator<org.apache.spark.sql.catalyst.InternalRow> executeToIterator()

executeCollectPublic

public static Row[] executeCollectPublic()

executeTake

public static org.apache.spark.sql.catalyst.InternalRow[] executeTake(int n)

children

public static final scala.collection.Seq<org.apache.spark.sql.execution.SparkPlan> children()

input

public scala.collection.Seq<org.apache.spark.sql.catalyst.expressions.Expression> input()

script

public String script()

output

public scala.collection.Seq<org.apache.spark.sql.catalyst.expressions.Attribute> output()

Specified by:

output in class org.apache.spark.sql.catalyst.plans.QueryPlan<org.apache.spark.sql.execution.SparkPlan>

child

public org.apache.spark.sql.execution.SparkPlan child()

Specified by:

child in interface org.apache.spark.sql.execution.UnaryExecNode

ioschema

public HiveScriptIOSchema ioschema()

producedAttributes

public org.apache.spark.sql.catalyst.expressions.AttributeSet producedAttributes()

Overrides:

producedAttributes in class org.apache.spark.sql.catalyst.plans.QueryPlan<org.apache.spark.sql.execution.SparkPlan>

outputPartitioning

public org.apache.spark.sql.catalyst.plans.physical.Partitioning outputPartitioning()

Overrides:

outputPartitioning in class org.apache.spark.sql.execution.SparkPlan

log_

public org.slf4j.Logger log_()

initializeLogging

public void initializeLogging(boolean isInterpreter)