case class GpuCachedDoublePassWindowExec(windowOps: Seq[NamedExpression], gpuPartitionSpec: Seq[Expression], gpuOrderSpec: Seq[SortOrder], child: SparkPlan)(cpuPartitionSpec: Seq[Expression], cpuOrderSpec: Seq[SortOrder]) extends SparkPlan with GpuWindowBaseExec with Product with Serializable
This allows for batches of data to be processed without needing them to correspond to the partition by boundaries. This is similar to GpuRunningWindowExec, but for operations that need a small amount of information from all of the batches associated with a partition instead of just the previous batch. It does this by processing a batch, collecting and updating a small cache of information about the last partition in the batch, and then putting that batch into a form that would let it be spilled if needed. A batch is released when the last partition key in the batch is fully processed. Before it is released it will be updated to include any information needed from the cached data.
Currently this only works for unbounded to unbounded windows, but could be extended to more.
Linear Supertypes
Ordering
- Alphabetic
- By Inheritance
Inherited
- GpuCachedDoublePassWindowExec
- GpuWindowBaseExec
- GpuExec
- ShimUnaryExecNode
- UnaryExecNode
- SparkPlan
- Serializable
- Serializable
- Logging
- QueryPlan
- SQLConfHelper
- TreeNode
- Product
- Equals
- AnyRef
- Any
- Hide All
- Show All
Visibility
- Public
- All
Instance Constructors
Value Members
-
final
def
!=(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
final
def
##(): Int
- Definition Classes
- AnyRef → Any
-
final
def
==(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
lazy val
additionalMetrics: Map[String, GpuMetric]
- Definition Classes
- GpuWindowBaseExec → GpuExec
-
lazy val
allAttributes: AttributeSeq
- Definition Classes
- QueryPlan
-
lazy val
allMetrics: Map[String, GpuMetric]
- Definition Classes
- GpuExec
-
def
apply(number: Int): TreeNode[_]
- Definition Classes
- TreeNode
-
def
argString(maxFields: Int): String
- Definition Classes
- TreeNode
-
def
asCode: String
- Definition Classes
- TreeNode
-
final
def
asInstanceOf[T0]: T0
- Definition Classes
- Any
-
final
lazy val
canonicalized: SparkPlan
- Definition Classes
- QueryPlan
- Annotations
- @transient()
-
val
child: SparkPlan
- Definition Classes
- GpuCachedDoublePassWindowExec → UnaryExecNode
-
final
def
children: Seq[SparkPlan]
- Definition Classes
- UnaryExecNode → TreeNode
-
def
childrenCoalesceGoal: Seq[CoalesceGoal]
A goal to coalesce batches as the input to this operation.
A goal to coalesce batches as the input to this operation. In some cases an operation will only work if all of the data is in a single batch. In other cases it may be much faster if it is in a single batch, but can tolerate multiple batches. This provides a way to express those desires.
- Definition Classes
- GpuExec
-
def
cleanupResources(): Unit
- Attributes
- protected[sql]
- Definition Classes
- SparkPlan
-
def
clone(): SparkPlan
- Definition Classes
- TreeNode → AnyRef
-
def
coalesceAfter: Boolean
If true is returned batches after this will be coalesced.
If true is returned batches after this will be coalesced. This should really be used in cases where it is known that the size of a batch may shrink a lot.
- Definition Classes
- GpuExec
-
def
collect[B](pf: PartialFunction[SparkPlan, B]): Seq[B]
- Definition Classes
- TreeNode
-
def
collectFirst[B](pf: PartialFunction[SparkPlan, B]): Option[B]
- Definition Classes
- TreeNode
-
def
collectLeaves(): Seq[SparkPlan]
- Definition Classes
- TreeNode
-
def
collectWithSubqueries[B](f: PartialFunction[SparkPlan, B]): Seq[B]
- Definition Classes
- QueryPlan
-
def
conf: SQLConf
- Definition Classes
- SQLConfHelper
-
lazy val
containsChild: Set[TreeNode[_]]
- Definition Classes
- TreeNode
-
def
copyTagsFrom(other: SparkPlan): Unit
- Definition Classes
- TreeNode
-
val
cpuOrderSpec: Seq[SortOrder]
- Definition Classes
- GpuCachedDoublePassWindowExec → GpuWindowBaseExec
-
lazy val
cpuPartitionOrdering: Seq[SortOrder]
- Definition Classes
- GpuWindowBaseExec
-
val
cpuPartitionSpec: Seq[Expression]
- Definition Classes
- GpuCachedDoublePassWindowExec → GpuWindowBaseExec
-
def
createAverageMetric(level: MetricsLevel, name: String): GpuMetric
- Attributes
- protected
- Definition Classes
- GpuExec
-
def
createFileCacheMetrics(): Map[String, GpuMetric]
- Attributes
- protected
- Definition Classes
- GpuExec
-
def
createMetric(level: MetricsLevel, name: String): GpuMetric
- Attributes
- protected
- Definition Classes
- GpuExec
-
def
createNanoTimingMetric(level: MetricsLevel, name: String): GpuMetric
- Attributes
- protected
- Definition Classes
- GpuExec
-
def
createSizeMetric(level: MetricsLevel, name: String): GpuMetric
- Attributes
- protected
- Definition Classes
- GpuExec
-
def
createTimingMetric(level: MetricsLevel, name: String): GpuMetric
- Attributes
- protected
- Definition Classes
- GpuExec
-
def
disableCoalesceUntilInput(): Boolean
Returns true if there is something in the exec that cannot work when batches between multiple file partitions are combined into a single batch (coalesce).
Returns true if there is something in the exec that cannot work when batches between multiple file partitions are combined into a single batch (coalesce).
- Definition Classes
- GpuExec
-
def
doCanonicalize(): SparkPlan
Defines how the canonicalization should work for the current plan.
Defines how the canonicalization should work for the current plan.
- Attributes
- protected
- Definition Classes
- GpuExec → QueryPlan
-
def
doExecute(): RDD[InternalRow]
- Attributes
- protected
- Definition Classes
- GpuWindowBaseExec → SparkPlan
-
def
doExecuteBroadcast[T](): Broadcast[T]
- Attributes
- protected[sql]
- Definition Classes
- SparkPlan
-
final
def
doExecuteColumnar(): RDD[ColumnarBatch]
- Definition Classes
- GpuExec → SparkPlan
-
def
doPrepare(): Unit
- Attributes
- protected
- Definition Classes
- SparkPlan
-
final
def
eq(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
final
def
execute(): RDD[InternalRow]
- Definition Classes
- SparkPlan
-
final
def
executeBroadcast[T](): Broadcast[T]
- Definition Classes
- SparkPlan
-
def
executeCollect(): Array[InternalRow]
- Definition Classes
- SparkPlan
-
def
executeCollectPublic(): Array[Row]
- Definition Classes
- SparkPlan
-
final
def
executeColumnar(): RDD[ColumnarBatch]
- Definition Classes
- SparkPlan
-
final
def
executeQuery[T](query: ⇒ T): T
- Attributes
- protected
- Definition Classes
- SparkPlan
-
def
executeTail(n: Int): Array[InternalRow]
- Definition Classes
- SparkPlan
-
def
executeTake(n: Int): Array[InternalRow]
- Definition Classes
- SparkPlan
-
def
executeToIterator(): Iterator[InternalRow]
- Definition Classes
- SparkPlan
-
final
def
expressions: Seq[Expression]
- Definition Classes
- QueryPlan
-
def
fastEquals(other: TreeNode[_]): Boolean
- Definition Classes
- TreeNode
-
def
finalize(): Unit
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( classOf[java.lang.Throwable] )
-
def
find(f: (SparkPlan) ⇒ Boolean): Option[SparkPlan]
- Definition Classes
- TreeNode
-
def
flatMap[A](f: (SparkPlan) ⇒ TraversableOnce[A]): Seq[A]
- Definition Classes
- TreeNode
-
def
foreach(f: (SparkPlan) ⇒ Unit): Unit
- Definition Classes
- TreeNode
-
def
foreachUp(f: (SparkPlan) ⇒ Unit): Unit
- Definition Classes
- TreeNode
-
def
formattedNodeName: String
- Attributes
- protected
- Definition Classes
- QueryPlan
-
def
generateTreeString(depth: Int, lastChildren: Seq[Boolean], append: (String) ⇒ Unit, verbose: Boolean, prefix: String, addSuffix: Boolean, maxFields: Int, printNodeId: Boolean, indent: Int): Unit
- Definition Classes
- TreeNode
-
final
def
getClass(): Class[_]
- Definition Classes
- AnyRef → Any
- Annotations
- @native()
-
def
getTagValue[T](tag: TreeNodeTag[T]): Option[T]
- Definition Classes
- TreeNode
-
def
getTaskMetrics: Option[GpuTaskMetrics]
- Definition Classes
- GpuExec
-
def
gpuExpressions: Seq[Expression]
Return the expressions for this plan node that should be GPU expressions.
Return the expressions for this plan node that should be GPU expressions. For most nodes this will be the same as the list of expressions, but some nodes use CPU expressions directly in some cases and will need to override this.
- Definition Classes
- GpuExec
-
def
gpuLongMetric(name: String): GpuMetric
- Definition Classes
- GpuExec
-
val
gpuOrderSpec: Seq[SortOrder]
- Definition Classes
- GpuCachedDoublePassWindowExec → GpuWindowBaseExec
-
lazy val
gpuPartitionOrdering: Seq[SortOrder]
- Definition Classes
- GpuWindowBaseExec
-
val
gpuPartitionSpec: Seq[Expression]
- Definition Classes
- GpuCachedDoublePassWindowExec → GpuWindowBaseExec
-
def
hashCode(): Int
- Definition Classes
- TreeNode → AnyRef → Any
-
val
id: Int
- Definition Classes
- SparkPlan
-
def
initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean
- Attributes
- protected
- Definition Classes
- Logging
-
def
initializeLogIfNecessary(isInterpreter: Boolean): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
innerChildren: Seq[QueryPlan[_]]
- Definition Classes
- QueryPlan → TreeNode
-
def
inputSet: AttributeSet
- Definition Classes
- QueryPlan
-
def
internalDoExecuteColumnar(): RDD[ColumnarBatch]
- Attributes
- protected
- Definition Classes
- GpuCachedDoublePassWindowExec → GpuExec
-
def
isCanonicalizedPlan: Boolean
- Attributes
- protected
- Definition Classes
- QueryPlan
-
final
def
isInstanceOf[T0]: Boolean
- Definition Classes
- Any
-
def
isTraceEnabled(): Boolean
- Attributes
- protected
- Definition Classes
- Logging
-
def
jsonFields: List[JField]
- Attributes
- protected
- Definition Classes
- TreeNode
-
def
log: Logger
- Attributes
- protected
- Definition Classes
- Logging
-
def
logDebug(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logDebug(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logError(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logError(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logInfo(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logInfo(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logName: String
- Attributes
- protected
- Definition Classes
- Logging
-
def
logTrace(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logTrace(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logWarning(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logWarning(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logicalLink: Option[LogicalPlan]
- Definition Classes
- SparkPlan
-
def
longMetric(name: String): SQLMetric
- Definition Classes
- SparkPlan
-
def
makeCopy(newArgs: Array[AnyRef]): SparkPlan
- Definition Classes
- SparkPlan → TreeNode
-
def
map[A](f: (SparkPlan) ⇒ A): Seq[A]
- Definition Classes
- TreeNode
-
def
mapChildren(f: (SparkPlan) ⇒ SparkPlan): SparkPlan
- Definition Classes
- TreeNode
-
def
mapExpressions(f: (Expression) ⇒ Expression): GpuCachedDoublePassWindowExec.this.type
- Definition Classes
- QueryPlan
-
def
mapProductIterator[B](f: (Any) ⇒ B)(implicit arg0: ClassTag[B]): Array[B]
- Attributes
- protected
- Definition Classes
- TreeNode
-
final
lazy val
metrics: Map[String, SQLMetric]
- Definition Classes
- GpuExec → SparkPlan
-
final
def
missingInput: AttributeSet
- Definition Classes
- QueryPlan
-
final
def
ne(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
def
nodeName: String
- Definition Classes
- TreeNode
-
final
def
notify(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
-
final
def
notifyAll(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
-
def
numberedTreeString: String
- Definition Classes
- TreeNode
-
val
origin: Origin
- Definition Classes
- TreeNode
-
def
otherCopyArgs: Seq[AnyRef]
- Definition Classes
- GpuCachedDoublePassWindowExec → TreeNode
-
def
output: Seq[Attribute]
- Definition Classes
- GpuWindowBaseExec → QueryPlan
-
val
outputBatchesLevel: MetricsLevel
- Attributes
- protected
- Definition Classes
- GpuExec
-
def
outputBatching: CoalesceGoal
Lets a SparkPlan indicate what guarantees, if any, its output batch has.
Lets a SparkPlan indicate what guarantees, if any, its output batch has. This lets us bypass GpuCoalesceBatch calls where ever possible. Returning a null indicates no guarantee at all, which is the default.
- Definition Classes
- GpuExec
-
def
outputOrdering: Seq[SortOrder]
- Definition Classes
- GpuWindowBaseExec → SparkPlan
-
def
outputPartitioning: Partitioning
- Definition Classes
- GpuWindowBaseExec → SparkPlan
-
val
outputRowsLevel: MetricsLevel
- Attributes
- protected
- Definition Classes
- GpuExec
-
lazy val
outputSet: AttributeSet
- Definition Classes
- QueryPlan
- Annotations
- @transient()
-
def
p(number: Int): SparkPlan
- Definition Classes
- TreeNode
-
final
def
prepare(): Unit
- Definition Classes
- SparkPlan
-
def
prepareSubqueries(): Unit
- Attributes
- protected
- Definition Classes
- SparkPlan
-
def
prettyJson: String
- Definition Classes
- TreeNode
-
def
printSchema(): Unit
- Definition Classes
- QueryPlan
-
def
producedAttributes: AttributeSet
- Definition Classes
- QueryPlan
-
lazy val
references: AttributeSet
- Definition Classes
- QueryPlan
- Annotations
- @transient()
-
def
requiredChildDistribution: Seq[Distribution]
- Definition Classes
- GpuWindowBaseExec → SparkPlan
-
def
requiredChildOrdering: Seq[Seq[SortOrder]]
- Definition Classes
- GpuWindowBaseExec → SparkPlan
-
def
resetMetrics(): Unit
- Definition Classes
- SparkPlan
-
final
def
sameResult(other: SparkPlan): Boolean
- Definition Classes
- QueryPlan
-
lazy val
schema: StructType
- Definition Classes
- QueryPlan
-
def
schemaString: String
- Definition Classes
- QueryPlan
-
final
def
semanticHash(): Int
- Definition Classes
- QueryPlan
-
def
setLogicalLink(logicalPlan: LogicalPlan): Unit
- Definition Classes
- SparkPlan
-
def
setTagValue[T](tag: TreeNodeTag[T], value: T): Unit
- Definition Classes
- TreeNode
-
def
setTaskMetrics(gpuTaskMetrics: GpuTaskMetrics): Unit
- Definition Classes
- GpuExec
-
def
simpleString(maxFields: Int): String
- Definition Classes
- QueryPlan → TreeNode
-
def
simpleStringWithNodeId(): String
- Definition Classes
- QueryPlan → TreeNode
-
def
sparkContext: SparkContext
- Attributes
- protected
- Definition Classes
- SparkPlan
-
def
sparkSession: SparkSession
- Definition Classes
- GpuExec
-
final
val
sqlContext: SQLContext
- Definition Classes
- SparkPlan
-
def
statePrefix: String
- Attributes
- protected
- Definition Classes
- QueryPlan
-
def
stringArgs: Iterator[Any]
- Attributes
- protected
- Definition Classes
- TreeNode
-
def
subqueries: Seq[SparkPlan]
- Definition Classes
- QueryPlan
-
def
subqueriesAll: Seq[SparkPlan]
- Definition Classes
- QueryPlan
-
def
supportsColumnar: Boolean
- Definition Classes
- GpuExec → SparkPlan
-
final
def
synchronized[T0](arg0: ⇒ T0): T0
- Definition Classes
- AnyRef
-
def
toJSON: String
- Definition Classes
- TreeNode
-
def
toString(): String
- Definition Classes
- TreeNode → AnyRef → Any
-
def
transform(rule: PartialFunction[SparkPlan, SparkPlan]): SparkPlan
- Definition Classes
- TreeNode
-
def
transformAllExpressions(rule: PartialFunction[Expression, Expression]): GpuCachedDoublePassWindowExec.this.type
- Definition Classes
- QueryPlan
-
def
transformDown(rule: PartialFunction[SparkPlan, SparkPlan]): SparkPlan
- Definition Classes
- TreeNode
-
def
transformExpressions(rule: PartialFunction[Expression, Expression]): GpuCachedDoublePassWindowExec.this.type
- Definition Classes
- QueryPlan
-
def
transformExpressionsDown(rule: PartialFunction[Expression, Expression]): GpuCachedDoublePassWindowExec.this.type
- Definition Classes
- QueryPlan
-
def
transformExpressionsUp(rule: PartialFunction[Expression, Expression]): GpuCachedDoublePassWindowExec.this.type
- Definition Classes
- QueryPlan
-
def
transformUp(rule: PartialFunction[SparkPlan, SparkPlan]): SparkPlan
- Definition Classes
- TreeNode
-
def
transformUpWithNewOutput(rule: PartialFunction[SparkPlan, (SparkPlan, Seq[(Attribute, Attribute)])], skipCond: (SparkPlan) ⇒ Boolean, canGetOutput: (SparkPlan) ⇒ Boolean): SparkPlan
- Definition Classes
- QueryPlan
-
def
treeString(append: (String) ⇒ Unit, verbose: Boolean, addSuffix: Boolean, maxFields: Int, printOperatorId: Boolean): Unit
- Definition Classes
- TreeNode
-
final
def
treeString(verbose: Boolean, addSuffix: Boolean, maxFields: Int, printOperatorId: Boolean): String
- Definition Classes
- TreeNode
-
final
def
treeString: String
- Definition Classes
- TreeNode
-
def
unsetTagValue[T](tag: TreeNodeTag[T]): Unit
- Definition Classes
- TreeNode
-
def
vectorTypes: Option[Seq[String]]
- Definition Classes
- SparkPlan
-
def
verboseString(maxFields: Int): String
- Definition Classes
- QueryPlan → TreeNode
-
def
verboseStringWithOperatorId(): String
- Definition Classes
- UnaryExecNode → QueryPlan
-
def
verboseStringWithSuffix(maxFields: Int): String
- Definition Classes
- TreeNode
-
final
def
wait(): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long, arg1: Int): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()
-
def
waitForSubqueries(): Unit
- Attributes
- protected
- Definition Classes
- SparkPlan
-
val
windowOps: Seq[NamedExpression]
- Definition Classes
- GpuCachedDoublePassWindowExec → GpuWindowBaseExec
-
def
withNewChildren(newChildren: Seq[SparkPlan]): SparkPlan
- Definition Classes
- TreeNode