case class GpuShuffledSymmetricHashJoinExec(joinType: JoinType, leftKeys: Seq[Expression], rightKeys: Seq[Expression], condition: Option[Expression], left: SparkPlan, right: SparkPlan, isGpuShuffle: Boolean, gpuBatchSizeBytes: Long, isSkewJoin: Boolean)(cpuLeftKeys: Seq[Expression], cpuRightKeys: Seq[Expression]) extends SparkPlan with GpuJoinExec with Product with Serializable
A GPU shuffled hash join optimized to handle symmetric joins like inner and full outer. Probes the sizes of the input tables before performing the join to determine which to use as the build side.
- leftKeys
join keys for the left table
- rightKeys
join keys for the right table
- condition
inequality portions of the join condition
- left
plan for the left table
- right
plan for the right table
- isGpuShuffle
whether the shuffle is GPU-centric (e.g.: UCX-based)
- gpuBatchSizeBytes
target GPU batch size
- cpuLeftKeys
original CPU expressions for the left join keys
- cpuRightKeys
original CPU expressions for the right join keys
Linear Supertypes
Ordering
- Alphabetic
- By Inheritance
Inherited
- GpuShuffledSymmetricHashJoinExec
- GpuJoinExec
- GpuExec
- ShimBinaryExecNode
- BinaryExecNode
- SparkPlan
- Serializable
- Serializable
- Logging
- QueryPlan
- SQLConfHelper
- TreeNode
- Product
- Equals
- AnyRef
- Any
- Hide All
- Show All
Visibility
- Public
- All
Instance Constructors
-
new
GpuShuffledSymmetricHashJoinExec(joinType: JoinType, leftKeys: Seq[Expression], rightKeys: Seq[Expression], condition: Option[Expression], left: SparkPlan, right: SparkPlan, isGpuShuffle: Boolean, gpuBatchSizeBytes: Long, isSkewJoin: Boolean)(cpuLeftKeys: Seq[Expression], cpuRightKeys: Seq[Expression])
- leftKeys
join keys for the left table
- rightKeys
join keys for the right table
- condition
inequality portions of the join condition
- left
plan for the left table
- right
plan for the right table
- isGpuShuffle
whether the shuffle is GPU-centric (e.g.: UCX-based)
- gpuBatchSizeBytes
target GPU batch size
- cpuLeftKeys
original CPU expressions for the left join keys
- cpuRightKeys
original CPU expressions for the right join keys
Value Members
-
final
def
!=(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
final
def
##(): Int
- Definition Classes
- AnyRef → Any
-
final
def
==(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
lazy val
additionalMetrics: Map[String, GpuMetric]
- Definition Classes
- GpuShuffledSymmetricHashJoinExec → GpuExec
-
lazy val
allAttributes: AttributeSeq
- Definition Classes
- QueryPlan
-
lazy val
allMetrics: Map[String, GpuMetric]
- Definition Classes
- GpuExec
-
def
apply(number: Int): TreeNode[_]
- Definition Classes
- TreeNode
-
def
argString(maxFields: Int): String
- Definition Classes
- TreeNode
-
def
asCode: String
- Definition Classes
- TreeNode
-
final
def
asInstanceOf[T0]: T0
- Definition Classes
- Any
-
final
lazy val
canonicalized: SparkPlan
- Definition Classes
- QueryPlan
- Annotations
- @transient()
-
final
def
children: Seq[SparkPlan]
- Definition Classes
- BinaryExecNode → TreeNode
-
def
childrenCoalesceGoal: Seq[CoalesceGoal]
A goal to coalesce batches as the input to this operation.
A goal to coalesce batches as the input to this operation. In some cases an operation will only work if all of the data is in a single batch. In other cases it may be much faster if it is in a single batch, but can tolerate multiple batches. This provides a way to express those desires.
- Definition Classes
- GpuExec
-
def
cleanupResources(): Unit
- Attributes
- protected[sql]
- Definition Classes
- SparkPlan
-
def
clone(): SparkPlan
- Definition Classes
- TreeNode → AnyRef
-
def
coalesceAfter: Boolean
If true is returned batches after this will be coalesced.
If true is returned batches after this will be coalesced. This should really be used in cases where it is known that the size of a batch may shrink a lot.
- Definition Classes
- GpuExec
-
def
collect[B](pf: PartialFunction[SparkPlan, B]): Seq[B]
- Definition Classes
- TreeNode
-
def
collectFirst[B](pf: PartialFunction[SparkPlan, B]): Option[B]
- Definition Classes
- TreeNode
-
def
collectLeaves(): Seq[SparkPlan]
- Definition Classes
- TreeNode
-
def
collectWithSubqueries[B](f: PartialFunction[SparkPlan, B]): Seq[B]
- Definition Classes
- QueryPlan
-
val
condition: Option[Expression]
- Definition Classes
- GpuShuffledSymmetricHashJoinExec → GpuJoinExec
-
def
conf: SQLConf
- Definition Classes
- SQLConfHelper
-
lazy val
containsChild: Set[TreeNode[_]]
- Definition Classes
- TreeNode
-
def
copyTagsFrom(other: SparkPlan): Unit
- Definition Classes
- TreeNode
-
def
createAverageMetric(level: MetricsLevel, name: String): GpuMetric
- Attributes
- protected
- Definition Classes
- GpuExec
-
def
createFileCacheMetrics(): Map[String, GpuMetric]
- Attributes
- protected
- Definition Classes
- GpuExec
-
def
createMetric(level: MetricsLevel, name: String): GpuMetric
- Attributes
- protected
- Definition Classes
- GpuExec
-
def
createNanoTimingMetric(level: MetricsLevel, name: String): GpuMetric
- Attributes
- protected
- Definition Classes
- GpuExec
-
def
createSizeMetric(level: MetricsLevel, name: String): GpuMetric
- Attributes
- protected
- Definition Classes
- GpuExec
-
def
createTimingMetric(level: MetricsLevel, name: String): GpuMetric
- Attributes
- protected
- Definition Classes
- GpuExec
-
def
disableCoalesceUntilInput(): Boolean
Returns true if there is something in the exec that cannot work when batches between multiple file partitions are combined into a single batch (coalesce).
Returns true if there is something in the exec that cannot work when batches between multiple file partitions are combined into a single batch (coalesce).
- Definition Classes
- GpuExec
-
def
doCanonicalize(): SparkPlan
Defines how the canonicalization should work for the current plan.
Defines how the canonicalization should work for the current plan.
- Attributes
- protected
- Definition Classes
- GpuExec → QueryPlan
-
def
doExecute(): RDD[InternalRow]
- Definition Classes
- GpuShuffledSymmetricHashJoinExec → SparkPlan
-
def
doExecuteBroadcast[T](): Broadcast[T]
- Attributes
- protected[sql]
- Definition Classes
- SparkPlan
-
final
def
doExecuteColumnar(): RDD[ColumnarBatch]
- Definition Classes
- GpuExec → SparkPlan
-
def
doPrepare(): Unit
- Attributes
- protected
- Definition Classes
- SparkPlan
-
final
def
eq(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
final
def
execute(): RDD[InternalRow]
- Definition Classes
- SparkPlan
-
final
def
executeBroadcast[T](): Broadcast[T]
- Definition Classes
- SparkPlan
-
def
executeCollect(): Array[InternalRow]
- Definition Classes
- SparkPlan
-
def
executeCollectPublic(): Array[Row]
- Definition Classes
- SparkPlan
-
final
def
executeColumnar(): RDD[ColumnarBatch]
- Definition Classes
- SparkPlan
-
final
def
executeQuery[T](query: ⇒ T): T
- Attributes
- protected
- Definition Classes
- SparkPlan
-
def
executeTail(n: Int): Array[InternalRow]
- Definition Classes
- SparkPlan
-
def
executeTake(n: Int): Array[InternalRow]
- Definition Classes
- SparkPlan
-
def
executeToIterator(): Iterator[InternalRow]
- Definition Classes
- SparkPlan
-
final
def
expressions: Seq[Expression]
- Definition Classes
- QueryPlan
-
def
fastEquals(other: TreeNode[_]): Boolean
- Definition Classes
- TreeNode
-
def
finalize(): Unit
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( classOf[java.lang.Throwable] )
-
def
find(f: (SparkPlan) ⇒ Boolean): Option[SparkPlan]
- Definition Classes
- TreeNode
-
def
flatMap[A](f: (SparkPlan) ⇒ TraversableOnce[A]): Seq[A]
- Definition Classes
- TreeNode
-
def
foreach(f: (SparkPlan) ⇒ Unit): Unit
- Definition Classes
- TreeNode
-
def
foreachUp(f: (SparkPlan) ⇒ Unit): Unit
- Definition Classes
- TreeNode
-
def
formattedNodeName: String
- Attributes
- protected
- Definition Classes
- QueryPlan
-
def
generateTreeString(depth: Int, lastChildren: Seq[Boolean], append: (String) ⇒ Unit, verbose: Boolean, prefix: String, addSuffix: Boolean, maxFields: Int, printNodeId: Boolean, indent: Int): Unit
- Definition Classes
- TreeNode
-
final
def
getClass(): Class[_]
- Definition Classes
- AnyRef → Any
- Annotations
- @native()
-
def
getTagValue[T](tag: TreeNodeTag[T]): Option[T]
- Definition Classes
- TreeNode
-
def
getTaskMetrics: Option[GpuTaskMetrics]
- Definition Classes
- GpuExec
- val gpuBatchSizeBytes: Long
-
def
gpuExpressions: Seq[Expression]
Return the expressions for this plan node that should be GPU expressions.
Return the expressions for this plan node that should be GPU expressions. For most nodes this will be the same as the list of expressions, but some nodes use CPU expressions directly in some cases and will need to override this.
- Definition Classes
- GpuExec
-
def
gpuLongMetric(name: String): GpuMetric
- Definition Classes
- GpuExec
-
def
hashCode(): Int
- Definition Classes
- TreeNode → AnyRef → Any
-
val
id: Int
- Definition Classes
- SparkPlan
-
def
initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean
- Attributes
- protected
- Definition Classes
- Logging
-
def
initializeLogIfNecessary(isInterpreter: Boolean): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
innerChildren: Seq[QueryPlan[_]]
- Definition Classes
- QueryPlan → TreeNode
-
def
inputSet: AttributeSet
- Definition Classes
- QueryPlan
-
def
internalDoExecuteColumnar(): RDD[ColumnarBatch]
- Definition Classes
- GpuShuffledSymmetricHashJoinExec → GpuExec
-
def
isCanonicalizedPlan: Boolean
- Attributes
- protected
- Definition Classes
- QueryPlan
- val isGpuShuffle: Boolean
-
final
def
isInstanceOf[T0]: Boolean
- Definition Classes
- Any
-
val
isSkewJoin: Boolean
- Definition Classes
- GpuShuffledSymmetricHashJoinExec → GpuJoinExec
-
def
isTraceEnabled(): Boolean
- Attributes
- protected
- Definition Classes
- Logging
-
val
joinType: JoinType
- Definition Classes
- GpuShuffledSymmetricHashJoinExec → GpuJoinExec
-
def
jsonFields: List[JField]
- Attributes
- protected
- Definition Classes
- TreeNode
-
val
left: SparkPlan
- Definition Classes
- GpuShuffledSymmetricHashJoinExec → BinaryExecNode
-
val
leftKeys: Seq[Expression]
- Definition Classes
- GpuShuffledSymmetricHashJoinExec → GpuJoinExec
-
def
log: Logger
- Attributes
- protected
- Definition Classes
- Logging
-
def
logDebug(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logDebug(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logError(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logError(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logInfo(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logInfo(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logName: String
- Attributes
- protected
- Definition Classes
- Logging
-
def
logTrace(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logTrace(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logWarning(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logWarning(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logicalLink: Option[LogicalPlan]
- Definition Classes
- SparkPlan
-
def
longMetric(name: String): SQLMetric
- Definition Classes
- SparkPlan
-
def
makeCopy(newArgs: Array[AnyRef]): SparkPlan
- Definition Classes
- SparkPlan → TreeNode
-
def
map[A](f: (SparkPlan) ⇒ A): Seq[A]
- Definition Classes
- TreeNode
-
def
mapChildren(f: (SparkPlan) ⇒ SparkPlan): SparkPlan
- Definition Classes
- TreeNode
-
def
mapExpressions(f: (Expression) ⇒ Expression): GpuShuffledSymmetricHashJoinExec.this.type
- Definition Classes
- QueryPlan
-
def
mapProductIterator[B](f: (Any) ⇒ B)(implicit arg0: ClassTag[B]): Array[B]
- Attributes
- protected
- Definition Classes
- TreeNode
-
final
lazy val
metrics: Map[String, SQLMetric]
- Definition Classes
- GpuExec → SparkPlan
-
final
def
missingInput: AttributeSet
- Definition Classes
- QueryPlan
-
final
def
ne(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
def
nodeName: String
- Definition Classes
- GpuShuffledSymmetricHashJoinExec → TreeNode
-
final
def
notify(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
-
final
def
notifyAll(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
-
def
numberedTreeString: String
- Definition Classes
- TreeNode
-
val
origin: Origin
- Definition Classes
- TreeNode
-
def
otherCopyArgs: Seq[AnyRef]
- Definition Classes
- GpuShuffledSymmetricHashJoinExec → TreeNode
-
def
output: Seq[Attribute]
- Definition Classes
- GpuShuffledSymmetricHashJoinExec → QueryPlan
-
val
outputBatchesLevel: MetricsLevel
- Definition Classes
- GpuShuffledSymmetricHashJoinExec → GpuExec
-
def
outputBatching: CoalesceGoal
Lets a SparkPlan indicate what guarantees, if any, its output batch has.
Lets a SparkPlan indicate what guarantees, if any, its output batch has. This lets us bypass GpuCoalesceBatch calls where ever possible. Returning a null indicates no guarantee at all, which is the default.
- Definition Classes
- GpuExec
-
def
outputOrdering: Seq[SortOrder]
- Definition Classes
- SparkPlan
-
def
outputPartitioning: Partitioning
- Definition Classes
- SparkPlan
-
val
outputRowsLevel: MetricsLevel
- Definition Classes
- GpuShuffledSymmetricHashJoinExec → GpuExec
-
lazy val
outputSet: AttributeSet
- Definition Classes
- QueryPlan
- Annotations
- @transient()
-
def
p(number: Int): SparkPlan
- Definition Classes
- TreeNode
-
final
def
prepare(): Unit
- Definition Classes
- SparkPlan
-
def
prepareSubqueries(): Unit
- Attributes
- protected
- Definition Classes
- SparkPlan
-
def
prettyJson: String
- Definition Classes
- TreeNode
-
def
printSchema(): Unit
- Definition Classes
- QueryPlan
-
def
producedAttributes: AttributeSet
- Definition Classes
- QueryPlan
-
lazy val
references: AttributeSet
- Definition Classes
- QueryPlan
- Annotations
- @transient()
-
def
requiredChildDistribution: Seq[Distribution]
- Definition Classes
- GpuShuffledSymmetricHashJoinExec → SparkPlan
-
def
requiredChildOrdering: Seq[Seq[SortOrder]]
- Definition Classes
- SparkPlan
-
def
resetMetrics(): Unit
- Definition Classes
- SparkPlan
-
val
right: SparkPlan
- Definition Classes
- GpuShuffledSymmetricHashJoinExec → BinaryExecNode
-
val
rightKeys: Seq[Expression]
- Definition Classes
- GpuShuffledSymmetricHashJoinExec → GpuJoinExec
-
final
def
sameResult(other: SparkPlan): Boolean
- Definition Classes
- QueryPlan
-
lazy val
schema: StructType
- Definition Classes
- QueryPlan
-
def
schemaString: String
- Definition Classes
- QueryPlan
-
final
def
semanticHash(): Int
- Definition Classes
- QueryPlan
-
def
setLogicalLink(logicalPlan: LogicalPlan): Unit
- Definition Classes
- SparkPlan
-
def
setTagValue[T](tag: TreeNodeTag[T], value: T): Unit
- Definition Classes
- TreeNode
-
def
setTaskMetrics(gpuTaskMetrics: GpuTaskMetrics): Unit
- Definition Classes
- GpuExec
-
def
simpleString(maxFields: Int): String
- Definition Classes
- QueryPlan → TreeNode
-
def
simpleStringWithNodeId(): String
- Definition Classes
- QueryPlan → TreeNode
-
def
sparkContext: SparkContext
- Attributes
- protected
- Definition Classes
- SparkPlan
-
def
sparkSession: SparkSession
- Definition Classes
- GpuExec
-
final
val
sqlContext: SQLContext
- Definition Classes
- SparkPlan
-
def
statePrefix: String
- Attributes
- protected
- Definition Classes
- QueryPlan
-
def
stringArgs: Iterator[Any]
- Attributes
- protected
- Definition Classes
- TreeNode
-
def
subqueries: Seq[SparkPlan]
- Definition Classes
- QueryPlan
-
def
subqueriesAll: Seq[SparkPlan]
- Definition Classes
- QueryPlan
-
def
supportsColumnar: Boolean
- Definition Classes
- GpuExec → SparkPlan
-
final
def
synchronized[T0](arg0: ⇒ T0): T0
- Definition Classes
- AnyRef
-
def
toJSON: String
- Definition Classes
- TreeNode
-
def
toString(): String
- Definition Classes
- TreeNode → AnyRef → Any
-
def
transform(rule: PartialFunction[SparkPlan, SparkPlan]): SparkPlan
- Definition Classes
- TreeNode
-
def
transformAllExpressions(rule: PartialFunction[Expression, Expression]): GpuShuffledSymmetricHashJoinExec.this.type
- Definition Classes
- QueryPlan
-
def
transformDown(rule: PartialFunction[SparkPlan, SparkPlan]): SparkPlan
- Definition Classes
- TreeNode
-
def
transformExpressions(rule: PartialFunction[Expression, Expression]): GpuShuffledSymmetricHashJoinExec.this.type
- Definition Classes
- QueryPlan
-
def
transformExpressionsDown(rule: PartialFunction[Expression, Expression]): GpuShuffledSymmetricHashJoinExec.this.type
- Definition Classes
- QueryPlan
-
def
transformExpressionsUp(rule: PartialFunction[Expression, Expression]): GpuShuffledSymmetricHashJoinExec.this.type
- Definition Classes
- QueryPlan
-
def
transformUp(rule: PartialFunction[SparkPlan, SparkPlan]): SparkPlan
- Definition Classes
- TreeNode
-
def
transformUpWithNewOutput(rule: PartialFunction[SparkPlan, (SparkPlan, Seq[(Attribute, Attribute)])], skipCond: (SparkPlan) ⇒ Boolean, canGetOutput: (SparkPlan) ⇒ Boolean): SparkPlan
- Definition Classes
- QueryPlan
-
def
treeString(append: (String) ⇒ Unit, verbose: Boolean, addSuffix: Boolean, maxFields: Int, printOperatorId: Boolean): Unit
- Definition Classes
- TreeNode
-
final
def
treeString(verbose: Boolean, addSuffix: Boolean, maxFields: Int, printOperatorId: Boolean): String
- Definition Classes
- TreeNode
-
final
def
treeString: String
- Definition Classes
- TreeNode
-
def
unsetTagValue[T](tag: TreeNodeTag[T]): Unit
- Definition Classes
- TreeNode
-
def
vectorTypes: Option[Seq[String]]
- Definition Classes
- SparkPlan
-
def
verboseString(maxFields: Int): String
- Definition Classes
- QueryPlan → TreeNode
-
def
verboseStringWithOperatorId(): String
- Definition Classes
- BinaryExecNode → QueryPlan
-
def
verboseStringWithSuffix(maxFields: Int): String
- Definition Classes
- TreeNode
-
final
def
wait(): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long, arg1: Int): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()
-
def
waitForSubqueries(): Unit
- Attributes
- protected
- Definition Classes
- SparkPlan
-
def
withNewChildren(newChildren: Seq[SparkPlan]): SparkPlan
- Definition Classes
- TreeNode