Value Members

1.  final def !=(arg0: Any): Boolean

   Definition Classes

   AnyRef → Any

2.  final def ##(): Int

   Definition Classes

   AnyRef → Any

3.  final def ==(arg0: Any): Boolean

   Definition Classes

   AnyRef → Any

4.  def addShutdownHook(priority: Int, runnable: Runnable): AnyRef

   Add shutdown hook with priority

5.  final def asInstanceOf[T0]: T0

   Definition Classes

   Any

6.  def asNullable(dt: DataType): DataType
7.  def bytesToString(size: Long): String

   Get a human-readable string, e.g.: "4.0 MiB", for a value in bytes.

8.  def classForName[C](className: String, initialize: Boolean = true, noSparkClassLoader: Boolean = false): Class[C]
9.  def cleanupAnyExistingSession(): Unit

   Shuts down and cleans up any existing Spark session

10.  def clone(): AnyRef

    Attributes

    protected[lang]

    Definition Classes

    AnyRef

    Annotations

    @throws( ... ) @native()

11.  def createCodec(conf: SparkConf): CompressionCodec
12.  def createCodec(conf: SparkConf, codecName: String): CompressionCodec
13.  def createSerializerManager(conf: SparkConf): SerializerManager
14.  def dataTypeExistsRecursively(dt: DataType, f: (DataType) ⇒ Boolean): Boolean

    Return true if the provided predicate function returns true for any type node within the datatype tree.

15.  def doExecuteBroadcast[T](child: SparkPlan): Broadcast[T]
16.  final def eq(arg0: AnyRef): Boolean

    Definition Classes

    AnyRef

17.  def equals(arg0: Any): Boolean

    Definition Classes

    AnyRef → Any

18.  def finalize(): Unit

    Attributes

    protected[lang]

    Definition Classes

    AnyRef

    Annotations

    @throws( classOf[java.lang.Throwable] )

19.  def fromAttributes(attrs: Seq[Attribute]): StructType
20.  final def getClass(): Class[_]

    Definition Classes

    AnyRef → Any

    Annotations

    @native()

21.  def getCodecShortName(codecName: String): String
22.  def getCoresInLocalMode(master: String, conf: SparkConf): Int
23.  def getFSBytesReadOnThreadCallback(): () ⇒ Long

    Returns a function that can be called to find Hadoop FileSystem bytes read.

    Returns a function that can be called to find Hadoop FileSystem bytes read. If getFSBytesReadOnThreadCallback is called from thread r at time t, the returned callback will return the bytes read on r since t.

24.  def getSerializerManager(): SerializerManager
25.  def getSimpleName(cls: Class[_]): String

    Get the simple name of a class with fixup for any Scala internal errors

26.  def getSparkConf(spark: SparkSession): SQLConf
27.  def getSparkHadoopUtilConf: Configuration
28.  def getTaskMemoryManager(): TaskMemoryManager
29.  def hashCode(): Int

    Definition Classes

    AnyRef → Any

    Annotations

    @native()

30.  def incBytesRead(inputMetrics: InputMetrics, bytesRead: Long): Unit

    Set the bytes read task input metric

31.  def incInputRecordsRows(inputMetrics: InputMetrics, rows: Long): Unit
32.  def incTaskMetricsDiskBytesSpilled(amountSpilled: Long): Unit

    Increment the task's disk bytes spilled metric.

    Increment the task's disk bytes spilled metric. If the current thread does not correspond to a Spark task then this call does nothing.

    amountSpilled

    amount of memory spilled in bytes

33.  def incTaskMetricsMemoryBytesSpilled(amountSpilled: Long): Unit

    Increment the task's memory bytes spilled metric.

    Increment the task's memory bytes spilled metric. If the current thread does not correspond to a Spark task then this call does nothing.

    amountSpilled

    amount of memory spilled in bytes

34.  def isCompressSpill(conf: SparkConf): Boolean
35.  def isDriver(sparkConf: SparkConf): Boolean
36.  def isDriver(env: SparkEnv): Boolean

    Returns true if called from code running on the Spark driver.

37.  final def isInstanceOf[T0]: Boolean

    Definition Classes

    Any

38.  def isSupportedRelation(mode: BroadcastMode): Boolean
39.  def jsonValue(dataType: DataType): JValue
40.  def makeSparkUpgradeException(version: String, message: String, cause: Throwable): SparkUpgradeException
41.  final def ne(arg0: AnyRef): Boolean

    Definition Classes

    AnyRef

42.  def newBlockManagerId(execId: String, host: String, port: Int, topologyInfo: Option[String] = None): BlockManagerId

    Create a BlockManagerId instance

43.  def newDaemonCachedThreadPool(prefix: String, maxThreadNumber: Int, keepAliveSeconds: Int = 60): ThreadPoolExecutor
44.  def newDaemonSingleThreadScheduledExecutor(threadName: String): ScheduledExecutorService
45.  def newInputMetrics(): InputMetrics

    Return a new InputMetrics instance

46.  final def notify(): Unit

    Definition Classes

    AnyRef

    Annotations

    @native()

47.  final def notifyAll(): Unit

    Definition Classes

    AnyRef

    Annotations

    @native()

48.  def postEvent(sc: SparkContext, sparkEvent: SparkListenerEvent): Unit
49.  def sameType(a: DataType, b: DataType): Boolean

    Check if a and b are the same data type when ignoring nullability (StructField.nullable, ArrayType.containsNull, and MapType.valueContainsNull).

50.  def setExecutorEnv(sc: SparkContext, key: String, value: String): Unit
51.  def setTaskContext(tc: TaskContext): Unit

    Set the task context for the current thread

52.  final def synchronized[T0](arg0: ⇒ T0): T0

    Definition Classes

    AnyRef

53.  def toAttributes(structType: StructType): Seq[Attribute]
54.  def toString(): String

    Definition Classes

    AnyRef → Any

55.  def unionLikeMerge(left: DataType, right: DataType): DataType
56.  def unsetTaskContext(): Unit

    Remove the task context for the current thread

57.  final def wait(): Unit

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

58.  final def wait(arg0: Long, arg1: Int): Unit

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

59.  final def wait(arg0: Long): Unit

    Definition Classes

    AnyRef

    Annotations

    @throws( ... ) @native()