c

org.apache.spark.sql.delta.stats

DeltaTaskStatisticsTracker

class DeltaTaskStatisticsTracker extends WriteTaskStatsTracker

A per-task (i.e. one instance per executor) WriteTaskStatsTracker that collects the statistics defined by StatisticsCollection for files that are being written into a delta table.

Linear Supertypes
WriteTaskStatsTracker, AnyRef, Any
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. DeltaTaskStatisticsTracker
  2. WriteTaskStatsTracker
  3. AnyRef
  4. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. Protected

Instance Constructors

  1. new DeltaTaskStatisticsTracker(dataCols: Seq[Attribute], statsColExpr: Expression, rootPath: Path, hadoopConf: Configuration)

    dataCols

    Resolved data (i.e. non-partitionBy) columns of the dataframe to be written.

    statsColExpr

    Resolved expression for computing all the statistics that we want to gather.

    rootPath

    The Reservoir's root path.

    hadoopConf

    Hadoop Config for being able to instantiate a FileSystem.

Value Members

  1. final def !=(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  2. final def ##: Int
    Definition Classes
    AnyRef → Any
  3. final def ==(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  4. val aggBufferAttrs: Seq[Attribute]
    Attributes
    protected
  5. final def asInstanceOf[T0]: T0
    Definition Classes
    Any
  6. def clone(): AnyRef
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.CloneNotSupportedException]) @native()
  7. def closeFile(filePath: String): Unit
    Definition Classes
    DeltaTaskStatisticsTracker → WriteTaskStatsTracker
  8. final def eq(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  9. def equals(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef → Any
  10. def finalize(): Unit
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.Throwable])
  11. final def getClass(): Class[_ <: AnyRef]
    Definition Classes
    AnyRef → Any
    Annotations
    @native()
  12. def getFinalStats(taskCommitTime: Long): DeltaFileStatistics
    Definition Classes
    DeltaTaskStatisticsTracker → WriteTaskStatsTracker
  13. def hashCode(): Int
    Definition Classes
    AnyRef → Any
    Annotations
    @native()
  14. def initializeAggBuf(buffer: SpecificInternalRow): InternalRow
    Attributes
    protected
  15. val initializeStats: MutableProjection
    Attributes
    protected
  16. final def isInstanceOf[T0]: Boolean
    Definition Classes
    Any
  17. final def ne(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  18. def newFile(newFilePath: String): Unit
    Definition Classes
    DeltaTaskStatisticsTracker → WriteTaskStatsTracker
  19. def newPartition(partitionValues: InternalRow): Unit
    Definition Classes
    DeltaTaskStatisticsTracker → WriteTaskStatsTracker
  20. def newRow(filePath: String, currentRow: InternalRow): Unit
    Definition Classes
    DeltaTaskStatisticsTracker → WriteTaskStatsTracker
  21. final def notify(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native()
  22. final def notifyAll(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native()
  23. val results: HashMap[String, String]
    Attributes
    protected
  24. val submittedFiles: HashMap[String, InternalRow]
    Attributes
    protected[this]
  25. final def synchronized[T0](arg0: => T0): T0
    Definition Classes
    AnyRef
  26. def toString(): String
    Definition Classes
    AnyRef → Any
  27. final def wait(): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.InterruptedException])
  28. final def wait(arg0: Long, arg1: Int): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.InterruptedException])
  29. final def wait(arg0: Long): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.InterruptedException]) @native()

Inherited from WriteTaskStatsTracker

Inherited from AnyRef

Inherited from Any

Ungrouped