t

org.apache.spark.sql.delta.sources

DeltaSourceCDCSupport

trait DeltaSourceCDCSupport extends AnyRef

Helper functions for CDC-specific handling for DeltaSource.

Self Type
DeltaSource
Linear Supertypes
AnyRef, Any
Known Subclasses
DeltaSource
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. DeltaSourceCDCSupport
  2. AnyRef
  3. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. All

Type Members

  1. class IndexedChangeFileSeq extends AnyRef

    This class represents an iterator of Change metadata(AddFile, RemoveFile, AddCDCFile) for a particular version.

Value Members

  1. final def !=(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  2. final def ##(): Int
    Definition Classes
    AnyRef → Any
  3. final def ==(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  4. final def asInstanceOf[T0]: T0
    Definition Classes
    Any
  5. def clone(): AnyRef
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native()
  6. final def eq(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  7. def equals(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  8. def finalize(): Unit
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( classOf[java.lang.Throwable] )
  9. def getCDCFileChangesAndCreateDataFrame(startVersion: Long, startIndex: Long, isInitialSnapshot: Boolean, endOffset: DeltaSourceOffset): DataFrame

    Get the changes from startVersion, startIndex to the end for CDC case.

    Get the changes from startVersion, startIndex to the end for CDC case. We need to call CDCReader to get the CDC DataFrame.

    startVersion

    - calculated starting version

    startIndex

    - calculated starting index

    isInitialSnapshot

    - whether the stream has to return the initial snapshot or not

    endOffset

    - Offset that signifies the end of the stream.

    returns

    the DataFrame containing the file changes (AddFile, RemoveFile, AddCDCFile)

    Attributes
    protected
  10. final def getClass(): Class[_]
    Definition Classes
    AnyRef → Any
    Annotations
    @native()
  11. def getFileChangesForCDC(fromVersion: Long, fromIndex: Long, isInitialSnapshot: Boolean, limits: Option[AdmissionLimits], endOffset: Option[DeltaSourceOffset], verifyMetadataAction: Boolean = true): Iterator[(Long, Iterator[IndexedFile], Option[CommitInfo])]

    Get the changes starting from (fromVersion, fromIndex).

    Get the changes starting from (fromVersion, fromIndex). fromVersion is included. It returns an iterator of (log_version, fileActions, Optional[CommitInfo]). The commit info is needed later on so that the InCommitTimestamp of the log files can be determined.

    If verifyMetadataAction = true, we will break the stream when we detect any read-incompatible metadata changes.

    Attributes
    protected
  12. def hashCode(): Int
    Definition Classes
    AnyRef → Any
    Annotations
    @native()
  13. final def isInstanceOf[T0]: Boolean
    Definition Classes
    Any
  14. final def ne(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  15. final def notify(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native()
  16. final def notifyAll(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native()
  17. final def synchronized[T0](arg0: ⇒ T0): T0
    Definition Classes
    AnyRef
  18. def toString(): String
    Definition Classes
    AnyRef → Any
  19. final def wait(): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  20. final def wait(arg0: Long, arg1: Int): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  21. final def wait(arg0: Long): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native()

Inherited from AnyRef

Inherited from Any

Ungrouped