c

org.apache.spark.sql.delta.files

DelayedCommitProtocol

class DelayedCommitProtocol extends FileCommitProtocol with Serializable with Logging

Writes out the files to path and returns a list of them in addedStatuses. Includes special handling for partitioning on CDC_PARTITION_COL for compatibility between enabled and disabled CDC; partitions with a value of false in this column produce no corresponding partitioning directory.

Linear Supertypes
Serializable, Serializable, FileCommitProtocol, Logging, AnyRef, Any
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. DelayedCommitProtocol
  2. Serializable
  3. Serializable
  4. FileCommitProtocol
  5. Logging
  6. AnyRef
  7. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. All

Instance Constructors

  1. new DelayedCommitProtocol(jobId: String, path: String, randomPrefixLength: Option[Int], subdir: Option[String])

    path

    The base path files will be written

    randomPrefixLength

    The length of random subdir name under 'path' that files been written

    subdir

    The immediate subdir under path; If randomPrefixLength and subdir both exist, file path will be path/subdir/[rand str of randomPrefixLength]/file

Value Members

  1. final def !=(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  2. final def ##(): Int
    Definition Classes
    AnyRef → Any
  3. final def ==(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  4. def abortJob(jobContext: JobContext): Unit
    Definition Classes
    DelayedCommitProtocol → FileCommitProtocol
  5. def abortTask(taskContext: TaskAttemptContext): Unit
    Definition Classes
    DelayedCommitProtocol → FileCommitProtocol
  6. var addedFiles: ArrayBuffer[(Map[String, String], String)]
    Attributes
    protected
  7. val addedStatuses: ArrayBuffer[AddFile]
  8. final def asInstanceOf[T0]: T0
    Definition Classes
    Any
  9. def buildActionFromAddedFile(f: (Map[String, String], String), stat: FileStatus, taskContext: TaskAttemptContext): FileAction
    Attributes
    protected
  10. val cdcPartitionFalse: String
    Attributes
    protected
  11. val cdcPartitionTrue: String
    Attributes
    protected
  12. val cdcPartitionTrueRegex: Regex
    Attributes
    protected
  13. val changeFiles: ArrayBuffer[AddCDCFile]
  14. def clone(): AnyRef
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native()
  15. def commitJob(jobContext: JobContext, taskCommits: Seq[TaskCommitMessage]): Unit

    Commits a job after the writes succeed.

    Commits a job after the writes succeed. Must be called on the driver. Partitions the written files into AddFiles and AddCDCFiles as these metadata actions are treated differently by TransactionalWrite (i.e. AddFile's may have additional statistics injected)

    Definition Classes
    DelayedCommitProtocol → FileCommitProtocol
  16. def commitTask(taskContext: TaskAttemptContext): TaskCommitMessage
    Definition Classes
    DelayedCommitProtocol → FileCommitProtocol
  17. def deleteWithJob(fs: FileSystem, path: Path, recursive: Boolean): Boolean
    Definition Classes
    FileCommitProtocol
  18. final def eq(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  19. def equals(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  20. def finalize(): Unit
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( classOf[java.lang.Throwable] )
  21. final def getClass(): Class[_]
    Definition Classes
    AnyRef → Any
    Annotations
    @native()
  22. def getFileName(taskContext: TaskAttemptContext, ext: String, partitionValues: Map[String, String]): String
    Attributes
    protected
  23. def hashCode(): Int
    Definition Classes
    AnyRef → Any
    Annotations
    @native()
  24. def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean
    Attributes
    protected
    Definition Classes
    Logging
  25. def initializeLogIfNecessary(isInterpreter: Boolean): Unit
    Attributes
    protected
    Definition Classes
    Logging
  26. final def isInstanceOf[T0]: Boolean
    Definition Classes
    Any
  27. def isTraceEnabled(): Boolean
    Attributes
    protected
    Definition Classes
    Logging
  28. def log: Logger
    Attributes
    protected
    Definition Classes
    Logging
  29. def logDebug(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  30. def logDebug(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  31. def logError(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  32. def logError(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  33. def logInfo(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  34. def logInfo(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  35. def logName: String
    Attributes
    protected
    Definition Classes
    Logging
  36. def logTrace(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  37. def logTrace(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  38. def logWarning(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  39. def logWarning(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  40. final def ne(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  41. def newTaskTempFile(taskContext: TaskAttemptContext, dir: Option[String], ext: String): String

    Notifies the commit protocol to add a new file, and gets back the full path that should be used.

    Notifies the commit protocol to add a new file, and gets back the full path that should be used.

    Includes special logic for CDC files and paths. Specifically, if the directory dir contains the CDC partition is_cdc=true then - the file name begins with cdc- instead of part- - the directory has the is_cdc=true partition removed and is placed in the _changed_data folder

    Definition Classes
    DelayedCommitProtocol → FileCommitProtocol
  42. def newTaskTempFile(taskContext: TaskAttemptContext, dir: Option[String], spec: FileNameSpec): String
    Definition Classes
    FileCommitProtocol
  43. def newTaskTempFileAbsPath(taskContext: TaskAttemptContext, absoluteDir: String, ext: String): String
    Definition Classes
    DelayedCommitProtocol → FileCommitProtocol
  44. def newTaskTempFileAbsPath(taskContext: TaskAttemptContext, absoluteDir: String, spec: FileNameSpec): String
    Definition Classes
    FileCommitProtocol
  45. final def notify(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native()
  46. final def notifyAll(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native()
  47. def onTaskCommit(taskCommit: TaskCommitMessage): Unit
    Definition Classes
    FileCommitProtocol
  48. def parsePartitions(dir: String): Map[String, String]
    Attributes
    protected
  49. def setupJob(jobContext: JobContext): Unit
    Definition Classes
    DelayedCommitProtocol → FileCommitProtocol
  50. def setupTask(taskContext: TaskAttemptContext): Unit
    Definition Classes
    DelayedCommitProtocol → FileCommitProtocol
  51. final def synchronized[T0](arg0: ⇒ T0): T0
    Definition Classes
    AnyRef
  52. val timestampPartitionPattern: String
  53. def toString(): String
    Definition Classes
    AnyRef → Any
  54. final def wait(): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  55. final def wait(arg0: Long, arg1: Int): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  56. final def wait(arg0: Long): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native()

Inherited from Serializable

Inherited from Serializable

Inherited from FileCommitProtocol

Inherited from Logging

Inherited from AnyRef

Inherited from Any

Ungrouped