c

org.apache.spark.sql.delta.files

DelayedCommitProtocol

class DelayedCommitProtocol extends FileCommitProtocol with Serializable with Logging

Writes out the files to path and returns a list of them in addedStatuses. Includes special handling for partitioning on CDC_PARTITION_COL for compatibility between enabled and disabled CDC; partitions with a value of false in this column produce no corresponding partitioning directory.

Linear Supertypes
Serializable, FileCommitProtocol, Logging, AnyRef, Any
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. DelayedCommitProtocol
  2. Serializable
  3. FileCommitProtocol
  4. Logging
  5. AnyRef
  6. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. Protected

Instance Constructors

  1. new DelayedCommitProtocol(jobId: String, path: String, randomPrefixLength: Option[Int], subdir: Option[String])

    path

    The base path files will be written

    randomPrefixLength

    The length of random subdir name under 'path' that files been written

    subdir

    The immediate subdir under path; If randomPrefixLength and subdir both exist, file path will be path/subdir/[rand str of randomPrefixLength]/file

Value Members

  1. final def !=(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  2. final def ##: Int
    Definition Classes
    AnyRef → Any
  3. final def ==(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  4. val FILE_NAME_PREFIX: String

    Prefix added in testing mode to all filenames to test special chars that need URL-encoding.

  5. def abortJob(jobContext: JobContext): Unit
    Definition Classes
    DelayedCommitProtocol → FileCommitProtocol
  6. def abortTask(taskContext: TaskAttemptContext): Unit
    Definition Classes
    DelayedCommitProtocol → FileCommitProtocol
  7. var addedFiles: ArrayBuffer[(Map[String, String], String)]
    Attributes
    protected
  8. val addedStatuses: ArrayBuffer[AddFile]
  9. final def asInstanceOf[T0]: T0
    Definition Classes
    Any
  10. def buildActionFromAddedFile(f: (Map[String, String], String), stat: FileStatus, taskContext: TaskAttemptContext): FileAction
    Attributes
    protected
  11. val cdcPartitionFalse: String
    Attributes
    protected
  12. val cdcPartitionTrue: String
    Attributes
    protected
  13. val cdcPartitionTrueRegex: Regex
    Attributes
    protected
  14. val changeFiles: ArrayBuffer[AddCDCFile]
  15. def clone(): AnyRef
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.CloneNotSupportedException]) @native()
  16. def commitJob(jobContext: JobContext, taskCommits: Seq[TaskCommitMessage]): Unit

    Commits a job after the writes succeed.

    Commits a job after the writes succeed. Must be called on the driver. Partitions the written files into AddFiles and AddCDCFiles as these metadata actions are treated differently by TransactionalWrite (i.e. AddFile's may have additional statistics injected)

    Definition Classes
    DelayedCommitProtocol → FileCommitProtocol
  17. def commitTask(taskContext: TaskAttemptContext): TaskCommitMessage
    Definition Classes
    DelayedCommitProtocol → FileCommitProtocol
  18. def deleteWithJob(fs: FileSystem, path: Path, recursive: Boolean): Boolean
    Definition Classes
    FileCommitProtocol
  19. final def eq(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  20. def equals(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef → Any
  21. def finalize(): Unit
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.Throwable])
  22. final def getClass(): Class[_ <: AnyRef]
    Definition Classes
    AnyRef → Any
    Annotations
    @native()
  23. def getFileName(taskContext: TaskAttemptContext, ext: String, partitionValues: Map[String, String]): String
    Attributes
    protected
  24. def hashCode(): Int
    Definition Classes
    AnyRef → Any
    Annotations
    @native()
  25. def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean
    Attributes
    protected
    Definition Classes
    Logging
  26. def initializeLogIfNecessary(isInterpreter: Boolean): Unit
    Attributes
    protected
    Definition Classes
    Logging
  27. final def isInstanceOf[T0]: Boolean
    Definition Classes
    Any
  28. def isTraceEnabled(): Boolean
    Attributes
    protected
    Definition Classes
    Logging
  29. def log: Logger
    Attributes
    protected
    Definition Classes
    Logging
  30. def logDebug(msg: => String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  31. def logDebug(msg: => String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  32. def logError(msg: => String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  33. def logError(msg: => String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  34. def logInfo(msg: => String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  35. def logInfo(msg: => String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  36. def logName: String
    Attributes
    protected
    Definition Classes
    Logging
  37. def logTrace(msg: => String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  38. def logTrace(msg: => String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  39. def logWarning(msg: => String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  40. def logWarning(msg: => String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  41. final def ne(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  42. def newTaskTempFile(taskContext: TaskAttemptContext, dir: Option[String], ext: String): String

    Notifies the commit protocol to add a new file, and gets back the full path that should be used.

    Notifies the commit protocol to add a new file, and gets back the full path that should be used.

    Includes special logic for CDC files and paths. Specifically, if the directory dir contains the CDC partition is_cdc=true then - the file name begins with cdc- instead of part- - the directory has the is_cdc=true partition removed and is placed in the _changed_data folder

    Definition Classes
    DelayedCommitProtocol → FileCommitProtocol
  43. def newTaskTempFile(taskContext: TaskAttemptContext, dir: Option[String], spec: FileNameSpec): String
    Definition Classes
    FileCommitProtocol
  44. def newTaskTempFileAbsPath(taskContext: TaskAttemptContext, absoluteDir: String, ext: String): String
    Definition Classes
    DelayedCommitProtocol → FileCommitProtocol
  45. def newTaskTempFileAbsPath(taskContext: TaskAttemptContext, absoluteDir: String, spec: FileNameSpec): String
    Definition Classes
    FileCommitProtocol
  46. final def notify(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native()
  47. final def notifyAll(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native()
  48. def onTaskCommit(taskCommit: TaskCommitMessage): Unit
    Definition Classes
    FileCommitProtocol
  49. def parsePartitions(dir: String, taskContext: TaskAttemptContext): Map[String, String]
    Attributes
    protected
  50. def setupJob(jobContext: JobContext): Unit
    Definition Classes
    DelayedCommitProtocol → FileCommitProtocol
  51. def setupTask(taskContext: TaskAttemptContext): Unit
    Definition Classes
    DelayedCommitProtocol → FileCommitProtocol
  52. final def synchronized[T0](arg0: => T0): T0
    Definition Classes
    AnyRef
  53. val timestampPartitionPattern: String
  54. def toString(): String
    Definition Classes
    AnyRef → Any
  55. final def wait(): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.InterruptedException])
  56. final def wait(arg0: Long, arg1: Int): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.InterruptedException])
  57. final def wait(arg0: Long): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.InterruptedException]) @native()

Inherited from Serializable

Inherited from FileCommitProtocol

Inherited from Logging

Inherited from AnyRef

Inherited from Any

Ungrouped