case class PersistedMetadata(tableId: String, deltaCommitVersion: Long, dataSchemaJson: String, partitionSchemaJson: String, sourceMetadataPath: String, tableConfigurations: Option[Map[String, String]] = None, protocolJson: Option[String] = None, previousMetadataSeqNum: Option[Long] = None) extends PartitionAndDataSchema with Product with Serializable
A PersistedMetadata is an entry in Delta streaming source schema log, which can be used to read data files during streaming.
- tableId
Delta table id
- deltaCommitVersion
Delta commit version in which this change is captured. It does not necessarily have to be the commit when there's an actual change, e.g. during initialization. The invariant is that the metadata must be read-compatible with the table snapshot at this version.
- dataSchemaJson
Full schema json
- partitionSchemaJson
Partition schema json
- sourceMetadataPath
The checkpoint path that is unique to each source.
- tableConfigurations
The configurations of the table inside the metadata when the schema change was detected. It is used to correctly create the right file format when we use a particular schema to read. Default to None for backward compatibility.
- protocolJson
JSON of the protocol change if any. Default to None for backward compatibility.
- previousMetadataSeqNum
When defined, it points to the batch ID / seq num for the previous metadata in the log sequence. It is used when we could not reliably tell if the currentBatchId - 1 is indeed the previous schema evolution, e.g. when we are merging consecutive schema changes during the analysis phase and we are appending an extra schema after the merge to the log. Default to None for backward compatibility.
- Alphabetic
- By Inheritance
- PersistedMetadata
- Serializable
- Product
- Equals
- PartitionAndDataSchema
- AnyRef
- Any
- Hide All
- Show All
- Public
- Protected
Instance Constructors
- new PersistedMetadata(tableId: String, deltaCommitVersion: Long, dataSchemaJson: String, partitionSchemaJson: String, sourceMetadataPath: String, tableConfigurations: Option[Map[String, String]] = None, protocolJson: Option[String] = None, previousMetadataSeqNum: Option[Long] = None)
- tableId
Delta table id
- deltaCommitVersion
Delta commit version in which this change is captured. It does not necessarily have to be the commit when there's an actual change, e.g. during initialization. The invariant is that the metadata must be read-compatible with the table snapshot at this version.
- dataSchemaJson
Full schema json
- partitionSchemaJson
Partition schema json
- sourceMetadataPath
The checkpoint path that is unique to each source.
- tableConfigurations
The configurations of the table inside the metadata when the schema change was detected. It is used to correctly create the right file format when we use a particular schema to read. Default to None for backward compatibility.
- protocolJson
JSON of the protocol change if any. Default to None for backward compatibility.
- previousMetadataSeqNum
When defined, it points to the batch ID / seq num for the previous metadata in the log sequence. It is used when we could not reliably tell if the currentBatchId - 1 is indeed the previous schema evolution, e.g. when we are merging consecutive schema changes during the analysis phase and we are appending an extra schema after the merge to the log. Default to None for backward compatibility.
Value Members
- final def !=(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
- final def ##: Int
- Definition Classes
- AnyRef → Any
- final def ==(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
- final def asInstanceOf[T0]: T0
- Definition Classes
- Any
- def clone(): AnyRef
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.CloneNotSupportedException]) @native()
- lazy val dataSchema: StructType
- Definition Classes
- PersistedMetadata → PartitionAndDataSchema
- Annotations
- @JsonIgnore()
- val dataSchemaJson: String
- val deltaCommitVersion: Long
- final def eq(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
- def finalize(): Unit
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.Throwable])
- final def getClass(): Class[_ <: AnyRef]
- Definition Classes
- AnyRef → Any
- Annotations
- @native()
- final def isInstanceOf[T0]: Boolean
- Definition Classes
- Any
- final def ne(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
- final def notify(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
- final def notifyAll(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
- lazy val partitionSchema: StructType
- Definition Classes
- PersistedMetadata → PartitionAndDataSchema
- Annotations
- @JsonIgnore()
- val partitionSchemaJson: String
- val previousMetadataSeqNum: Option[Long]
- def productElementNames: Iterator[String]
- Definition Classes
- Product
- lazy val protocol: Option[Protocol]
- Annotations
- @JsonIgnore()
- val protocolJson: Option[String]
- val sourceMetadataPath: String
- final def synchronized[T0](arg0: => T0): T0
- Definition Classes
- AnyRef
- val tableConfigurations: Option[Map[String, String]]
- val tableId: String
- def validateAgainstSnapshot(snapshot: SnapshotDescriptor): Unit
- final def wait(): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.InterruptedException])
- final def wait(arg0: Long, arg1: Int): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.InterruptedException])
- final def wait(arg0: Long): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.InterruptedException]) @native()