case class LastCheckpointInfo(version: Long, size: Long, parts: Option[Int], sizeInBytes: Option[Long], numOfAddFiles: Option[Long], checkpointSchema: Option[StructType], v2Checkpoint: Option[LastCheckpointV2] = None, checksum: Option[String] = None) extends Product with Serializable
Records information about a checkpoint.
This class provides the checksum validation logic, needed to ensure that content of LAST_CHECKPOINT file points to a valid json. The readers might read some part from old file and some part from the new file (if the file is read across multiple requests). In some rare scenarios, the split read might produce a valid json and readers will be able to parse it and convert it into a LastCheckpointInfo object that contains invalid data. In order to prevent using it, we do a checksum match on the read json to validate that it is consistent.
For old Delta versions, which do not have checksum logic, we want to make sure that the old
fields (i.e. version, size, parts) are together in the beginning of last_checkpoint json. All
these fields together are less than 50 bytes, so even in split read scenario, we want to make
sure that old delta readers which do not do have checksum validation logic, gets all 3 fields
from one read request. For this reason, we use JsonPropertyOrder to force them in the beginning
together.
- version
the version of this checkpoint
- size
the number of actions in the checkpoint, -1 if the information is unavailable.
- parts
the number of parts when the checkpoint has multiple parts. None if this is a singular checkpoint
- sizeInBytes
the number of bytes of the checkpoint
- numOfAddFiles
the number of AddFile actions in the checkpoint
- checkpointSchema
the schema of the underlying checkpoint files
- checksum
the checksum of the LastCheckpointInfo.
- Annotations
- @JsonPropertyOrder()
- Alphabetic
- By Inheritance
- LastCheckpointInfo
- Serializable
- Serializable
- Product
- Equals
- AnyRef
- Any
- Hide All
- Show All
- Public
- All
Instance Constructors
-
new
LastCheckpointInfo(version: Long, size: Long, parts: Option[Int], sizeInBytes: Option[Long], numOfAddFiles: Option[Long], checkpointSchema: Option[StructType], v2Checkpoint: Option[LastCheckpointV2] = None, checksum: Option[String] = None)
- version
the version of this checkpoint
- size
the number of actions in the checkpoint, -1 if the information is unavailable.
- parts
the number of parts when the checkpoint has multiple parts. None if this is a singular checkpoint
- sizeInBytes
the number of bytes of the checkpoint
- numOfAddFiles
the number of AddFile actions in the checkpoint
- checkpointSchema
the schema of the underlying checkpoint files
- checksum
the checksum of the LastCheckpointInfo.
Value Members
-
final
def
!=(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
final
def
##(): Int
- Definition Classes
- AnyRef → Any
-
final
def
==(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
final
def
asInstanceOf[T0]: T0
- Definition Classes
- Any
- val checkpointSchema: Option[StructType]
- val checksum: Option[String]
-
def
clone(): AnyRef
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()
-
final
def
eq(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
def
finalize(): Unit
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( classOf[java.lang.Throwable] )
-
final
def
getClass(): Class[_]
- Definition Classes
- AnyRef → Any
- Annotations
- @native()
-
def
getFormatEnum(): Format
- Annotations
- @JsonIgnore()
-
final
def
isInstanceOf[T0]: Boolean
- Definition Classes
- Any
-
final
def
ne(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
final
def
notify(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
-
final
def
notifyAll(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
- val numOfAddFiles: Option[Long]
- val parts: Option[Int]
-
def
semanticEquals(other: LastCheckpointInfo): Boolean
Whether two LastCheckpointInfo represents the same checkpoint
- val size: Long
- val sizeInBytes: Option[Long]
-
final
def
synchronized[T0](arg0: ⇒ T0): T0
- Definition Classes
- AnyRef
- val v2Checkpoint: Option[LastCheckpointV2]
- val version: Long
-
final
def
wait(): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long, arg1: Int): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()