Package io.debezium.pipeline.source.snapshot.incremental

Class AbstractIncrementalSnapshotChangeEventSource<T extends DataCollectionId>

java.lang.Object

io.debezium.pipeline.source.snapshot.incremental.AbstractIncrementalSnapshotChangeEventSource<T>

All Implemented Interfaces:

IncrementalSnapshotChangeEventSource<T>

Direct Known Subclasses:

SignalBasedIncrementalSnapshotChangeEventSource

@NotThreadSafe
public abstract class AbstractIncrementalSnapshotChangeEventSource<T extends DataCollectionId>
extends Object
implements IncrementalSnapshotChangeEventSource<T>

An incremental snapshot change event source that emits events from a DB log interleaved with snapshot events.

Field Summary

Fields

Modifier and Type	Field	Description
private Clock	clock
private RelationalDatabaseConnectorConfig	connectorConfig
protected IncrementalSnapshotContext<T>	context
private Table	currentTable
private RelationalDatabaseSchema	databaseSchema
private DataChangeEventListener	dataListener
protected EventDispatcher<T>	dispatcher
protected JdbcConnection	jdbcConnection
private static org.slf4j.Logger	LOGGER
private SnapshotProgressListener	progressListener
private long	totalRowsScanned
protected Map<org.apache.kafka.connect.data.Struct,Object[]>	window

Constructor Summary

Constructors

Constructor	Description
AbstractIncrementalSnapshotChangeEventSource(RelationalDatabaseConnectorConfig config, JdbcConnection jdbcConnection, EventDispatcher<T> dispatcher, DatabaseSchema<?> databaseSchema, Clock clock, SnapshotProgressListener progressListener, DataChangeEventListener dataChangeEventListener)

Method Summary

Modifier and Type	Method	Description
void	addDataCollectionNamesToSnapshot(List<String> dataCollectionIds, OffsetContext offsetContext)
protected void	addKeyColumnsToCondition(Table table, StringBuilder sql, String predicate)
private void	addLowerBound(Table table, StringBuilder sql)
protected String	buildChunkQuery(Table table)
protected String	buildMaxPrimaryKeyQuery(Table table)
void	closeWindow(Partition partition, String id, OffsetContext offsetContext)
private void	createDataEventsForTable()	Dispatches the data change events for the records of a single table.
protected void	deduplicateWindow(DataCollectionId dataCollectionId, Object key)
protected abstract void	emitWindowClose()	Update high watermark for the incremental snapshot chunk
protected abstract void	emitWindowOpen()	Update low watermark for the incremental snapshot chunk
protected ChangeRecordEmitter	getChangeRecordEmitter(Partition partition, T dataCollectionId, OffsetContext offsetContext, Object[] row)	Returns a ChangeRecordEmitter producing the change records for the given table row.
private Key.KeyMapper	getKeyMapper()
protected String	getSignalTableName(String dataCollectionId)
private Threads.Timer	getTableScanLogTimer()
private void	incrementTableRowsScanned(long rows)
void	init(OffsetContext offsetContext)
private Object[]	keyFromRow(Object[] row)
private void	nextDataCollection()
protected void	postIncrementalSnapshotCompleted()
protected void	postReadChunk(IncrementalSnapshotContext<T> context)
protected void	preReadChunk(IncrementalSnapshotContext<T> context)
protected void	readChunk()
protected PreparedStatement	readTableChunkStatement(String sql)
protected Table	refreshTableSchema(Table table)
protected void	sendEvent(Partition partition, EventDispatcher<T> dispatcher, OffsetContext offsetContext, Object[] row)
protected void	sendWindowEvents(Partition partition, OffsetContext offsetContext)
protected void	setContext(IncrementalSnapshotContext<T> context)
private void	tableScanCompleted()

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface io.debezium.pipeline.source.snapshot.incremental.IncrementalSnapshotChangeEventSource

processFilteredEvent, processHeartbeat, processMessage, processTransactionCommittedEvent, processTransactionStartedEvent

Field Detail

LOGGER

private static final org.slf4j.Logger LOGGER

connectorConfig

private final RelationalDatabaseConnectorConfig connectorConfig

clock

private final Clock clock

databaseSchema

private final RelationalDatabaseSchema databaseSchema

progressListener

private final SnapshotProgressListener progressListener

dataListener

private final DataChangeEventListener dataListener

totalRowsScanned

private long totalRowsScanned

currentTable

private Table currentTable

dispatcher

protected EventDispatcher<T extends DataCollectionId> dispatcher

context

protected IncrementalSnapshotContext<T extends DataCollectionId> context

jdbcConnection

protected JdbcConnection jdbcConnection

window

protected final Map<org.apache.kafka.connect.data.Struct,Object[]> window

Constructor Detail

AbstractIncrementalSnapshotChangeEventSource

public AbstractIncrementalSnapshotChangeEventSource(RelationalDatabaseConnectorConfig config,
                                                    JdbcConnection jdbcConnection,
                                                    EventDispatcher<T> dispatcher,
                                                    DatabaseSchema<?> databaseSchema,
                                                    Clock clock,
                                                    SnapshotProgressListener progressListener,
                                                    DataChangeEventListener dataChangeEventListener)

Method Detail

closeWindow

public void closeWindow(Partition partition,
                        String id,
                        OffsetContext offsetContext)
                 throws InterruptedException

Specified by:

closeWindow in interface IncrementalSnapshotChangeEventSource<T extends DataCollectionId>

Throws:

InterruptedException

getSignalTableName

protected String getSignalTableName(String dataCollectionId)

sendWindowEvents

protected void sendWindowEvents(Partition partition,
                                OffsetContext offsetContext)
                         throws InterruptedException

Throws:

InterruptedException

sendEvent

protected void sendEvent(Partition partition,
                         EventDispatcher<T> dispatcher,
                         OffsetContext offsetContext,
                         Object[] row)
                  throws InterruptedException

Throws:

InterruptedException

getChangeRecordEmitter

protected ChangeRecordEmitter getChangeRecordEmitter(Partition partition,
                                                     T dataCollectionId,
                                                     OffsetContext offsetContext,
                                                     Object[] row)

Returns a ChangeRecordEmitter producing the change records for the given table row.

deduplicateWindow

protected void deduplicateWindow(DataCollectionId dataCollectionId,
                                 Object key)

emitWindowOpen

protected abstract void emitWindowOpen()
                                throws SQLException

Update low watermark for the incremental snapshot chunk

Throws:

SQLException

emitWindowClose

protected abstract void emitWindowClose()
                                 throws SQLException,
                                        InterruptedException

Update high watermark for the incremental snapshot chunk

Throws:

SQLException

InterruptedException

buildChunkQuery

protected String buildChunkQuery(Table table)

addLowerBound

private void addLowerBound(Table table,
                           StringBuilder sql)

buildMaxPrimaryKeyQuery

protected String buildMaxPrimaryKeyQuery(Table table)

init

public void init(OffsetContext offsetContext)

Specified by:

init in interface IncrementalSnapshotChangeEventSource<T extends DataCollectionId>

readChunk

protected void readChunk()
                  throws InterruptedException

Throws:

InterruptedException

nextDataCollection

private void nextDataCollection()

addDataCollectionNamesToSnapshot

public void addDataCollectionNamesToSnapshot(List<String> dataCollectionIds,
                                             OffsetContext offsetContext)
                                      throws InterruptedException

Specified by:

addDataCollectionNamesToSnapshot in interface IncrementalSnapshotChangeEventSource<T extends DataCollectionId>

Throws:

InterruptedException

addKeyColumnsToCondition

protected void addKeyColumnsToCondition(Table table,
                                        StringBuilder sql,
                                        String predicate)

createDataEventsForTable

private void createDataEventsForTable()

Dispatches the data change events for the records of a single table.

incrementTableRowsScanned

private void incrementTableRowsScanned(long rows)

tableScanCompleted

private void tableScanCompleted()

readTableChunkStatement

protected PreparedStatement readTableChunkStatement(String sql)
                                             throws SQLException

Throws:

SQLException

getTableScanLogTimer

private Threads.Timer getTableScanLogTimer()

keyFromRow

private Object[] keyFromRow(Object[] row)

setContext

protected void setContext(IncrementalSnapshotContext<T> context)

preReadChunk

protected void preReadChunk(IncrementalSnapshotContext<T> context)

postReadChunk

protected void postReadChunk(IncrementalSnapshotContext<T> context)

postIncrementalSnapshotCompleted

protected void postIncrementalSnapshotCompleted()

refreshTableSchema

protected Table refreshTableSchema(Table table)
                            throws SQLException

Throws:

SQLException

getKeyMapper

private Key.KeyMapper getKeyMapper()