org.apache.nifi.processor.util.bin

Class BinFiles

java.lang.Object

org.apache.nifi.components.AbstractConfigurableComponent

org.apache.nifi.processor.AbstractSessionFactoryProcessor

org.apache.nifi.processor.util.bin.BinFiles

All Implemented Interfaces:

ConfigurableComponent, Processor

public abstract class BinFiles
extends AbstractSessionFactoryProcessor

Base class for file-binning processors.

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
private static class	BinFiles.BinningResult

Field Summary

Fields

Modifier and Type	Field and Description
private BinManager	binManager
static PropertyDescriptor	MAX_BIN_AGE
static PropertyDescriptor	MAX_BIN_COUNT
static PropertyDescriptor	MAX_ENTRIES
static PropertyDescriptor	MAX_SIZE
static PropertyDescriptor	MIN_ENTRIES
static PropertyDescriptor	MIN_SIZE
private Queue<Bin>	readyBins
static Relationship	REL_FAILURE
static Relationship	REL_ORIGINAL

Constructor Summary

Constructors

Constructor and Description
BinFiles()

Method Summary

Modifier and Type	Method and Description
protected Collection<ValidationResult>	additionalCustomValidation(ValidationContext context) Allows additional custom validation to be done.
private BinFiles.BinningResult	binFlowFiles(ProcessContext context, ProcessSessionFactory sessionFactory)
protected Collection<ValidationResult>	customValidate(ValidationContext context)
protected abstract String	getGroupId(ProcessContext context, FlowFile flowFile, ProcessSession session) Returns a group ID representing a bin.
protected int	getMaxBinAgeSeconds(PropertyContext context)
protected long	getMaxBytes(PropertyContext context)
protected int	getMaxEntries(PropertyContext context)
protected long	getMinBytes(PropertyContext context)
protected int	getMinEntries(PropertyContext context)
protected Queue<Bin>	getReadyBins()
private int	migrateBins(ProcessContext context, boolean relaxFullnessConstraint, boolean newBinNeeded)
void	onScheduled(ProcessContext context)
void	onTrigger(ProcessContext context, ProcessSessionFactory sessionFactory)
protected abstract FlowFile	preprocessFlowFile(ProcessContext context, ProcessSession session, FlowFile flowFile) Allows general pre-processing of a flow file before it is offered to a bin.
protected abstract BinProcessingResult	processBin(Bin unmodifiableBin, ProcessContext context) Processes a single bin.
protected int	processBins(ProcessContext context, ProcessSessionFactory sessionFactory)
void	resetState()
protected abstract void	setUpBinManager(BinManager binManager, ProcessContext context) Performs any additional setup of the bin manager.

Methods inherited from class org.apache.nifi.processor.AbstractSessionFactoryProcessor

getControllerServiceLookup, getIdentifier, getLogger, getNodeTypeProvider, getRelationships, init, initialize, isConfigurationRestored, isScheduled, toString, updateConfiguredRestoredTrue, updateScheduledFalse, updateScheduledTrue

Methods inherited from class org.apache.nifi.components.AbstractConfigurableComponent

equals, getPropertyDescriptor, getPropertyDescriptors, getSupportedDynamicPropertyDescriptor, getSupportedPropertyDescriptors, hashCode, onPropertyModified, validate

Methods inherited from class java.lang.Object

clone, finalize, getClass, notify, notifyAll, wait, wait, wait

Methods inherited from interface org.apache.nifi.processor.Processor

isStateful

Methods inherited from interface org.apache.nifi.components.ConfigurableComponent

getPropertyDescriptor, getPropertyDescriptors, onPropertyModified, validate

Field Detail

MIN_SIZE

public static final PropertyDescriptor MIN_SIZE

MAX_SIZE

public static final PropertyDescriptor MAX_SIZE

MIN_ENTRIES

public static final PropertyDescriptor MIN_ENTRIES

MAX_ENTRIES

public static final PropertyDescriptor MAX_ENTRIES

MAX_BIN_COUNT

public static final PropertyDescriptor MAX_BIN_COUNT

MAX_BIN_AGE

public static final PropertyDescriptor MAX_BIN_AGE

REL_ORIGINAL

public static final Relationship REL_ORIGINAL

REL_FAILURE

public static final Relationship REL_FAILURE

binManager

private final BinManager binManager

readyBins

private final Queue<Bin> readyBins

Constructor Detail

BinFiles

public BinFiles()

Method Detail

resetState

@OnStopped
public final void resetState()

preprocessFlowFile

protected abstract FlowFile preprocessFlowFile(ProcessContext context,
                                               ProcessSession session,
                                               FlowFile flowFile)

Allows general pre-processing of a flow file before it is offered to a bin. This is called before getGroupId().

Parameters:

context - context

session - session

flowFile - flowFile

Returns:

The flow file, possibly altered

getGroupId

protected abstract String getGroupId(ProcessContext context,
                                     FlowFile flowFile,
                                     ProcessSession session)

Returns a group ID representing a bin. This allows flow files to be binned into like groups.

Parameters:

context - context

flowFile - flowFile

session - the session for accessing the FlowFile

Returns:

The appropriate group ID

setUpBinManager

protected abstract void setUpBinManager(BinManager binManager,
                                        ProcessContext context)

Performs any additional setup of the bin manager. Called during the OnScheduled phase.

Parameters:

binManager - The bin manager

context - context

processBin

protected abstract BinProcessingResult processBin(Bin unmodifiableBin,
                                                  ProcessContext context)
                                           throws ProcessException

Processes a single bin. Implementing class is responsible for committing each session

Parameters:

unmodifiableBin - A reference to a single bin of flow files

context - The context

Returns:

true if the input bin was already committed. E.g., in case of a failure, the implementation may choose to transfer all binned files to Failure and commit their sessions. If false, the processBins() method will transfer the files to Original and commit the sessions

Throws:

ProcessException - if any problem arises while processing a bin of FlowFiles. All flow files in the bin will be transferred to failure and the ProcessSession provided by the 'session' argument rolled back

additionalCustomValidation

protected Collection<ValidationResult> additionalCustomValidation(ValidationContext context)

Allows additional custom validation to be done. This will be called from the parent's customValidation method.

Parameters:

context - The context

Returns:

Validation results indicating problems

onTrigger

public final void onTrigger(ProcessContext context,
                            ProcessSessionFactory sessionFactory)
                     throws ProcessException

Throws:

ProcessException

migrateBins

private int migrateBins(ProcessContext context,
                        boolean relaxFullnessConstraint,
                        boolean newBinNeeded)

getReadyBins

protected Queue<Bin> getReadyBins()

processBins

protected int processBins(ProcessContext context,
                          ProcessSessionFactory sessionFactory)

binFlowFiles

private BinFiles.BinningResult binFlowFiles(ProcessContext context,
                                            ProcessSessionFactory sessionFactory)

onScheduled

@OnScheduled
public final void onScheduled(ProcessContext context)
                                    throws IOException

Throws:

IOException

getMaxBinAgeSeconds

protected int getMaxBinAgeSeconds(PropertyContext context)

getMinBytes

protected long getMinBytes(PropertyContext context)

getMaxBytes

protected long getMaxBytes(PropertyContext context)

getMinEntries

protected int getMinEntries(PropertyContext context)

getMaxEntries

protected int getMaxEntries(PropertyContext context)

customValidate

protected final Collection<ValidationResult> customValidate(ValidationContext context)

Overrides:

customValidate in class AbstractConfigurableComponent