org.apache.nifi.processors.standard

Class MergeContent

java.lang.Object

org.apache.nifi.components.AbstractConfigurableComponent

org.apache.nifi.processor.AbstractSessionFactoryProcessor

org.apache.nifi.processor.util.bin.BinFiles

org.apache.nifi.processors.standard.MergeContent

All Implemented Interfaces:

ConfigurableComponent, Processor

@SideEffectFree
 @TriggerWhenEmpty
 @InputRequirement(value=INPUT_REQUIRED)
 @Tags(value={"merge","content","correlation","tar","zip","stream","concatenation","archive","flowfile-stream","flowfile-stream-v3"})
 @CapabilityDescription(value="Merges a Group of FlowFiles together based on a user-defined strategy and packages them into a single FlowFile. It is recommended that the Processor be configured with only a single incoming connection, as Group of FlowFiles will not be created from FlowFiles in different connections. This processor updates the mime.type attribute as appropriate. NOTE: this processor should NOT be configured with Cron Driven for the Scheduling Strategy.")
 @ReadsAttribute(attribute="fragment.identifier",description="Applicable only if the <Merge Strategy> property is set to Defragment. All FlowFiles with the same value for this attribute will be bundled together.") @ReadsAttribute(attribute="fragment.index",description="Applicable only if the <Merge Strategy> property is set to Defragment. This attribute indicates the order in which the fragments should be assembled. This attribute must be present on all FlowFiles when using the Defragment Merge Strategy and must be a unique (i.e., unique across all FlowFiles that have the same value for the \"fragment.identifier\" attribute) integer between 0 and the value of the fragment.count attribute. If two or more FlowFiles have the same value for the \"fragment.identifier\" attribute and the same value for the \"fragment.index\" attribute, the first FlowFile processed will be accepted and subsequent FlowFiles will not be accepted into the Bin.") @ReadsAttribute(attribute="fragment.count",description="Applicable only if the <Merge Strategy> property is set to Defragment. This attribute indicates how many FlowFiles should be expected in the given bundle. At least one FlowFile must have this attribute in the bundle. If multiple FlowFiles contain the \"fragment.count\" attribute in a given bundle, all must have the same value.") @ReadsAttribute(attribute="segment.original.filename",description="Applicable only if the <Merge Strategy> property is set to Defragment. This attribute must be present on all FlowFiles with the same value for the fragment.identifier attribute. All FlowFiles in the same bundle must have the same value for this attribute. The value of this attribute will be used for the filename of the completed merged FlowFile.") @ReadsAttribute(attribute="tar.permissions",description="Applicable only if the <Merge Format> property is set to TAR. The value of this attribute must be 3 characters; each character must be in the range 0 to 7 (inclusive) and indicates the file permissions that should be used for the FlowFile\'s TAR entry. If this attribute is missing or has an invalid value, the default value of 644 will be used")
 @WritesAttribute(attribute="filename",description="When more than 1 file is merged, the filename comes from the segment.original.filename attribute. If that attribute does not exist in the source FlowFiles, then the filename is set to the number of nanoseconds matching system time. Then a filename extension may be applied:if Merge Format is TAR, then the filename will be appended with .tar, if Merge Format is ZIP, then the filename will be appended with .zip, if Merge Format is FlowFileStream, then the filename will be appended with .pkg") @WritesAttribute(attribute="merge.count",description="The number of FlowFiles that were merged into this bundle") @WritesAttribute(attribute="merge.bin.age",description="The age of the bin, in milliseconds, when it was merged and output. Effectively this is the greatest amount of time that any FlowFile in this bundle remained waiting in this processor before it was output") @WritesAttribute(attribute="merge.uuid",description="UUID of the merged flow file that will be added to the original flow files attributes.") @WritesAttribute(attribute="merge.reason",description="This processor allows for several thresholds to be configured for merging FlowFiles. This attribute indicates which of the Thresholds resulted in the FlowFiles being merged. For an explanation of each of the possible values and their meanings, see the Processor\'s Usage / documentation and see the \'Additional Details\' page.")
 @SeeAlso(value={SegmentContent.class,MergeRecord.class})
 @SystemResourceConsideration(resource=MEMORY,
                             description="While content is not stored in memory, the FlowFiles\' attributes are. The configuration of MergeContent (maximum bin size, maximum group size, maximum bin age, max number of entries) will influence how much memory is used. If merging together many small FlowFiles, a two-stage approach may be necessary in order to avoid excessive use of memory.")
public class MergeContent
extends BinFiles

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
private class	MergeContent.AvroMerge
private class	MergeContent.BinaryConcatenationMerge
private class	MergeContent.FlowFileStreamMerger
private static class	MergeContent.FragmentComparator
private static interface	MergeContent.MergeBin
private class	MergeContent.TarMerge
private class	MergeContent.ZipMerge

Field Summary

Fields

Modifier and Type	Field and Description
static PropertyDescriptor	COMPRESSION_LEVEL
static PropertyDescriptor	CORRELATION_ATTRIBUTE_NAME
static PropertyDescriptor	DELIMITER_STRATEGY
static AllowableValue	DELIMITER_STRATEGY_FILENAME
static AllowableValue	DELIMITER_STRATEGY_NONE
static AllowableValue	DELIMITER_STRATEGY_TEXT
static PropertyDescriptor	DEMARCATOR
static PropertyDescriptor	FOOTER
static String	FRAGMENT_COUNT_ATTRIBUTE
static String	FRAGMENT_ID_ATTRIBUTE
static String	FRAGMENT_INDEX_ATTRIBUTE
static PropertyDescriptor	HEADER
static PropertyDescriptor	KEEP_PATH
static String	MERGE_BIN_AGE_ATTRIBUTE
static String	MERGE_COUNT_ATTRIBUTE
static PropertyDescriptor	MERGE_FORMAT
static AllowableValue	MERGE_FORMAT_AVRO
static String	MERGE_FORMAT_AVRO_VALUE
static AllowableValue	MERGE_FORMAT_CONCAT
static String	MERGE_FORMAT_CONCAT_VALUE
static AllowableValue	MERGE_FORMAT_FLOWFILE_STREAM_V2
static String	MERGE_FORMAT_FLOWFILE_STREAM_V2_VALUE
static AllowableValue	MERGE_FORMAT_FLOWFILE_STREAM_V3
static String	MERGE_FORMAT_FLOWFILE_STREAM_V3_VALUE
static AllowableValue	MERGE_FORMAT_FLOWFILE_TAR_V1
static String	MERGE_FORMAT_FLOWFILE_TAR_V1_VALUE
static AllowableValue	MERGE_FORMAT_TAR
static String	MERGE_FORMAT_TAR_VALUE
static AllowableValue	MERGE_FORMAT_ZIP
static String	MERGE_FORMAT_ZIP_VALUE
static PropertyDescriptor	MERGE_STRATEGY
static AllowableValue	MERGE_STRATEGY_BIN_PACK
static AllowableValue	MERGE_STRATEGY_DEFRAGMENT
static String	MERGE_UUID_ATTRIBUTE
static PropertyDescriptor	METADATA_STRATEGY
static AllowableValue	METADATA_STRATEGY_ALL_COMMON
static AllowableValue	METADATA_STRATEGY_DO_NOT_MERGE
static AllowableValue	METADATA_STRATEGY_IGNORE
static AllowableValue	METADATA_STRATEGY_USE_FIRST
static Pattern	NUMBER_PATTERN
static String	REASON_FOR_MERGING
static Relationship	REL_MERGED
static String	SEGMENT_COUNT_ATTRIBUTE
static String	SEGMENT_ID_ATTRIBUTE
static String	SEGMENT_INDEX_ATTRIBUTE
static String	SEGMENT_ORIGINAL_FILENAME
static PropertyDescriptor	TAR_MODIFIED_TIME
static String	TAR_PERMISSIONS_ATTRIBUTE

Fields inherited from class org.apache.nifi.processor.util.bin.BinFiles

MAX_BIN_AGE, MAX_BIN_COUNT, MAX_ENTRIES, MAX_SIZE, MIN_ENTRIES, MIN_SIZE, REL_FAILURE, REL_ORIGINAL

Constructor Summary

Constructors

Constructor and Description
MergeContent()

Method Summary

Modifier and Type	Method and Description
private PropertyDescriptor	addBinPackingDependency(PropertyDescriptor original)
protected Collection<ValidationResult>	additionalCustomValidation(ValidationContext context)
private String	createFilename(List<FlowFile> flowFiles)
private String	getDefragmentValidationError(List<FlowFile> binContents)
protected String	getGroupId(ProcessContext context, FlowFile flowFile, ProcessSession session)
private String	getPath(FlowFile flowFile)
Set<Relationship>	getRelationships()
protected List<PropertyDescriptor>	getSupportedPropertyDescriptors()
private boolean	isNumber(String value)
protected FlowFile	preprocessFlowFile(ProcessContext context, ProcessSession session, FlowFile flowFile)
protected BinProcessingResult	processBin(Bin bin, ProcessContext context)
private byte[]	readContent(String filename)
private void	removeFlowFileFromSession(ProcessSession session, FlowFile flowFile, ProcessContext context)
protected void	setUpBinManager(BinManager binManager, ProcessContext context)

Methods inherited from class org.apache.nifi.processor.util.bin.BinFiles

customValidate, getMaxBinAgeSeconds, getMaxBytes, getMaxEntries, getMinBytes, getMinEntries, getReadyBins, onScheduled, onTrigger, processBins, resetState

Methods inherited from class org.apache.nifi.processor.AbstractSessionFactoryProcessor

getControllerServiceLookup, getIdentifier, getLogger, getNodeTypeProvider, init, initialize, isConfigurationRestored, isScheduled, toString, updateConfiguredRestoredTrue, updateScheduledFalse, updateScheduledTrue

Methods inherited from class org.apache.nifi.components.AbstractConfigurableComponent

equals, getPropertyDescriptor, getPropertyDescriptors, getSupportedDynamicPropertyDescriptor, hashCode, onPropertyModified, validate

Methods inherited from class java.lang.Object

clone, finalize, getClass, notify, notifyAll, wait, wait, wait

Methods inherited from interface org.apache.nifi.processor.Processor

isStateful

Methods inherited from interface org.apache.nifi.components.ConfigurableComponent

getPropertyDescriptor, getPropertyDescriptors, onPropertyModified, validate

Field Detail

FRAGMENT_ID_ATTRIBUTE

public static final String FRAGMENT_ID_ATTRIBUTE

FRAGMENT_INDEX_ATTRIBUTE

public static final String FRAGMENT_INDEX_ATTRIBUTE

FRAGMENT_COUNT_ATTRIBUTE

public static final String FRAGMENT_COUNT_ATTRIBUTE

SEGMENT_ID_ATTRIBUTE

public static final String SEGMENT_ID_ATTRIBUTE

See Also:

Constant Field Values

SEGMENT_INDEX_ATTRIBUTE

public static final String SEGMENT_INDEX_ATTRIBUTE

See Also:

Constant Field Values

SEGMENT_COUNT_ATTRIBUTE

public static final String SEGMENT_COUNT_ATTRIBUTE

See Also:

Constant Field Values

SEGMENT_ORIGINAL_FILENAME

public static final String SEGMENT_ORIGINAL_FILENAME

METADATA_STRATEGY_USE_FIRST

public static final AllowableValue METADATA_STRATEGY_USE_FIRST

METADATA_STRATEGY_ALL_COMMON

public static final AllowableValue METADATA_STRATEGY_ALL_COMMON

METADATA_STRATEGY_IGNORE

public static final AllowableValue METADATA_STRATEGY_IGNORE

METADATA_STRATEGY_DO_NOT_MERGE

public static final AllowableValue METADATA_STRATEGY_DO_NOT_MERGE

MERGE_STRATEGY_BIN_PACK

public static final AllowableValue MERGE_STRATEGY_BIN_PACK

MERGE_STRATEGY_DEFRAGMENT

public static final AllowableValue MERGE_STRATEGY_DEFRAGMENT

DELIMITER_STRATEGY_FILENAME

public static final AllowableValue DELIMITER_STRATEGY_FILENAME

DELIMITER_STRATEGY_TEXT

public static final AllowableValue DELIMITER_STRATEGY_TEXT

DELIMITER_STRATEGY_NONE

public static final AllowableValue DELIMITER_STRATEGY_NONE

MERGE_FORMAT_TAR_VALUE

public static final String MERGE_FORMAT_TAR_VALUE

See Also:

Constant Field Values

MERGE_FORMAT_ZIP_VALUE

public static final String MERGE_FORMAT_ZIP_VALUE

See Also:

Constant Field Values

MERGE_FORMAT_FLOWFILE_STREAM_V3_VALUE

public static final String MERGE_FORMAT_FLOWFILE_STREAM_V3_VALUE

See Also:

Constant Field Values

MERGE_FORMAT_FLOWFILE_STREAM_V2_VALUE

public static final String MERGE_FORMAT_FLOWFILE_STREAM_V2_VALUE

See Also:

Constant Field Values

MERGE_FORMAT_FLOWFILE_TAR_V1_VALUE

public static final String MERGE_FORMAT_FLOWFILE_TAR_V1_VALUE

See Also:

Constant Field Values

MERGE_FORMAT_CONCAT_VALUE

public static final String MERGE_FORMAT_CONCAT_VALUE

See Also:

Constant Field Values

MERGE_FORMAT_AVRO_VALUE

public static final String MERGE_FORMAT_AVRO_VALUE

See Also:

Constant Field Values

MERGE_FORMAT_TAR

public static final AllowableValue MERGE_FORMAT_TAR

MERGE_FORMAT_ZIP

public static final AllowableValue MERGE_FORMAT_ZIP

MERGE_FORMAT_FLOWFILE_STREAM_V3

public static final AllowableValue MERGE_FORMAT_FLOWFILE_STREAM_V3

MERGE_FORMAT_FLOWFILE_STREAM_V2

public static final AllowableValue MERGE_FORMAT_FLOWFILE_STREAM_V2

MERGE_FORMAT_FLOWFILE_TAR_V1

public static final AllowableValue MERGE_FORMAT_FLOWFILE_TAR_V1

MERGE_FORMAT_CONCAT

public static final AllowableValue MERGE_FORMAT_CONCAT

MERGE_FORMAT_AVRO

public static final AllowableValue MERGE_FORMAT_AVRO

TAR_PERMISSIONS_ATTRIBUTE

public static final String TAR_PERMISSIONS_ATTRIBUTE

See Also:

Constant Field Values

MERGE_COUNT_ATTRIBUTE

public static final String MERGE_COUNT_ATTRIBUTE

See Also:

Constant Field Values

MERGE_BIN_AGE_ATTRIBUTE

public static final String MERGE_BIN_AGE_ATTRIBUTE

See Also:

Constant Field Values

MERGE_UUID_ATTRIBUTE

public static final String MERGE_UUID_ATTRIBUTE

See Also:

Constant Field Values

REASON_FOR_MERGING

public static final String REASON_FOR_MERGING

See Also:

Constant Field Values

MERGE_STRATEGY

public static final PropertyDescriptor MERGE_STRATEGY

MERGE_FORMAT

public static final PropertyDescriptor MERGE_FORMAT

METADATA_STRATEGY

public static final PropertyDescriptor METADATA_STRATEGY

CORRELATION_ATTRIBUTE_NAME

public static final PropertyDescriptor CORRELATION_ATTRIBUTE_NAME

DELIMITER_STRATEGY

public static final PropertyDescriptor DELIMITER_STRATEGY

HEADER

public static final PropertyDescriptor HEADER

FOOTER

public static final PropertyDescriptor FOOTER

DEMARCATOR

public static final PropertyDescriptor DEMARCATOR

COMPRESSION_LEVEL

public static final PropertyDescriptor COMPRESSION_LEVEL

KEEP_PATH

public static final PropertyDescriptor KEEP_PATH

TAR_MODIFIED_TIME

public static final PropertyDescriptor TAR_MODIFIED_TIME

REL_MERGED

public static final Relationship REL_MERGED

NUMBER_PATTERN

public static final Pattern NUMBER_PATTERN

Constructor Detail

MergeContent

public MergeContent()

Method Detail

getRelationships

public Set<Relationship> getRelationships()

Specified by:

getRelationships in interface Processor

Overrides:

getRelationships in class AbstractSessionFactoryProcessor

getSupportedPropertyDescriptors

protected List<PropertyDescriptor> getSupportedPropertyDescriptors()

Overrides:

getSupportedPropertyDescriptors in class AbstractConfigurableComponent

addBinPackingDependency

private PropertyDescriptor addBinPackingDependency(PropertyDescriptor original)

additionalCustomValidation

protected Collection<ValidationResult> additionalCustomValidation(ValidationContext context)

Overrides:

additionalCustomValidation in class BinFiles

readContent

private byte[] readContent(String filename)
                    throws IOException

Throws:

IOException

preprocessFlowFile

protected FlowFile preprocessFlowFile(ProcessContext context,
                                      ProcessSession session,
                                      FlowFile flowFile)

Specified by:

preprocessFlowFile in class BinFiles

getGroupId

protected String getGroupId(ProcessContext context,
                            FlowFile flowFile,
                            ProcessSession session)

Specified by:

getGroupId in class BinFiles

setUpBinManager

protected void setUpBinManager(BinManager binManager,
                               ProcessContext context)

Specified by:

setUpBinManager in class BinFiles

processBin

protected BinProcessingResult processBin(Bin bin,
                                         ProcessContext context)
                                  throws ProcessException

Specified by:

processBin in class BinFiles

Throws:

ProcessException

getDefragmentValidationError

private String getDefragmentValidationError(List<FlowFile> binContents)

isNumber

private boolean isNumber(String value)

removeFlowFileFromSession

private void removeFlowFileFromSession(ProcessSession session,
                                       FlowFile flowFile,
                                       ProcessContext context)

getPath

private String getPath(FlowFile flowFile)

createFilename

private String createFilename(List<FlowFile> flowFiles)