org.apache.nifi.processors.standard

Class DeduplicateRecord

java.lang.Object

org.apache.nifi.components.AbstractConfigurableComponent

org.apache.nifi.processor.AbstractSessionFactoryProcessor

org.apache.nifi.processor.AbstractProcessor

org.apache.nifi.processors.standard.DeduplicateRecord

All Implemented Interfaces:

ConfigurableComponent, Processor

@EventDriven
 @SupportsBatching
 @InputRequirement(value=INPUT_REQUIRED)
 @SystemResourceConsideration(resource=MEMORY,description="The HashSet filter type will grow memory space proportionate to the number of unique records processed. The BloomFilter type will use constant memory regardless of the number of records processed.") @SystemResourceConsideration(resource=CPU,description="If a more advanced hash algorithm is chosen, the amount of time required to hash any particular record could increase substantially.")
 @Tags(value={"text","record","update","change","replace","modify","distinct","unique","filter","hash","dupe","duplicate","dedupe"})
 @CapabilityDescription(value="This processor attempts to deduplicate a record set in memory using either a hashset or a bloom filter. It operates on a per-file basis rather than across an entire data set that spans multiple files.")
 @WritesAttribute(attribute="record.count",
                 description="The number of records processed.")
 @DynamicProperty(name="RecordPath",
                 value="An expression language statement used to determine how the RecordPath is resolved. The following variables are availible: ${field.name}, ${field.value}, ${field.type}",
                 description="The name of each user-defined property must be a valid RecordPath.")
 @SeeAlso(classNames={"org.apache.nifi.distributed.cache.client.DistributedMapCacheClientService","org.apache.nifi.distributed.cache.server.map.DistributedMapCacheServer","org.apache.nifi.processors.standard.DetectDuplicate"})
public class DeduplicateRecord
extends AbstractProcessor

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
private static class	DeduplicateRecord.BloomFilterWrapper
private static class	DeduplicateRecord.CacheValue
private static class	DeduplicateRecord.DistributedMapCacheClientWrapper
private static class	DeduplicateRecord.FilterWrapper
private static class	DeduplicateRecord.HashSetFilterWrapper

Field Summary

Fields

Modifier and Type	Field and Description
(package private) static PropertyDescriptor	BLOOM_FILTER_FPP
(package private) static AllowableValue	BLOOM_FILTER_VALUE
private static Serializer<Boolean>	BOOLEAN_SERIALIZER
(package private) static PropertyDescriptor	CACHE_IDENTIFIER
(package private) static PropertyDescriptor	DEDUPLICATION_STRATEGY
private List<PropertyDescriptor>	descriptors
(package private) static PropertyDescriptor	DISTRIBUTED_MAP_CACHE
private List<PropertyDescriptor>	dynamicProperties
private static String	FIELD_NAME
private static String	FIELD_TYPE
private static String	FIELD_VALUE
(package private) static PropertyDescriptor	FILTER_CAPACITY_HINT
(package private) static PropertyDescriptor	FILTER_TYPE
(package private) static AllowableValue	HASH_SET_VALUE
(package private) static PropertyDescriptor	INCLUDE_ZERO_RECORD_FLOWFILES
static char	JOIN_CHAR
private DistributedMapCacheClient	mapCacheClient
(package private) static AllowableValue	NONE_ALGORITHM_VALUE
(package private) static AllowableValue	OPTION_MULTIPLE_FILES
(package private) static AllowableValue	OPTION_SINGLE_FILE
private RecordReaderFactory	readerFactory
(package private) static PropertyDescriptor	RECORD_HASHING_ALGORITHM
(package private) static PropertyDescriptor	RECORD_READER
(package private) static PropertyDescriptor	RECORD_WRITER
private RecordPathCache	recordPathCache
(package private) static Relationship	REL_DUPLICATE
(package private) static Relationship	REL_FAILURE
(package private) static Relationship	REL_NON_DUPLICATE
(package private) static Relationship	REL_ORIGINAL
private Set<Relationship>	relationships
(package private) static AllowableValue	SHA256_ALGORITHM_VALUE
(package private) static AllowableValue	SHA512_ALGORITHM_VALUE
private static Serializer<String>	STRING_SERIALIZER
private boolean	useInMemoryStrategy
private RecordSetWriterFactory	writerFactory

Constructor Summary

Constructors

Constructor and Description
DeduplicateRecord()

Method Summary

Modifier and Type	Method and Description
protected Collection<ValidationResult>	customValidate(ValidationContext context)
private String	executeDynamicRecordPaths(ProcessContext context, Record record, FlowFile flowFile)
private DeduplicateRecord.FilterWrapper	getFilter(ProcessContext context)
Set<Relationship>	getRelationships()
protected PropertyDescriptor	getSupportedDynamicPropertyDescriptor(String propertyDescriptorName)
List<PropertyDescriptor>	getSupportedPropertyDescriptors()
protected void	init(ProcessorInitializationContext context)
void	onScheduled(ProcessContext context)
void	onTrigger(ProcessContext context, ProcessSession session)
private void	sendOrRemove(ProcessSession session, FlowFile outputFlowFile, Relationship targetRelationship, String mimeType, boolean includeZeroRecordFlowFiles, WriteResult writeResult)

Methods inherited from class org.apache.nifi.processor.AbstractProcessor

onTrigger

Methods inherited from class org.apache.nifi.processor.AbstractSessionFactoryProcessor

getControllerServiceLookup, getIdentifier, getLogger, getNodeTypeProvider, initialize, isConfigurationRestored, isScheduled, toString, updateConfiguredRestoredTrue, updateScheduledFalse, updateScheduledTrue

Methods inherited from class org.apache.nifi.components.AbstractConfigurableComponent

equals, getPropertyDescriptor, getPropertyDescriptors, hashCode, onPropertyModified, validate

Methods inherited from class java.lang.Object

clone, finalize, getClass, notify, notifyAll, wait, wait, wait

Methods inherited from interface org.apache.nifi.processor.Processor

isStateful

Methods inherited from interface org.apache.nifi.components.ConfigurableComponent

getPropertyDescriptor, getPropertyDescriptors, onPropertyModified, validate

Field Detail

JOIN_CHAR

public static final char JOIN_CHAR

See Also:

Constant Field Values

FIELD_NAME

private static final String FIELD_NAME

See Also:

Constant Field Values

FIELD_VALUE

private static final String FIELD_VALUE

See Also:

Constant Field Values

FIELD_TYPE

private static final String FIELD_TYPE

See Also:

Constant Field Values

recordPathCache

private volatile RecordPathCache recordPathCache

dynamicProperties

private volatile List<PropertyDescriptor> dynamicProperties

NONE_ALGORITHM_VALUE

static final AllowableValue NONE_ALGORITHM_VALUE

SHA256_ALGORITHM_VALUE

static final AllowableValue SHA256_ALGORITHM_VALUE

SHA512_ALGORITHM_VALUE

static final AllowableValue SHA512_ALGORITHM_VALUE

HASH_SET_VALUE

static final AllowableValue HASH_SET_VALUE

BLOOM_FILTER_VALUE

static final AllowableValue BLOOM_FILTER_VALUE

RECORD_READER

static final PropertyDescriptor RECORD_READER

RECORD_WRITER

static final PropertyDescriptor RECORD_WRITER

OPTION_SINGLE_FILE

static final AllowableValue OPTION_SINGLE_FILE

OPTION_MULTIPLE_FILES

static final AllowableValue OPTION_MULTIPLE_FILES

DEDUPLICATION_STRATEGY

static final PropertyDescriptor DEDUPLICATION_STRATEGY

DISTRIBUTED_MAP_CACHE

static final PropertyDescriptor DISTRIBUTED_MAP_CACHE

CACHE_IDENTIFIER

static final PropertyDescriptor CACHE_IDENTIFIER

INCLUDE_ZERO_RECORD_FLOWFILES

static final PropertyDescriptor INCLUDE_ZERO_RECORD_FLOWFILES

RECORD_HASHING_ALGORITHM

static final PropertyDescriptor RECORD_HASHING_ALGORITHM

FILTER_TYPE

static final PropertyDescriptor FILTER_TYPE

FILTER_CAPACITY_HINT

static final PropertyDescriptor FILTER_CAPACITY_HINT

BLOOM_FILTER_FPP

static final PropertyDescriptor BLOOM_FILTER_FPP

REL_DUPLICATE

static final Relationship REL_DUPLICATE

REL_NON_DUPLICATE

static final Relationship REL_NON_DUPLICATE

REL_ORIGINAL

static final Relationship REL_ORIGINAL

REL_FAILURE

static final Relationship REL_FAILURE

descriptors

private List<PropertyDescriptor> descriptors

relationships

private Set<Relationship> relationships

mapCacheClient

private DistributedMapCacheClient mapCacheClient

readerFactory

private RecordReaderFactory readerFactory

writerFactory

private RecordSetWriterFactory writerFactory

useInMemoryStrategy

private boolean useInMemoryStrategy

STRING_SERIALIZER

private static final Serializer<String> STRING_SERIALIZER

BOOLEAN_SERIALIZER

private static final Serializer<Boolean> BOOLEAN_SERIALIZER

Constructor Detail

DeduplicateRecord

public DeduplicateRecord()

Method Detail

init

protected void init(ProcessorInitializationContext context)

Overrides:

init in class AbstractSessionFactoryProcessor

getRelationships

public Set<Relationship> getRelationships()

Specified by:

getRelationships in interface Processor

Overrides:

getRelationships in class AbstractSessionFactoryProcessor

getSupportedPropertyDescriptors

public final List<PropertyDescriptor> getSupportedPropertyDescriptors()

Overrides:

getSupportedPropertyDescriptors in class AbstractConfigurableComponent

getSupportedDynamicPropertyDescriptor

protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(String propertyDescriptorName)

Overrides:

getSupportedDynamicPropertyDescriptor in class AbstractConfigurableComponent

customValidate

protected Collection<ValidationResult> customValidate(ValidationContext context)

Overrides:

customValidate in class AbstractConfigurableComponent

onScheduled

@OnScheduled
public void onScheduled(ProcessContext context)

getFilter

private DeduplicateRecord.FilterWrapper getFilter(ProcessContext context)

onTrigger

public void onTrigger(ProcessContext context,
                      ProcessSession session)
               throws ProcessException

Specified by:

onTrigger in class AbstractProcessor

Throws:

ProcessException

sendOrRemove

private void sendOrRemove(ProcessSession session,
                          FlowFile outputFlowFile,
                          Relationship targetRelationship,
                          String mimeType,
                          boolean includeZeroRecordFlowFiles,
                          WriteResult writeResult)

executeDynamicRecordPaths

private String executeDynamicRecordPaths(ProcessContext context,
                                         Record record,
                                         FlowFile flowFile)