org.apache.nifi.processors.standard

Class ListFile

java.lang.Object

org.apache.nifi.components.AbstractConfigurableComponent

org.apache.nifi.processor.AbstractSessionFactoryProcessor

org.apache.nifi.processor.AbstractProcessor

org.apache.nifi.processor.util.list.AbstractListProcessor<FileInfo>

org.apache.nifi.processors.standard.ListFile

All Implemented Interfaces:

ConfigurableComponent, Processor, VerifiableProcessor

@TriggerSerially
 @InputRequirement(value=INPUT_FORBIDDEN)
 @Tags(value={"file","get","list","ingest","source","filesystem"})
 @CapabilityDescription(value="Retrieves a listing of files from the input directory. For each file listed, creates a FlowFile that represents the file so that it can be fetched in conjunction with FetchFile. This Processor is designed to run on Primary Node only in a cluster when \'Input Directory Location\' is set to \'Remote\'. If the primary node changes, the new Primary Node will pick up where the previous node left off without duplicating all the data. When \'Input Directory Location\' is \'Local\', the \'Execution\' mode can be anything, and synchronization won\'t happen. Unlike GetFile, this Processor does not delete any data from the local filesystem.")
 @WritesAttribute(attribute="filename",description="The name of the file that was read from filesystem.") @WritesAttribute(attribute="path",description="The path is set to the relative path of the file\'s directory on filesystem compared to the Input Directory property. For example, if Input Directory is set to /tmp, then files picked up from /tmp will have the path attribute set to \"/\". If the Recurse Subdirectories property is set to true and a file is picked up from /tmp/abc/1/2/3, then the path attribute will be set to \"abc/1/2/3/\".") @WritesAttribute(attribute="absolute.path",description="The absolute.path is set to the absolute path of the file\'s directory on filesystem. For example, if the Input Directory property is set to /tmp, then files picked up from /tmp will have the path attribute set to \"/tmp/\". If the Recurse Subdirectories property is set to true and a file is picked up from /tmp/abc/1/2/3, then the path attribute will be set to \"/tmp/abc/1/2/3/\".") @WritesAttribute(attribute="file.owner",description="The user that owns the file in filesystem") @WritesAttribute(attribute="file.group",description="The group that owns the file in filesystem") @WritesAttribute(attribute="file.size",description="The number of bytes in the file in filesystem") @WritesAttribute(attribute="file.permissions",description="The permissions for the file in filesystem. This is formatted as 3 characters for the owner, 3 for the group, and 3 for other users. For example rw-rw-r--") @WritesAttribute(attribute="file.lastModifiedTime",description="The timestamp of when the file in filesystem was last modified as \'yyyy-MM-dd\'T\'HH:mm:ssZ\'") @WritesAttribute(attribute="file.lastAccessTime",description="The timestamp of when the file in filesystem was last accessed as \'yyyy-MM-dd\'T\'HH:mm:ssZ\'") @WritesAttribute(attribute="file.creationTime",description="The timestamp of when the file in filesystem was created as \'yyyy-MM-dd\'T\'HH:mm:ssZ\'")
 @SeeAlso(value={GetFile.class,PutFile.class,FetchFile.class})
 @Stateful(scopes={LOCAL,CLUSTER},
          description="After performing a listing of files, the timestamp of the newest file is stored. This allows the Processor to list only files that have been added or modified after this date the next time that the Processor is run. Whether the state is stored with a Local or Cluster scope depends on the value of the <Input Directory Location> property.")
 @DefaultSchedule(strategy=TIMER_DRIVEN,
                 period="1 min")
public class ListFile
extends AbstractListProcessor<FileInfo>

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
private static class	ListFile.DiskOperation
(package private) static class	ListFile.MonitorActiveTasks
(package private) static interface	ListFile.OperationStatistics
(package private) static interface	ListFile.PerformanceTracker PerformanceTracker is responsible for providing a mechanism by which any disk operation can be timed and the timing information can both be used to issue warnings as well as be aggregated for some amount of time, in order to understand how long certain disk operations take and which files may be responsible for causing longer-than-usual operations to be performed.
private static class	ListFile.ProcessorStoppedException
static class	ListFile.RollingMetricPerformanceTracker Tracks metrics using a rolling window of time, in which older metrics are 'aged off' by calling ListFile.RollingMetricPerformanceTracker.purgeTimingInfo(long).
private static class	ListFile.StandardOperationStatistics
private static class	ListFile.TimedOperationKey
private static class	ListFile.TimingInfo Provides a mechanism for timing how long a particular operation takes to complete, logging if it takes longer than the configured threshold.
static class	ListFile.UntrackedPerformanceTracker A PerformanceTracker that is capable of tracking which disk access operation is active and which directory is actively being listed, as well as timing specific operations, but does not track metrics over any amount of time.

Nested classes/interfaces inherited from class org.apache.nifi.processor.util.list.AbstractListProcessor

AbstractListProcessor.ListingMode

Field Summary

Fields

Modifier and Type	Field and Description
static PropertyDescriptor	DIRECTORY
static PropertyDescriptor	DIRECTORY_LOCATION
static String	FILE_CREATION_TIME_ATTRIBUTE
static PropertyDescriptor	FILE_FILTER
static String	FILE_GROUP_ATTRIBUTE
static String	FILE_LAST_ACCESS_TIME_ATTRIBUTE
static String	FILE_LAST_MODIFY_TIME_ATTRIBUTE
static String	FILE_MODIFY_DATE_ATTR_FORMAT
static String	FILE_OWNER_ATTRIBUTE
static String	FILE_PERMISSIONS_ATTRIBUTE
static String	FILE_SIZE_ATTRIBUTE
static PropertyDescriptor	IGNORE_HIDDEN_FILES
static PropertyDescriptor	INCLUDE_FILE_ATTRIBUTES
private boolean	includeFileAttributes
(package private) static AllowableValue	LOCATION_LOCAL
(package private) static AllowableValue	LOCATION_REMOTE
static PropertyDescriptor	MAX_AGE
static PropertyDescriptor	MAX_DISK_OPERATION_TIME
static PropertyDescriptor	MAX_LISTING_TIME
static PropertyDescriptor	MAX_SIZE
static PropertyDescriptor	MAX_TRACKED_FILES
static PropertyDescriptor	MIN_AGE
static PropertyDescriptor	MIN_SIZE
private Future<?>	monitoringFuture
private ScheduledExecutorService	monitoringThreadPool
static PropertyDescriptor	PATH_FILTER
private long	performanceLoggingTimestamp
private ListFile.PerformanceTracker	performanceTracker
private List<PropertyDescriptor>	properties
static PropertyDescriptor	RECURSE
private Set<Relationship>	relationships
static PropertyDescriptor	TRACK_PERFORMANCE

Fields inherited from class org.apache.nifi.processor.util.list.AbstractListProcessor

BY_ENTITIES, BY_TIME_WINDOW, BY_TIMESTAMPS, DISTRIBUTED_CACHE_SERVICE, LISTING_LAG_MILLIS, LISTING_STRATEGY, NO_TRACKING, PRECISION_AUTO_DETECT, PRECISION_MILLIS, PRECISION_MINUTES, PRECISION_SECONDS, RECORD_WRITER, REL_SUCCESS, TARGET_SYSTEM_TIMESTAMP_PRECISION

Constructor Summary

Constructors

Constructor and Description
ListFile()

Method Summary

Modifier and Type	Method and Description
protected Integer	countUnfilteredListing(ProcessContext context)
protected Map<String,String>	createAttributes(FileInfo fileInfo, ProcessContext context)
private BiPredicate<Path,BasicFileAttributes>	createFileFilter(ProcessContext context, ListFile.PerformanceTracker performanceTracker, boolean applyFilters, Path basePath)
protected String	getListingContainerName(ProcessContext context)
protected String	getPath(ProcessContext context)
protected ListFile.PerformanceTracker	getPerformanceTracker()
protected RecordSchema	getRecordSchema()
Set<Relationship>	getRelationships()
protected Scope	getStateScope(PropertyContext context)
protected List<PropertyDescriptor>	getSupportedPropertyDescriptors()
protected void	init(ProcessorInitializationContext context)
protected boolean	isListingResetNecessary(PropertyDescriptor property)
void	logPerformance()
void	onScheduled(ProcessContext context)
void	onStopped(ProcessContext context)
protected List<FileInfo>	performListing(ProcessContext context, Long minTimestamp, AbstractListProcessor.ListingMode listingMode)
private List<FileInfo>	performListing(ProcessContext context, Long minTimestamp, AbstractListProcessor.ListingMode listingMode, boolean applyFilters)

Methods inherited from class org.apache.nifi.processor.util.list.AbstractListProcessor

createListedEntityTracker, customValidate, customValidate, getCurrentNanoTime, getCurrentTime, getDefaultTimePrecision, getKey, getPersistenceFile, initListedEntityTracker, listByNoTracking, listByTimeWindow, listByTrackingTimestamps, onPrimaryNodeChange, onPropertyModified, onTrigger, updateState, verify

Methods inherited from class org.apache.nifi.processor.AbstractProcessor

onTrigger

Methods inherited from class org.apache.nifi.processor.AbstractSessionFactoryProcessor

getControllerServiceLookup, getIdentifier, getLogger, getNodeTypeProvider, initialize, isConfigurationRestored, isScheduled, toString, updateConfiguredRestoredTrue, updateScheduledFalse, updateScheduledTrue

Methods inherited from class org.apache.nifi.components.AbstractConfigurableComponent

equals, getPropertyDescriptor, getPropertyDescriptors, getSupportedDynamicPropertyDescriptor, hashCode, validate

Methods inherited from class java.lang.Object

clone, finalize, getClass, notify, notifyAll, wait, wait, wait

Methods inherited from interface org.apache.nifi.processor.Processor

isStateful

Methods inherited from interface org.apache.nifi.components.ConfigurableComponent

getPropertyDescriptor, getPropertyDescriptors, validate

Field Detail

LOCATION_LOCAL

static final AllowableValue LOCATION_LOCAL

LOCATION_REMOTE

static final AllowableValue LOCATION_REMOTE

DIRECTORY

public static final PropertyDescriptor DIRECTORY

RECURSE

public static final PropertyDescriptor RECURSE

DIRECTORY_LOCATION

public static final PropertyDescriptor DIRECTORY_LOCATION

FILE_FILTER

public static final PropertyDescriptor FILE_FILTER

PATH_FILTER

public static final PropertyDescriptor PATH_FILTER

INCLUDE_FILE_ATTRIBUTES

public static final PropertyDescriptor INCLUDE_FILE_ATTRIBUTES

MIN_AGE

public static final PropertyDescriptor MIN_AGE

MAX_AGE

public static final PropertyDescriptor MAX_AGE

MIN_SIZE

public static final PropertyDescriptor MIN_SIZE

MAX_SIZE

public static final PropertyDescriptor MAX_SIZE

IGNORE_HIDDEN_FILES

public static final PropertyDescriptor IGNORE_HIDDEN_FILES

TRACK_PERFORMANCE

public static final PropertyDescriptor TRACK_PERFORMANCE

MAX_TRACKED_FILES

public static final PropertyDescriptor MAX_TRACKED_FILES

MAX_DISK_OPERATION_TIME

public static final PropertyDescriptor MAX_DISK_OPERATION_TIME

MAX_LISTING_TIME

public static final PropertyDescriptor MAX_LISTING_TIME

properties

private List<PropertyDescriptor> properties

relationships

private Set<Relationship> relationships

monitoringThreadPool

private volatile ScheduledExecutorService monitoringThreadPool

monitoringFuture

private volatile Future<?> monitoringFuture

includeFileAttributes

private volatile boolean includeFileAttributes

performanceTracker

private volatile ListFile.PerformanceTracker performanceTracker

performanceLoggingTimestamp

private volatile long performanceLoggingTimestamp

FILE_CREATION_TIME_ATTRIBUTE

public static final String FILE_CREATION_TIME_ATTRIBUTE

See Also:

Constant Field Values

FILE_LAST_MODIFY_TIME_ATTRIBUTE

public static final String FILE_LAST_MODIFY_TIME_ATTRIBUTE

See Also:

Constant Field Values

FILE_LAST_ACCESS_TIME_ATTRIBUTE

public static final String FILE_LAST_ACCESS_TIME_ATTRIBUTE

See Also:

Constant Field Values

FILE_SIZE_ATTRIBUTE

public static final String FILE_SIZE_ATTRIBUTE

See Also:

Constant Field Values

FILE_OWNER_ATTRIBUTE

public static final String FILE_OWNER_ATTRIBUTE

See Also:

Constant Field Values

FILE_GROUP_ATTRIBUTE

public static final String FILE_GROUP_ATTRIBUTE

See Also:

Constant Field Values

FILE_PERMISSIONS_ATTRIBUTE

public static final String FILE_PERMISSIONS_ATTRIBUTE

See Also:

Constant Field Values

FILE_MODIFY_DATE_ATTR_FORMAT

public static final String FILE_MODIFY_DATE_ATTR_FORMAT

See Also:

Constant Field Values

Constructor Detail

ListFile

public ListFile()

Method Detail

init

protected void init(ProcessorInitializationContext context)

Overrides:

init in class AbstractSessionFactoryProcessor

getSupportedPropertyDescriptors

protected List<PropertyDescriptor> getSupportedPropertyDescriptors()

Overrides:

getSupportedPropertyDescriptors in class AbstractConfigurableComponent

getRelationships

public Set<Relationship> getRelationships()

Specified by:

getRelationships in interface Processor

Overrides:

getRelationships in class AbstractListProcessor<FileInfo>

onScheduled

@OnScheduled
public void onScheduled(ProcessContext context)

onStopped

@OnStopped
public void onStopped(ProcessContext context)

getPerformanceTracker

protected ListFile.PerformanceTracker getPerformanceTracker()

logPerformance

public void logPerformance()

createAttributes

protected Map<String,String> createAttributes(FileInfo fileInfo,
                                              ProcessContext context)

Specified by:

createAttributes in class AbstractListProcessor<FileInfo>

getPath

protected String getPath(ProcessContext context)

Specified by:

getPath in class AbstractListProcessor<FileInfo>

getStateScope

protected Scope getStateScope(PropertyContext context)

Specified by:

getStateScope in class AbstractListProcessor<FileInfo>

getRecordSchema

protected RecordSchema getRecordSchema()

Specified by:

getRecordSchema in class AbstractListProcessor<FileInfo>

countUnfilteredListing

protected Integer countUnfilteredListing(ProcessContext context)
                                  throws IOException

Specified by:

countUnfilteredListing in class AbstractListProcessor<FileInfo>

Throws:

IOException

performListing

protected List<FileInfo> performListing(ProcessContext context,
                                        Long minTimestamp,
                                        AbstractListProcessor.ListingMode listingMode)
                                 throws IOException

Specified by:

performListing in class AbstractListProcessor<FileInfo>

Throws:

IOException

performListing

private List<FileInfo> performListing(ProcessContext context,
                                      Long minTimestamp,
                                      AbstractListProcessor.ListingMode listingMode,
                                      boolean applyFilters)
                               throws IOException

Throws:

IOException

getListingContainerName

protected String getListingContainerName(ProcessContext context)

Specified by:

getListingContainerName in class AbstractListProcessor<FileInfo>

isListingResetNecessary

protected boolean isListingResetNecessary(PropertyDescriptor property)

Specified by:

isListingResetNecessary in class AbstractListProcessor<FileInfo>

createFileFilter

private BiPredicate<Path,BasicFileAttributes> createFileFilter(ProcessContext context,
                                                               ListFile.PerformanceTracker performanceTracker,
                                                               boolean applyFilters,
                                                               Path basePath)