Serialized Form

Package com.digitalpebble.stormcrawler.opensearch.bolt

Class com.digitalpebble.stormcrawler.opensearch.bolt.DeletionBolt

class DeletionBolt extends org.apache.storm.topology.base.BaseRichBolt implements Serializable

Serialized Fields

_collector

org.apache.storm.task.OutputCollector _collector

connection

OpenSearchConnection connection

indexName

String indexName

waitAck

com.github.benmanes.caffeine.cache.Cache<String,List<org.apache.storm.tuple.Tuple>> waitAck

waitAckLock

ReentrantLock waitAckLock

Class com.digitalpebble.stormcrawler.opensearch.bolt.IndexerBolt

class IndexerBolt extends AbstractIndexerBolt implements Serializable

Serialized Fields

_collector

org.apache.storm.task.OutputCollector _collector

connection

OpenSearchConnection connection

create

boolean create

eventCounter

org.apache.storm.metric.api.MultiCountMetric eventCounter

indexName

String indexName

perSecMetrics

org.apache.storm.metric.api.MultiReducedMetric perSecMetrics

pipeline

String pipeline

waitAck

com.github.benmanes.caffeine.cache.Cache<String,List<org.apache.storm.tuple.Tuple>> waitAck

waitAckLock

ReentrantLock waitAckLock

Package com.digitalpebble.stormcrawler.opensearch.metrics

Class com.digitalpebble.stormcrawler.opensearch.metrics.StatusMetricsBolt

class StatusMetricsBolt extends org.apache.storm.topology.base.BaseRichBolt implements Serializable

Serialized Fields

_collector

org.apache.storm.task.OutputCollector _collector

connection

OpenSearchConnection connection

freqStats

int freqStats

indexName

String indexName

latestStatusCounts

Map<String,Long> latestStatusCounts

Package com.digitalpebble.stormcrawler.opensearch.persistence

Class com.digitalpebble.stormcrawler.opensearch.persistence.AbstractSpout

class AbstractSpout extends AbstractQueryingSpout implements Serializable

Serialized Fields

bucketSortField

List<String> bucketSortField

filterQueries

List<String> filterQueries

Query to use as a positive filter, set by es.status.filterQuery

indexName

String indexName

logIdprefix

String logIdprefix

Used to distinguish between instances in the logs *

maxBucketNum

int maxBucketNum

maxURLsPerBucket

int maxURLsPerBucket

partitionField

String partitionField

Field name used for field collapsing e.g. key *

queryDate

Date queryDate

queryTimeout

int queryTimeout

shardID

int shardID

when using multiple instances - each one is in charge of a specific shard useful when sharding based on host or domain to guarantee a good mix of URLs

totalSortField

String totalSortField

Class com.digitalpebble.stormcrawler.opensearch.persistence.AggregationSpout

class AggregationSpout extends AbstractSpout implements Serializable

Serialized Fields

currentBuckets

Set<String> currentBuckets

recentDateIncrease

int recentDateIncrease

recentDateMinGap

int recentDateMinGap

sample

boolean sample

Class com.digitalpebble.stormcrawler.opensearch.persistence.HybridSpout

class HybridSpout extends AggregationSpout implements Serializable

Serialized Fields

bufferReloadSize

int bufferReloadSize

hrl

com.digitalpebble.stormcrawler.opensearch.persistence.HybridSpout.HostResultListener hrl

searchAfterCache

com.github.benmanes.caffeine.cache.Cache<String,Object[]> searchAfterCache

Class com.digitalpebble.stormcrawler.opensearch.persistence.StatusUpdaterBolt

class StatusUpdaterBolt extends AbstractStatusUpdaterBolt implements Serializable

Serialized Fields

connection

OpenSearchConnection connection

doRouting

boolean doRouting

whether to apply the same partitioning logic used for politeness for routing, e.g byHost

eventCounter

org.apache.storm.metric.api.MultiCountMetric eventCounter

fieldNameForRoutingKey

String fieldNameForRoutingKey

Store the key used for routing explicitly as a field in metadata *

indexName

String indexName

OSBoltType

String OSBoltType

partitioner

URLPartitioner partitioner

routingFieldNameInMetadata

boolean routingFieldNameInMetadata

waitAck

com.github.benmanes.caffeine.cache.Cache<String,List<org.apache.storm.tuple.Tuple>> waitAck

waitAckLock

ReentrantLock waitAckLock