org.apache.nifi.remote.util

Class SiteToSiteRestApiClient

java.lang.Object

org.apache.nifi.remote.util.SiteToSiteRestApiClient

All Implemented Interfaces:

Closeable, AutoCloseable

public class SiteToSiteRestApiClient
extends Object
implements Closeable

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
class	SiteToSiteRestApiClient.HttpGetFailedException
private class	SiteToSiteRestApiClient.HttpsResponseInterceptor
private static class	SiteToSiteRestApiClient.RemoteGroupContents

Field Summary

Fields

Modifier and Type	Field and Description
private String	baseUrl
private int	batchCount
private long	batchDurationMillis
private long	batchSize
private long	cacheExpirationMillis
private boolean	compress
private int	connectTimeoutMillis
private static ConcurrentMap<String,SiteToSiteRestApiClient.RemoteGroupContents>	contentsMap
private org.apache.http.client.CredentialsProvider	credentialsProvider
private static int	DATA_PACKET_CHANNEL_READ_BUFFER_SIZE
private static String	EVENT_CATEGORY
private EventReporter	eventReporter
private static Pattern	HTTP_ABS_URL
private org.apache.http.impl.nio.client.CloseableHttpAsyncClient	httpAsyncClient
private org.apache.http.impl.client.CloseableHttpClient	httpClient
private long	lastPruneTimestamp
private InetAddress	localAddress
private static org.slf4j.Logger	logger
private Future<org.apache.http.HttpResponse>	postResult
protected HttpProxy	proxy
private AtomicBoolean	proxyAuthRequiresResend
private int	readTimeoutMillis
private org.apache.http.client.config.RequestConfig	requestConfig
private long	requestExpirationMillis
private static int	RESPONSE_CODE_ACCEPTED
private static int	RESPONSE_CODE_BAD_REQUEST
private static int	RESPONSE_CODE_CREATED
private static int	RESPONSE_CODE_FORBIDDEN
private static int	RESPONSE_CODE_NOT_FOUND
private static int	RESPONSE_CODE_OK
private int	serverTransactionTtl
protected SSLContext	sslContext
private CountDownLatch	transferDataLatch
private TransportProtocolVersionNegotiator	transportProtocolVersionNegotiator
private String	trustedPeerDn
private ScheduledFuture<?>	ttlExtendingFuture
private ScheduledExecutorService	ttlExtendTaskExecutor

Constructor Summary

Constructors

Constructor and Description
SiteToSiteRestApiClient(SSLContext sslContext, HttpProxy proxy, EventReporter eventReporter)

Method Summary

Modifier and Type	Method and Description
void	close()
private void	closeSilently(Closeable closeable)
TransactionResultEntity	commitReceivingFlowFiles(String transactionUrl, ResponseCode clientResponse, String checksum)
TransactionResultEntity	commitTransferFlowFiles(String transactionUrl, ResponseCode clientResponse)
private org.apache.http.client.methods.HttpDelete	createDelete(String path)
private org.apache.http.client.methods.HttpGet	createGet(String path)
private org.apache.http.client.methods.HttpGet	createGetControllerRequest()
private org.apache.http.client.methods.HttpPost	createPost(String path)
private org.apache.http.client.methods.HttpPut	createPut(String path)
private void	debugProxyAuthState(org.apache.http.protocol.HttpContext context) Print AuthState in HttpContext for debugging purpose.
private String	execute(org.apache.http.client.methods.HttpGet get)
private <T> T	execute(org.apache.http.client.methods.HttpGet get, Class<T> entityClass)
TransactionResultEntity	extendTransaction(String transactionUrl)
private ControllerDTO	fetchController()
void	finishTransferFlowFiles(CommunicationsSession commSession)
String	getBaseUrl()
private ControllerDTO	getController()
ControllerDTO	getController(Set<String> clusterUrls) Try each URL in clusterUrls one by one to get a controller resource from those remote NiFi instances until a controller is successfully returned or try out all URLs.
ControllerDTO	getController(String clusterUrls) Parse the clusterUrls String, and try each URL in clusterUrls one by one to get a controller resource from those remote NiFi instances until a controller is successfully returned or try out all URLs.
private org.apache.http.client.CredentialsProvider	getCredentialsProvider()
static String	getFirstUrl(String clusterUrlStr)
private org.apache.http.impl.nio.client.CloseableHttpAsyncClient	getHttpAsyncClient()
private org.apache.http.impl.client.CloseableHttpClient	getHttpClient()
Collection<PeerDTO>	getPeers()
private org.apache.http.client.config.RequestConfig	getRequestConfig()
Integer	getTransactionProtocolVersion()
String	getTrustedPeerDn()
private URI	getUri(String path)
private IOException	handleErrResponse(int responseCode, InputStream in)
String	initiateTransaction(TransferDirection direction, String portId)
private org.apache.http.HttpResponse	initiateTransactionForReceive(org.apache.http.client.methods.HttpPost post) Initiate a transaction for receiving data.
private org.apache.http.HttpResponse	initiateTransactionForSend(org.apache.http.client.methods.HttpPost post) Initiate a transaction for sending data.
boolean	openConnectionForReceive(String transactionUrl, Peer peer)
void	openConnectionForSend(String transactionUrl, Peer peer)
static Set<String>	parseClusterUrls(String clusterUrlStr) Parse the comma-separated URLs string for the remote NiFi instances.
private void	pruneCache()
private TransactionResultEntity	readResponse(InputStream inputStream)
private String	readTransactionUrl(org.apache.http.HttpResponse response)
private static String	resolveBaseUrl(String clusterUrl)
private static String	resolveBaseUrl(URI clusterUrl) Resolve NiFi API url with leniency.
void	setBaseUrl(String baseUrl) Set the baseUrl as it is, without altering or adjusting the specified url string.
void	setBaseUrl(String scheme, String host, int port)
private void	setBaseUrl(String scheme, String host, int port, String path)
void	setBatchCount(int batchCount)
void	setBatchDurationMillis(long batchDurationMillis)
void	setBatchSize(long batchSize)
void	setCacheExpirationMillis(long expirationMillis)
void	setCompress(boolean compress)
void	setConnectTimeoutMillis(int connectTimeoutMillis)
private void	setHandshakeProperties(org.apache.http.client.methods.HttpRequestBase httpRequest)
void	setLocalAddress(InetAddress localAddress)
void	setReadTimeoutMillis(int readTimeoutMillis)
void	setRequestExpirationMillis(long requestExpirationMillis)
private void	setupAsyncClient()
private void	setupClient()
private void	setupCredentialsProvider()
private void	setupRequestConfig()
private boolean	shouldCheckProxyAuth()
private void	startExtendingTransaction(String transactionUrl)
private void	stopExtendingTransaction()
private IOException	toIOException(ExecutionException e)
private static void	validateUriString(String s)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

EVENT_CATEGORY

private static final String EVENT_CATEGORY

See Also:

Constant Field Values

DATA_PACKET_CHANNEL_READ_BUFFER_SIZE

private static final int DATA_PACKET_CHANNEL_READ_BUFFER_SIZE

See Also:

Constant Field Values

RESPONSE_CODE_OK

private static final int RESPONSE_CODE_OK

See Also:

Constant Field Values

RESPONSE_CODE_CREATED

private static final int RESPONSE_CODE_CREATED

See Also:

Constant Field Values

RESPONSE_CODE_ACCEPTED

private static final int RESPONSE_CODE_ACCEPTED

See Also:

Constant Field Values

RESPONSE_CODE_BAD_REQUEST

private static final int RESPONSE_CODE_BAD_REQUEST

See Also:

Constant Field Values

RESPONSE_CODE_FORBIDDEN

private static final int RESPONSE_CODE_FORBIDDEN

See Also:

Constant Field Values

RESPONSE_CODE_NOT_FOUND

private static final int RESPONSE_CODE_NOT_FOUND

See Also:

Constant Field Values

logger

private static final org.slf4j.Logger logger

baseUrl

private String baseUrl

sslContext

protected final SSLContext sslContext

proxy

protected final HttpProxy proxy

proxyAuthRequiresResend

private final AtomicBoolean proxyAuthRequiresResend

eventReporter

private final EventReporter eventReporter

requestConfig

private org.apache.http.client.config.RequestConfig requestConfig

credentialsProvider

private org.apache.http.client.CredentialsProvider credentialsProvider

httpClient

private org.apache.http.impl.client.CloseableHttpClient httpClient

httpAsyncClient

private org.apache.http.impl.nio.client.CloseableHttpAsyncClient httpAsyncClient

compress

private boolean compress

localAddress

private InetAddress localAddress

requestExpirationMillis

private long requestExpirationMillis

serverTransactionTtl

private int serverTransactionTtl

batchCount

private int batchCount

batchSize

private long batchSize

batchDurationMillis

private long batchDurationMillis

transportProtocolVersionNegotiator

private final TransportProtocolVersionNegotiator transportProtocolVersionNegotiator

trustedPeerDn

private String trustedPeerDn

ttlExtendTaskExecutor

private final ScheduledExecutorService ttlExtendTaskExecutor

ttlExtendingFuture

private ScheduledFuture<?> ttlExtendingFuture

connectTimeoutMillis

private int connectTimeoutMillis

readTimeoutMillis

private int readTimeoutMillis

cacheExpirationMillis

private long cacheExpirationMillis

HTTP_ABS_URL

private static final Pattern HTTP_ABS_URL

postResult

private Future<org.apache.http.HttpResponse> postResult

transferDataLatch

private CountDownLatch transferDataLatch

contentsMap

private static final ConcurrentMap<String,SiteToSiteRestApiClient.RemoteGroupContents> contentsMap

lastPruneTimestamp

private volatile long lastPruneTimestamp

Constructor Detail

SiteToSiteRestApiClient

public SiteToSiteRestApiClient(SSLContext sslContext,
                               HttpProxy proxy,
                               EventReporter eventReporter)

Method Detail

close

public void close()
           throws IOException

Specified by:

close in interface Closeable

Specified by:

close in interface AutoCloseable

Throws:

IOException

getHttpClient

private org.apache.http.impl.client.CloseableHttpClient getHttpClient()

getHttpAsyncClient

private org.apache.http.impl.nio.client.CloseableHttpAsyncClient getHttpAsyncClient()

getRequestConfig

private org.apache.http.client.config.RequestConfig getRequestConfig()

getCredentialsProvider

private org.apache.http.client.CredentialsProvider getCredentialsProvider()

setupRequestConfig

private void setupRequestConfig()

setupCredentialsProvider

private void setupCredentialsProvider()

setupClient

private void setupClient()

setupAsyncClient

private void setupAsyncClient()

getController

public ControllerDTO getController(String clusterUrls)
                            throws IOException

Parse the clusterUrls String, and try each URL in clusterUrls one by one to get a controller resource from those remote NiFi instances until a controller is successfully returned or try out all URLs. After this method execution, the base URL is set with the successful URL.

Parameters:

clusterUrls - url of the remote NiFi instance, multiple urls can be specified in comma-separated format

Throws:

IllegalArgumentException - when it fails to parse the URLs string, URLs string contains multiple protocols (http and https mix), or none of URL is specified.

IOException

getController

public ControllerDTO getController(Set<String> clusterUrls)
                            throws IOException

Try each URL in clusterUrls one by one to get a controller resource from those remote NiFi instances until a controller is successfully returned or try out all URLs. After this method execution, the base URL is set with the successful URL.

Throws:

IOException

getController

private ControllerDTO getController()
                             throws IOException

Throws:

IOException

fetchController

private ControllerDTO fetchController()
                               throws IOException

Throws:

IOException

pruneCache

private void pruneCache()

createGetControllerRequest

private org.apache.http.client.methods.HttpGet createGetControllerRequest()

getPeers

public Collection<PeerDTO> getPeers()
                             throws IOException

Throws:

IOException

initiateTransaction

public String initiateTransaction(TransferDirection direction,
                                  String portId)
                           throws IOException

Throws:

IOException

initiateTransactionForReceive

private org.apache.http.HttpResponse initiateTransactionForReceive(org.apache.http.client.methods.HttpPost post)
                                                            throws IOException

Initiate a transaction for receiving data.

Parameters:

post - a POST request to establish transaction

Returns:

POST request response

Throws:

IOException - thrown if the post request failed

initiateTransactionForSend

private org.apache.http.HttpResponse initiateTransactionForSend(org.apache.http.client.methods.HttpPost post)
                                                         throws IOException

Initiate a transaction for sending data.

If a proxy server requires auth, the proxy server returns 407 response with available auth schema such as basic or digest. Then client has to resend the same request with its credential added. This mechanism is problematic for sending data from NiFi.

In order to resend a POST request with auth param, NiFi has to either read flow-file contents to send again, or keep the POST body somewhere. If we store that in memory, it would causes OOM, or storing it on disk slows down performance. Rolling back processing session would be overkill. Reading flow-file contents only when it's ready to send in a streaming way is ideal.

Additionally, the way proxy authentication is done is vary among Proxy server software. Some requires 407 and resend cycle for every requests, while others keep a connection between a client and the proxy server, then consecutive requests skip auth steps. The problem is, that how should we behave is only told after sending a request to the proxy.

In order to handle above concerns correctly and efficiently, this method do the followings:

1. Send a GET request to controller resource, to initiate an HttpAsyncClient. The instance will be used for further requests. This is not required by the Site-to-Site protocol, but it can setup proxy auth state safely.
2. Send a POST request to initiate a transaction. While doing so, it captures how a proxy server works. If 407 and resend cycle occurs here, it implies that we need to do the same thing again when we actually send the data. Because if the proxy keeps using the same connection and doesn't require an auth step, it doesn't do so here.
3. Then this method stores whether the final POST request should wait for the auth step. So that openConnectionForSend(java.lang.String, org.apache.nifi.remote.Peer) can determine when to produce contents.

The above special sequence is only executed when a proxy instance is set, and its username is set.

Parameters:

post - a POST request to establish transaction

Returns:

POST request response

Throws:

IOException - thrown if the post request failed

debugProxyAuthState

private void debugProxyAuthState(org.apache.http.protocol.HttpContext context)

Print AuthState in HttpContext for debugging purpose.

If the proxy server requires 407 and resend cycle, this method logs as followings, for Basic Auth:

• state:UNCHALLENGED;
• state:CHALLENGED;auth scheme:basic;credentials present

For Digest Auth:

• state:UNCHALLENGED;
• state:CHALLENGED;auth scheme:digest;credentials present

But if the proxy uses the same connection, it doesn't return 407, in such case this method is called only once with:

• state:UNCHALLENGED

toIOException

private IOException toIOException(ExecutionException e)

shouldCheckProxyAuth

private boolean shouldCheckProxyAuth()

openConnectionForReceive

public boolean openConnectionForReceive(String transactionUrl,
                                        Peer peer)
                                 throws IOException

Throws:

IOException

openConnectionForSend

public void openConnectionForSend(String transactionUrl,
                                  Peer peer)
                           throws IOException

Throws:

IOException

finishTransferFlowFiles

public void finishTransferFlowFiles(CommunicationsSession commSession)
                             throws IOException

Throws:

IOException

startExtendingTransaction

private void startExtendingTransaction(String transactionUrl)

closeSilently

private void closeSilently(Closeable closeable)

extendTransaction

public TransactionResultEntity extendTransaction(String transactionUrl)
                                          throws IOException

Throws:

IOException

stopExtendingTransaction

private void stopExtendingTransaction()

handleErrResponse

private IOException handleErrResponse(int responseCode,
                                      InputStream in)
                               throws IOException

Throws:

IOException

readResponse

private TransactionResultEntity readResponse(InputStream inputStream)
                                      throws IOException

Throws:

IOException

readTransactionUrl

private String readTransactionUrl(org.apache.http.HttpResponse response)

setHandshakeProperties

private void setHandshakeProperties(org.apache.http.client.methods.HttpRequestBase httpRequest)

getUri

private URI getUri(String path)

createGet

private org.apache.http.client.methods.HttpGet createGet(String path)

createPost

private org.apache.http.client.methods.HttpPost createPost(String path)

createPut

private org.apache.http.client.methods.HttpPut createPut(String path)

createDelete

private org.apache.http.client.methods.HttpDelete createDelete(String path)

execute

private String execute(org.apache.http.client.methods.HttpGet get)
                throws IOException

Throws:

IOException

execute

private <T> T execute(org.apache.http.client.methods.HttpGet get,
                      Class<T> entityClass)
               throws IOException

Throws:

IOException

getBaseUrl

public String getBaseUrl()

setBaseUrl

public void setBaseUrl(String baseUrl)

Set the baseUrl as it is, without altering or adjusting the specified url string. If the url is specified by user input, and if it needs to be resolved with leniency, then use resolveBaseUrl(String) method before passing it to this method.

Parameters:

baseUrl - url to set

setConnectTimeoutMillis

public void setConnectTimeoutMillis(int connectTimeoutMillis)

setReadTimeoutMillis

public void setReadTimeoutMillis(int readTimeoutMillis)

setCacheExpirationMillis

public void setCacheExpirationMillis(long expirationMillis)

getFirstUrl

public static String getFirstUrl(String clusterUrlStr)

parseClusterUrls

public static Set<String> parseClusterUrls(String clusterUrlStr)

Parse the comma-separated URLs string for the remote NiFi instances.

Returns:

A set containing one or more URLs

Throws:

IllegalArgumentException - when it fails to parse the URLs string, URLs string contains multiple protocols (http and https mix), or none of URL is specified.

validateUriString

private static void validateUriString(String s)

resolveBaseUrl

private static String resolveBaseUrl(String clusterUrl)

resolveBaseUrl

private static String resolveBaseUrl(URI clusterUrl)

Resolve NiFi API url with leniency. This method does following conversion on uri path:

• / to /nifi-api
• /nifi to /nifi-api
• /some/path/ to /some/path/nifi-api

Parameters:

clusterUrl - url to be resolved

Returns:

resolved url

setBaseUrl

public void setBaseUrl(String scheme,
                       String host,
                       int port)

setBaseUrl

private void setBaseUrl(String scheme,
                        String host,
                        int port,
                        String path)

setCompress

public void setCompress(boolean compress)

setLocalAddress

public void setLocalAddress(InetAddress localAddress)

setRequestExpirationMillis

public void setRequestExpirationMillis(long requestExpirationMillis)

setBatchCount

public void setBatchCount(int batchCount)

setBatchSize

public void setBatchSize(long batchSize)

setBatchDurationMillis

public void setBatchDurationMillis(long batchDurationMillis)

getTransactionProtocolVersion

public Integer getTransactionProtocolVersion()

getTrustedPeerDn

public String getTrustedPeerDn()

commitReceivingFlowFiles

public TransactionResultEntity commitReceivingFlowFiles(String transactionUrl,
                                                        ResponseCode clientResponse,
                                                        String checksum)
                                                 throws IOException

Throws:

IOException

commitTransferFlowFiles

public TransactionResultEntity commitTransferFlowFiles(String transactionUrl,
                                                       ResponseCode clientResponse)
                                                throws IOException

Throws:

IOException