Package net.snowflake.ingest.utils

Class ParameterProvider

  • public class ParameterProvider
extends Object
    Utility class to provide configurable constants

    • Field Detail

      • BUFFER_FLUSH_INTERVAL_IN_MILLIS

        public static final String BUFFER_FLUSH_INTERVAL_IN_MILLIS

      • BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS

        public static final String BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS

      • INSERT_THROTTLE_INTERVAL_IN_MILLIS

        public static final String INSERT_THROTTLE_INTERVAL_IN_MILLIS

      • INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE

        public static final String INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE

      • INSERT_THROTTLE_THRESHOLD_IN_BYTES

        public static final String INSERT_THROTTLE_THRESHOLD_IN_BYTES

      • ENABLE_SNOWPIPE_STREAMING_METRICS

        public static final String ENABLE_SNOWPIPE_STREAMING_METRICS

      • BLOB_FORMAT_VERSION

        public static final String BLOB_FORMAT_VERSION

      • IO_TIME_CPU_RATIO

        public static final String IO_TIME_CPU_RATIO

      • BLOB_UPLOAD_MAX_RETRY_COUNT

        public static final String BLOB_UPLOAD_MAX_RETRY_COUNT

      • MAX_MEMORY_LIMIT_IN_BYTES

        public static final String MAX_MEMORY_LIMIT_IN_BYTES

      • ENABLE_PARQUET_INTERNAL_BUFFERING

        public static final String ENABLE_PARQUET_INTERNAL_BUFFERING

      • MAX_CHANNEL_SIZE_IN_BYTES

        public static final String MAX_CHANNEL_SIZE_IN_BYTES

      • MAX_CHUNK_SIZE_IN_BYTES

        public static final String MAX_CHUNK_SIZE_IN_BYTES

      • MAX_ALLOWED_ROW_SIZE_IN_BYTES

        public static final String MAX_ALLOWED_ROW_SIZE_IN_BYTES

      • MAX_CHUNKS_IN_BLOB

        public static final String MAX_CHUNKS_IN_BLOB

      • MAX_CHUNKS_IN_REGISTRATION_REQUEST

        public static final String MAX_CHUNKS_IN_REGISTRATION_REQUEST

      • MAX_CLIENT_LAG

        public static final String MAX_CLIENT_LAG

      • BDEC_PARQUET_COMPRESSION_ALGORITHM

        public static final String BDEC_PARQUET_COMPRESSION_ALGORITHM

      • BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS_DEFAULT

        public static final long BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS_DEFAULT
        See Also:
        Constant Field Values

      • INSERT_THROTTLE_INTERVAL_IN_MILLIS_DEFAULT

        public static final long INSERT_THROTTLE_INTERVAL_IN_MILLIS_DEFAULT
        See Also:
        Constant Field Values

      • INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE_DEFAULT

        public static final int INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE_DEFAULT
        See Also:
        Constant Field Values

      • INSERT_THROTTLE_THRESHOLD_IN_BYTES_DEFAULT

        public static final int INSERT_THROTTLE_THRESHOLD_IN_BYTES_DEFAULT
        See Also:
        Constant Field Values

      • SNOWPIPE_STREAMING_METRICS_DEFAULT

        public static final boolean SNOWPIPE_STREAMING_METRICS_DEFAULT
        See Also:
        Constant Field Values

      • IO_TIME_CPU_RATIO_DEFAULT

        public static final int IO_TIME_CPU_RATIO_DEFAULT
        See Also:
        Constant Field Values

      • BLOB_UPLOAD_MAX_RETRY_COUNT_DEFAULT

        public static final int BLOB_UPLOAD_MAX_RETRY_COUNT_DEFAULT
        See Also:
        Constant Field Values

      • MAX_MEMORY_LIMIT_IN_BYTES_DEFAULT

        public static final long MAX_MEMORY_LIMIT_IN_BYTES_DEFAULT
        See Also:
        Constant Field Values

      • MAX_CHANNEL_SIZE_IN_BYTES_DEFAULT

        public static final long MAX_CHANNEL_SIZE_IN_BYTES_DEFAULT
        See Also:
        Constant Field Values

      • MAX_CHUNK_SIZE_IN_BYTES_DEFAULT

        public static final long MAX_CHUNK_SIZE_IN_BYTES_DEFAULT
        See Also:
        Constant Field Values

      • MAX_CLIENT_LAG_DEFAULT

        public static final long MAX_CLIENT_LAG_DEFAULT
        See Also:
        Constant Field Values

      • MAX_CLIENT_LAG_ICEBERG_MODE_DEFAULT

        public static final long MAX_CLIENT_LAG_ICEBERG_MODE_DEFAULT
        See Also:
        Constant Field Values

      • MAX_ALLOWED_ROW_SIZE_IN_BYTES_DEFAULT

        public static final long MAX_ALLOWED_ROW_SIZE_IN_BYTES_DEFAULT
        See Also:
        Constant Field Values

      • MAX_CHUNKS_IN_BLOB_DEFAULT

        public static final int MAX_CHUNKS_IN_BLOB_DEFAULT
        See Also:
        Constant Field Values

      • MAX_CHUNKS_IN_REGISTRATION_REQUEST_DEFAULT

        public static final int MAX_CHUNKS_IN_REGISTRATION_REQUEST_DEFAULT
        See Also:
        Constant Field Values

      • MAX_CHUNKS_IN_BLOB_ICEBERG_MODE_DEFAULT

        public static final int MAX_CHUNKS_IN_BLOB_ICEBERG_MODE_DEFAULT
        See Also:
        Constant Field Values

      • ENABLE_PARQUET_INTERNAL_BUFFERING_DEFAULT

        public static final boolean ENABLE_PARQUET_INTERNAL_BUFFERING_DEFAULT
        See Also:
        Constant Field Values

    • Constructor Detail

      • ParameterProvider

        public ParameterProvider​(Map<String,​Object> parameterOverrides,
                         Properties props,
                         boolean isIcebergMode)
        Constructor. Takes properties from profile file and properties from client constructor and resolves final parameter value
        Parameters:
        parameterOverrides - Map of parameter name to value
        props - Properties from profile file
        isIcebergMode - If the provided parameters need to be verified and modified to meet Iceberg mode

      • ParameterProvider

        public ParameterProvider​(boolean isIcebergMode)
        Empty constructor for tests

    • Method Detail

      • getCachedMaxClientLagInMs

        public long getCachedMaxClientLagInMs()
        Returns:
        Longest interval in milliseconds between buffer flushes

      • getBufferFlushCheckIntervalInMs

        public long getBufferFlushCheckIntervalInMs()
        Returns:
        Time in milliseconds between checks to see if the buffer should be flushed

      • getInsertThrottleIntervalInMs

        public long getInsertThrottleIntervalInMs()
        Returns:
        Duration in milliseconds to delay data insertion to the buffer when throttled

      • getInsertThrottleThresholdInPercentage

        public int getInsertThrottleThresholdInPercentage()
        Returns:
        Percent of free total memory at which we throttle row inserts

      • getInsertThrottleThresholdInBytes

        public int getInsertThrottleThresholdInBytes()
        Returns:
        Absolute size in bytes of free total memory at which we throttle row inserts

      • hasEnabledSnowpipeStreamingMetrics

        public boolean hasEnabledSnowpipeStreamingMetrics()
        Returns:
        true if jmx metrics are enabled for a client

      • getBlobFormatVersion

        public Constants.BdecVersion getBlobFormatVersion()
        Returns:
        Blob format version

      • getIOTimeCpuRatio

        public int getIOTimeCpuRatio()
        Returns:
        the IO_TIME/CPU ratio that we will use to determine the number of buildAndUpload threads

      • getBlobUploadMaxRetryCount

        public int getBlobUploadMaxRetryCount()
        Returns:
        the max retry count when waiting for a blob upload task to finish

      • getMaxMemoryLimitInBytes

        public long getMaxMemoryLimitInBytes()
        Returns:
        The max memory limit in bytes

      • getEnableParquetInternalBuffering

        public boolean getEnableParquetInternalBuffering()
        Returns:
        Return whether memory optimization for Parquet is enabled.

      • getMaxChannelSizeInBytes

        public long getMaxChannelSizeInBytes()
        Returns:
        The max channel size in bytes

      • getMaxChunkSizeInBytes

        public long getMaxChunkSizeInBytes()
        Returns:
        The max chunk size in bytes that could avoid OOM at server side

      • getMaxAllowedRowSizeInBytes

        public long getMaxAllowedRowSizeInBytes()

      • getMaxChunksInBlob

        public int getMaxChunksInBlob()
        Returns:
        The max number of chunks that can be put into a single BDEC.

      • getMaxChunksInRegistrationRequest

        public int getMaxChunksInRegistrationRequest()
        Returns:
        The max number of chunks that can be put into a single blob registration request.