org.apache.druid.indexer

Class Utils

java.lang.Object

org.apache.druid.indexer.Utils

public class Utils
extends Object

Constructor Summary

Constructors

Constructor and Description
Utils()

Method Summary

Modifier and Type	Method and Description
static boolean	checkAppSuccessForJobIOException(IOException ioe, org.apache.hadoop.mapreduce.Job job, boolean useYarnRMJobStatusFallback) It is possible for a Hadoop Job to succeed, but for `job.waitForCompletion()` to fail because of issues with the JobHistory server.
static boolean	checkAppSuccessFromYarnRM(org.apache.hadoop.mapreduce.Job job)
static boolean	exists(org.apache.hadoop.mapreduce.JobContext job, org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path inputPath)
static String	getFailureMessage(org.apache.hadoop.mapreduce.Job failedJob, com.fasterxml.jackson.databind.ObjectMapper jsonMapper)
static Map<String,Object>	getStats(org.apache.hadoop.mapreduce.JobContext job, org.apache.hadoop.fs.Path statsPath)
static OutputStream	makePathAndOutputStream(org.apache.hadoop.mapreduce.JobContext job, org.apache.hadoop.fs.Path outputPath, boolean deleteExisting)
static InputStream	openInputStream(org.apache.hadoop.mapreduce.JobContext job, org.apache.hadoop.fs.Path inputPath)
static InputStream	openInputStream(org.apache.hadoop.mapreduce.JobContext job, org.apache.hadoop.fs.Path inputPath, org.apache.hadoop.fs.FileSystem fileSystem)
static void	storeStats(org.apache.hadoop.mapreduce.JobContext job, org.apache.hadoop.fs.Path path, Map<String,Object> stats)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

Utils

public Utils()

Method Detail

makePathAndOutputStream

public static OutputStream makePathAndOutputStream(org.apache.hadoop.mapreduce.JobContext job,
                                                   org.apache.hadoop.fs.Path outputPath,
                                                   boolean deleteExisting)
                                            throws IOException

Throws:

IOException

openInputStream

public static InputStream openInputStream(org.apache.hadoop.mapreduce.JobContext job,
                                          org.apache.hadoop.fs.Path inputPath)
                                   throws IOException

Throws:

IOException

exists

public static boolean exists(org.apache.hadoop.mapreduce.JobContext job,
                             org.apache.hadoop.fs.FileSystem fs,
                             org.apache.hadoop.fs.Path inputPath)
                      throws IOException

Throws:

IOException

openInputStream

public static InputStream openInputStream(org.apache.hadoop.mapreduce.JobContext job,
                                          org.apache.hadoop.fs.Path inputPath,
                                          org.apache.hadoop.fs.FileSystem fileSystem)
                                   throws IOException

Throws:

IOException

getStats

public static Map<String,Object> getStats(org.apache.hadoop.mapreduce.JobContext job,
                                          org.apache.hadoop.fs.Path statsPath)
                                   throws IOException

Throws:

IOException

storeStats

public static void storeStats(org.apache.hadoop.mapreduce.JobContext job,
                              org.apache.hadoop.fs.Path path,
                              Map<String,Object> stats)
                       throws IOException

Throws:

IOException

getFailureMessage

@Nullable
public static String getFailureMessage(org.apache.hadoop.mapreduce.Job failedJob,
                                                 com.fasterxml.jackson.databind.ObjectMapper jsonMapper)

checkAppSuccessForJobIOException

public static boolean checkAppSuccessForJobIOException(IOException ioe,
                                                       org.apache.hadoop.mapreduce.Job job,
                                                       boolean useYarnRMJobStatusFallback)

It is possible for a Hadoop Job to succeed, but for `job.waitForCompletion()` to fail because of issues with the JobHistory server. When the JobHistory server is unavailable, it's possible to fetch the application's status from the YARN ResourceManager instead. Returns true if both `useYarnRMJobStatusFallback` is enabled and YARN ResourceManager reported success for the target job.

checkAppSuccessFromYarnRM

public static boolean checkAppSuccessFromYarnRM(org.apache.hadoop.mapreduce.Job job)