case class AutoTaskJob(name: String, defaultArea: StorageArea, format: Option[String], coalesce: Boolean, udf: Option[String], views: Views, engine: Engine, task: AutoTaskDesc, sqlParameters: Map[String, String])(implicit settings: Settings, storageHandler: StorageHandler, schemaHandler: SchemaHandler) extends SparkJob with Product with Serializable
Execute the SQL Task and store it in parquet/orc/.... If Hive support is enabled, also store it as a Hive Table. If analyze support is active, also compute basic statistics for twhe dataset.
- name
: Job Name as defined in the YML job description file
- defaultArea
: Where the resulting dataset is stored by default if not specified in the task
- task
: Task to run
- sqlParameters
: Sql Parameters to pass to SQL statements
- Alphabetic
- By Inheritance
- AutoTaskJob
- Serializable
- Serializable
- Product
- Equals
- SparkJob
- JobBase
- StrictLogging
- AnyRef
- Any
- Hide All
- Show All
- Public
- All
Instance Constructors
-
new
AutoTaskJob(name: String, defaultArea: StorageArea, format: Option[String], coalesce: Boolean, udf: Option[String], views: Views, engine: Engine, task: AutoTaskDesc, sqlParameters: Map[String, String])(implicit settings: Settings, storageHandler: StorageHandler, schemaHandler: SchemaHandler)
- name
: Job Name as defined in the YML job description file
- defaultArea
: Where the resulting dataset is stored by default if not specified in the task
- task
: Task to run
- sqlParameters
: Sql Parameters to pass to SQL statements
Value Members
-
final
def
!=(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
final
def
##(): Int
- Definition Classes
- AnyRef → Any
-
final
def
==(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
def
analyze(fullTableName: String): Any
- Attributes
- protected
- Definition Classes
- SparkJob
-
final
def
asInstanceOf[T0]: T0
- Definition Classes
- Any
- def buildQueryBQ(): (List[String], String, List[String])
- def buildQuerySpark(): (List[String], String, List[String])
-
def
clone(): AnyRef
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()
- val coalesce: Boolean
- val createDisposition: String
-
def
createSparkViews(views: Views, sqlParameters: Map[String, String]): Unit
- Attributes
- protected
- Definition Classes
- SparkJob
- val defaultArea: StorageArea
- val engine: Engine
-
final
def
eq(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
def
finalize(): Unit
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( classOf[java.lang.Throwable] )
- val format: Option[String]
-
final
def
getClass(): Class[_]
- Definition Classes
- AnyRef → Any
- Annotations
- @native()
-
final
def
isInstanceOf[T0]: Boolean
- Definition Classes
- Any
-
val
logger: Logger
- Attributes
- protected
- Definition Classes
- StrictLogging
-
val
name: String
- Definition Classes
- AutoTaskJob → JobBase
-
final
def
ne(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
final
def
notify(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
-
final
def
notifyAll(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
-
def
parseViewDefinition(valueWithEnv: String): (SinkType, Option[JdbcConfigName], String)
- valueWithEnv
in the form [SinkType:[configName:]]viewName
- returns
(SinkType, configName, viewName)
- Attributes
- protected
- Definition Classes
- JobBase
-
def
partitionDataset(dataset: DataFrame, partition: List[String]): DataFrame
- Attributes
- protected
- Definition Classes
- SparkJob
-
def
partitionedDatasetWriter(dataset: DataFrame, partition: List[String]): DataFrameWriter[Row]
Partition a dataset using dataset columns.
Partition a dataset using dataset columns. To partition the dataset using the ingestion time, use the reserved column names :
- comet_date
- comet_year
- comet_month
- comet_day
- comet_hour
- comet_minute These columns are renamed to "date", "year", "month", "day", "hour", "minute" in the dataset and their values is set to the current date/time.
- dataset
: Input dataset
- partition
: list of columns to use for partitioning.
- returns
The Spark session used to run this job
- Attributes
- protected
- Definition Classes
- SparkJob
-
def
registerUdf(udf: String): Unit
- Attributes
- protected
- Definition Classes
- SparkJob
-
def
run(): Try[JobResult]
Just to force any job to implement its entry point using within the "run" method
Just to force any job to implement its entry point using within the "run" method
- returns
: Spark Dataframe for Spark Jobs None otherwise
- Definition Classes
- AutoTaskJob → JobBase
- def runBQ(): Try[JobResult]
- def runSpark(): Try[SparkJobResult]
- def runView(viewName: String, viewDir: Option[String], viewCount: Int): Try[JobResult]
-
lazy val
session: SparkSession
- Definition Classes
- SparkJob
-
implicit
val
settings: Settings
- Definition Classes
- AutoTaskJob → JobBase
-
lazy val
sparkEnv: SparkEnv
- Definition Classes
- SparkJob
- val sqlParameters: Map[String, String]
-
final
def
synchronized[T0](arg0: ⇒ T0): T0
- Definition Classes
- AnyRef
- val task: AutoTaskDesc
- val udf: Option[String]
- val views: Views
-
final
def
wait(): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long, arg1: Int): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()
- val writeDisposition: String