trait SparkJob extends JobBase
Linear Supertypes
Known Subclasses
Ordering
- Alphabetic
- By Inheritance
Inherited
- SparkJob
- JobBase
- StrictLogging
- AnyRef
- Any
- Hide All
- Show All
Visibility
- Public
- All
Abstract Value Members
-
abstract
def
name: String
- Definition Classes
- JobBase
-
abstract
def
run(): Try[JobResult]
Just to force any job to implement its entry point using within the "run" method
Just to force any job to implement its entry point using within the "run" method
- returns
: Spark Dataframe for Spark Jobs None otherwise
- Definition Classes
- JobBase
-
implicit abstract
def
settings: Settings
- Definition Classes
- JobBase
Concrete Value Members
-
final
def
!=(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
final
def
##(): Int
- Definition Classes
- AnyRef → Any
-
final
def
==(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
def
analyze(fullTableName: String): Any
- Attributes
- protected
-
final
def
asInstanceOf[T0]: T0
- Definition Classes
- Any
-
def
clone(): AnyRef
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()
-
def
createSparkViews(views: Views, sqlParameters: Map[String, String]): Unit
- Attributes
- protected
-
final
def
eq(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
def
equals(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
def
finalize(): Unit
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( classOf[java.lang.Throwable] )
-
final
def
getClass(): Class[_]
- Definition Classes
- AnyRef → Any
- Annotations
- @native()
-
def
hashCode(): Int
- Definition Classes
- AnyRef → Any
- Annotations
- @native()
-
final
def
isInstanceOf[T0]: Boolean
- Definition Classes
- Any
-
val
logger: Logger
- Attributes
- protected
- Definition Classes
- StrictLogging
-
final
def
ne(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
final
def
notify(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
-
final
def
notifyAll(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
-
def
parseViewDefinition(valueWithEnv: String): (SinkType, Option[JdbcConfigName], String)
- valueWithEnv
in the form [SinkType:[configName:]]viewName
- returns
(SinkType, configName, viewName)
- Attributes
- protected
- Definition Classes
- JobBase
-
def
partitionDataset(dataset: DataFrame, partition: List[String]): DataFrame
- Attributes
- protected
-
def
partitionedDatasetWriter(dataset: DataFrame, partition: List[String]): DataFrameWriter[Row]
Partition a dataset using dataset columns.
Partition a dataset using dataset columns. To partition the dataset using the ingestion time, use the reserved column names :
- comet_date
- comet_year
- comet_month
- comet_day
- comet_hour
- comet_minute These columns are renamed to "date", "year", "month", "day", "hour", "minute" in the dataset and their values is set to the current date/time.
- dataset
: Input dataset
- partition
: list of columns to use for partitioning.
- returns
The Spark session used to run this job
- Attributes
- protected
-
def
registerUdf(udf: String): Unit
- Attributes
- protected
- lazy val session: SparkSession
- lazy val sparkEnv: SparkEnv
-
final
def
synchronized[T0](arg0: ⇒ T0): T0
- Definition Classes
- AnyRef
-
def
toString(): String
- Definition Classes
- AnyRef → Any
-
final
def
wait(): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long, arg1: Int): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()