case class Schema(name: String, pattern: Pattern, attributes: List[Attribute], metadata: Option[Metadata], merge: Option[MergeOptions], comment: Option[String], presql: Option[List[String]], postsql: Option[List[String]] = None, tags: Option[Set[String]] = None, rls: Option[List[RowLevelSecurity]] = None, assertions: Option[Map[String, String]] = None, primaryKey: Option[List[String]] = None) extends Product with Serializable
Dataset Schema
- name
: Schema name, must be unique among all the schemas belonging to the same domain. Will become the hive table name On Premise or BigQuery Table name on GCP.
- pattern
: filename pattern to which this schema must be applied. This instructs the framework to use this schema to parse any file with a filename that match this pattern.
- attributes
: Attributes parsing rules. See :ref:
attribute_concept- metadata
: Dataset metadata See :ref:
metadata_concept- comment
: free text
- presql
: Reserved for future use.
- postsql
: We use this attribute to execute sql queries before writing the final dataFrame after ingestion
- tags
: Set of string to attach to this Schema
- rls
: Experimental. Row level security to this to this schema. See :ref:
rowlevelsecurity_concept
- Alphabetic
- By Inheritance
- Schema
- Serializable
- Serializable
- Product
- Equals
- AnyRef
- Any
- Hide All
- Show All
- Public
- All
Instance Constructors
-
new
Schema(name: String, pattern: Pattern, attributes: List[Attribute], metadata: Option[Metadata], merge: Option[MergeOptions], comment: Option[String], presql: Option[List[String]], postsql: Option[List[String]] = None, tags: Option[Set[String]] = None, rls: Option[List[RowLevelSecurity]] = None, assertions: Option[Map[String, String]] = None, primaryKey: Option[List[String]] = None)
- name
: Schema name, must be unique among all the schemas belonging to the same domain. Will become the hive table name On Premise or BigQuery Table name on GCP.
- pattern
: filename pattern to which this schema must be applied. This instructs the framework to use this schema to parse any file with a filename that match this pattern.
- attributes
: Attributes parsing rules. See :ref:
attribute_concept- metadata
: Dataset metadata See :ref:
metadata_concept- comment
: free text
- presql
: Reserved for future use.
- postsql
: We use this attribute to execute sql queries before writing the final dataFrame after ingestion
- tags
: Set of string to attach to this Schema
- rls
: Experimental. Row level security to this to this schema. See :ref:
rowlevelsecurity_concept
Value Members
-
final
def
!=(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
final
def
##(): Int
- Definition Classes
- AnyRef → Any
-
final
def
==(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
- def asDot(domain: String, includeAllAttrs: Boolean): String
-
final
def
asInstanceOf[T0]: T0
- Definition Classes
- Any
- val assertions: Option[Map[String, String]]
- val attributes: List[Attribute]
-
lazy val
attributesWithoutScript: List[Attribute]
- Annotations
- @JsonIgnore()
- def attributesWithoutScriptedFields(): List[Attribute]
- def bqSchema(schemaHandler: SchemaHandler): google.cloud.bigquery.Schema
-
def
checkValidity(domainMetaData: Option[Metadata], schemaHandler: SchemaHandler): Either[List[String], Boolean]
Check attribute definition correctness :
Check attribute definition correctness :
- schema name should be a valid table identifier
- attribute name should be a valid Hive column identifier
- attribute name can occur only once in the schema
- returns
error list or true
-
def
clone(): AnyRef
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()
- val comment: Option[String]
- def continuousAttrs(schemaHandler: SchemaHandler): List[Attribute]
- def discreteAttrs(schemaHandler: SchemaHandler): List[Attribute]
-
final
def
eq(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
def
finalSparkSchema(schemaHandler: SchemaHandler): StructType
This Schema as a Spark Catalyst Schema, with renamed attributes
This Schema as a Spark Catalyst Schema, with renamed attributes
- returns
Spark Catalyst Schema
-
def
finalize(): Unit
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( classOf[java.lang.Throwable] )
-
final
def
getClass(): Class[_]
- Definition Classes
- AnyRef → Any
- Annotations
- @native()
-
final
def
isInstanceOf[T0]: Boolean
- Definition Classes
- Any
- def mapping(template: Option[String], domainName: String, schemaHandler: SchemaHandler)(implicit settings: Settings): String
- val merge: Option[MergeOptions]
- def mergedMetadata(domainMetadata: Option[Metadata]): Metadata
- val metadata: Option[Metadata]
- val name: String
-
final
def
ne(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
final
def
notify(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
-
final
def
notifyAll(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
- val pattern: Pattern
- val postsql: Option[List[String]]
- val presql: Option[List[String]]
- val primaryKey: Option[List[String]]
-
def
renamedAttributes(): List[(String, String)]
return the list of renamed attributes
return the list of renamed attributes
- returns
list of tuples (oldname, newname)
- val rls: Option[List[RowLevelSecurity]]
-
def
sparkSchema(schemaHandler: SchemaHandler): StructType
This Schema as a Spark Catalyst Schema
This Schema as a Spark Catalyst Schema
- returns
Spark Catalyst Schema
-
def
sparkSchemaWithoutScriptedFields(schemaHandler: SchemaHandler): StructType
This Schema as a Spark Catalyst Schema, without scripted fields
This Schema as a Spark Catalyst Schema, without scripted fields
- returns
Spark Catalyst Schema
-
final
def
synchronized[T0](arg0: ⇒ T0): T0
- Definition Classes
- AnyRef
- val tags: Option[Set[String]]
-
def
validatePartitionColumns(): Boolean
- returns
Are the parittions columns defined in the metadata valid column names
-
final
def
wait(): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long, arg1: Int): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()