Packages

o

ai.h2o.sparkling.ml.utils

SchemaUtils

object SchemaUtils

Utilities for working with Spark SQL component.

Linear Supertypes
AnyRef, Any
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. SchemaUtils
  2. AnyRef
  3. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. All

Type Members

  1. case class FieldWithOrder(field: StructField, order: Iterable[Any]) extends Product with Serializable

Value Members

  1. final def !=(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  2. final def ##(): Int
    Definition Classes
    AnyRef → Any
  3. final def ==(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  4. def appendFlattenedStructsToDataFrame(df: DataFrame, prefixForNewColumns: String): DataFrame
  5. final def asInstanceOf[T0]: T0
    Definition Classes
    Any
  6. def clone(): AnyRef
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native()
  7. def collectMaxElementSizes(rdd: RDD[Row], schema: StructType): Array[Int]

    Collect max size of each element in DataFrame.

    Collect max size of each element in DataFrame. For array -> max array size For vectors -> max vector size For simple types -> 1

    returns

    array containing size of each element

  8. def collectVectorLikeTypes(flatSchema: StructType): Seq[Int]
  9. final def eq(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  10. def equals(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  11. def expandedSchema(flatSchema: StructType, elemMaxSizes: Array[Int]): Seq[StructField]

    Returns expanded schema

    Returns expanded schema

    • schema is represented as list of types
    • all arrays are expanded into columns based on the longest one
    • all vectors are expanded into columns based on the longest one
    flatSchema

    flat schema of spark data frame

    returns

    list of types with their positions

  12. def finalize(): Unit
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( classOf[java.lang.Throwable] )
  13. def flattenDataFrame(df: DataFrame, flatSchema: StructType): DataFrame
  14. def flattenDataFrame(df: DataFrame): DataFrame
  15. def flattenSchema(df: DataFrame): StructType
  16. def flattenStructsInDataFrame(df: DataFrame): DataFrame
  17. def flattenStructsInSchema(schema: StructType, sourceColPrefix: Option[String] = None, targetColPrefix: Option[String] = None, nullable: Boolean = false): Seq[(StructField, String)]
  18. final def getClass(): Class[_]
    Definition Classes
    AnyRef → Any
    Annotations
    @native()
  19. def hashCode(): Int
    Definition Classes
    AnyRef → Any
    Annotations
    @native()
  20. final def isInstanceOf[T0]: Boolean
    Definition Classes
    Any
  21. final def ne(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  22. final def notify(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native()
  23. final def notifyAll(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native()
  24. def rowsToRowSchemas(df: DataFrame): Dataset[ArrayBuffer[FieldWithOrder]]
  25. final def synchronized[T0](arg0: ⇒ T0): T0
    Definition Classes
    AnyRef
  26. def toString(): String
    Definition Classes
    AnyRef → Any
  27. final def wait(): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  28. final def wait(arg0: Long, arg1: Int): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  29. final def wait(arg0: Long): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native()

Inherited from AnyRef

Inherited from Any

Ungrouped