Packages

c

org.apache.spark.sql.connect.client.arrow

ConcatenatingArrowStreamReader

class ConcatenatingArrowStreamReader extends ArrowReader

An ArrowReader that concatenates multiple MessageIterators into a single stream. Each iterator represents a single IPC stream. The concatenated streams all must have the same schema. If the schema is different an exception is thrown.

In some cases we want to retain the messages (see SparkResult). Normally a stream reader closes its messages when it consumes them. In order to prevent that from happening in non-destructive mode we clone the messages before passing them to the reading logic.

Linear Supertypes
ArrowReader, AutoCloseable, DictionaryProvider, AnyRef, Any
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. ConcatenatingArrowStreamReader
  2. ArrowReader
  3. AutoCloseable
  4. DictionaryProvider
  5. AnyRef
  6. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. All

Instance Constructors

  1. new ConcatenatingArrowStreamReader(allocator: BufferAllocator, input: Iterator[AbstractMessageIterator], destructive: Boolean)

Value Members

  1. final def !=(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  2. final def ##(): Int
    Definition Classes
    AnyRef → Any
  3. final def ==(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  4. final def asInstanceOf[T0]: T0
    Definition Classes
    Any
  5. def bytesRead(): Long
    Definition Classes
    ConcatenatingArrowStreamReader → ArrowReader
  6. def clone(): AnyRef
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native()
  7. def close(arg0: Boolean): Unit
    Definition Classes
    ArrowReader
    Annotations
    @throws( classOf[java.io.IOException] )
  8. def close(): Unit
    Definition Classes
    ArrowReader → AutoCloseable
    Annotations
    @throws( classOf[java.io.IOException] )
  9. def closeReadSource(): Unit
    Definition Classes
    ConcatenatingArrowStreamReader → ArrowReader
  10. def ensureInitialized(): Unit
    Attributes
    protected[ipc]
    Definition Classes
    ArrowReader
    Annotations
    @throws( classOf[java.io.IOException] )
  11. final def eq(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  12. def equals(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  13. def finalize(): Unit
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( classOf[java.lang.Throwable] )
  14. final def getClass(): Class[_]
    Definition Classes
    AnyRef → Any
    Annotations
    @native()
  15. def getDictionaryIds(): Set[Long]
    Definition Classes
    ArrowReader → DictionaryProvider
  16. def getDictionaryVectors(): Map[Long, Dictionary]
    Definition Classes
    ArrowReader
    Annotations
    @throws( classOf[java.io.IOException] )
  17. def getVectorSchemaRoot(): VectorSchemaRoot
    Definition Classes
    ArrowReader
    Annotations
    @throws( classOf[java.io.IOException] )
  18. def hashCode(): Int
    Definition Classes
    AnyRef → Any
    Annotations
    @native()
  19. def initialize(): Unit
    Attributes
    protected[ipc]
    Definition Classes
    ArrowReader
    Annotations
    @throws( classOf[java.io.IOException] )
  20. final def isInstanceOf[T0]: Boolean
    Definition Classes
    Any
  21. def loadDictionary(arg0: ArrowDictionaryBatch): Unit
    Attributes
    protected[ipc]
    Definition Classes
    ArrowReader
  22. def loadNextBatch(): Boolean
    Definition Classes
    ConcatenatingArrowStreamReader → ArrowReader
  23. def loadRecordBatch(arg0: ArrowRecordBatch): Unit
    Attributes
    protected[ipc]
    Definition Classes
    ArrowReader
  24. def lookup(arg0: Long): Dictionary
    Definition Classes
    ArrowReader → DictionaryProvider
  25. final def ne(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  26. final def notify(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native()
  27. final def notifyAll(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native()
  28. def prepareLoadNextBatch(): Unit
    Attributes
    protected[ipc]
    Definition Classes
    ArrowReader
    Annotations
    @throws( classOf[java.io.IOException] )
  29. def readSchema(): Schema
    Attributes
    protected
    Definition Classes
    ConcatenatingArrowStreamReader → ArrowReader
  30. final def synchronized[T0](arg0: ⇒ T0): T0
    Definition Classes
    AnyRef
  31. def toString(): String
    Definition Classes
    AnyRef → Any
  32. final def wait(): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  33. final def wait(arg0: Long, arg1: Int): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  34. final def wait(arg0: Long): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native()

Inherited from ArrowReader

Inherited from AutoCloseable

Inherited from DictionaryProvider

Inherited from AnyRef

Inherited from Any

Ungrouped