Packages

object HdfsUtils extends Logging

Linear Supertypes
Logging, AnyRef, Any
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. HdfsUtils
  2. Logging
  3. AnyRef
  4. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. All

Type Members

  1. implicit class StringPrefix extends AnyRef

Value Members

  1. final def !=(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  2. final def ##(): Int
    Definition Classes
    AnyRef → Any
  3. final def ==(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  4. final def asInstanceOf[T0]: T0
    Definition Classes
    Any
  5. def backupFiles(fs: FileSystem)(filesToBackup: Seq[Path], backupParentDir: Path, dataPath: Path): Try[Path]

    Create a new directory inside backupParentDir, called "backup_{randomUUID}".

    Create a new directory inside backupParentDir, called "backup_{randomUUID}". Each of the files inside filesToBackup will be copied in this directory, also maintaining the eventual HDFS partitioning. The new file path is created by removing the base directory (that is dataPath) from the file path, and replacing it with the path of the backup directory. Example: filesToBackup = ["/user/data/p1=a/p2=b/file.parquet"] backupParentDir = "/user" dataPath = "/user/data"

    • This function creates: backupDir = "/user/backup_123'
    • then it copies the file into this directory, replacing the prefix "/user/data" with "/user/backup_123": "/user/backup_123/p1=a/p2=b/file.parquet"
    filesToBackup

    Files that should be copied in the backup directory

    backupParentDir

    Base path where to create the backup directory

    dataPath

    Path containing the data that will be backup

    returns

    Path of the newly created backup directory

  6. def clone(): AnyRef
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native() @HotSpotIntrinsicCandidate()
  7. def deletePath(fs: FileSystem)(sourcePath: Path): Try[Unit]
  8. final def eq(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  9. def equals(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  10. def findPartitionColumns(uri: String): List[(String, String)]
  11. def foldIterator[T, B](iterator: RemoteIterator[T], acc: Try[B])(f: (B, T) ⇒ B)(exitPath: (B) ⇒ Boolean): Try[B]
  12. final def getClass(): Class[_]
    Definition Classes
    AnyRef → Any
    Annotations
    @native() @HotSpotIntrinsicCandidate()
  13. def getRawModelPathToToLoad(rawModel: RawModel, sc: SparkContext): String
  14. def getRawModelPathToWrite(rawModel: RawModel): String
  15. def hashCode(): Int
    Definition Classes
    AnyRef → Any
    Annotations
    @native() @HotSpotIntrinsicCandidate()
  16. final def isInstanceOf[T0]: Boolean
    Definition Classes
    Any
  17. val logger: WaspLogger
    Attributes
    protected
    Definition Classes
    Logging
  18. final def ne(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  19. final def notify(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native() @HotSpotIntrinsicCandidate()
  20. final def notifyAll(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native() @HotSpotIntrinsicCandidate()
  21. def readRawModel(rawModel: RawModel, spark: SparkSession): Try[DataFrame]
  22. def replacePathPrefix(filePath: Path, prefixPathToChange: Path, newPrefix: Path): Path
  23. final def synchronized[T0](arg0: ⇒ T0): T0
    Definition Classes
    AnyRef
  24. def toString(): String
    Definition Classes
    AnyRef → Any
  25. final def wait(arg0: Long, arg1: Int): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  26. final def wait(arg0: Long): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native()
  27. final def wait(): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  28. def writeRawModel(rawModel: RawModel, df: DataFrame): Try[Unit]

Deprecated Value Members

  1. def finalize(): Unit
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( classOf[java.lang.Throwable] ) @Deprecated @deprecated
    Deprecated

    (Since version ) see corresponding Javadoc for more information.

Inherited from Logging

Inherited from AnyRef

Inherited from Any

Ungrouped