VwCrossValidation

Abstract Value Members

abstract def get(name: String): File

Definition Classes
FileFunctions
abstract def save(file: File): String

Definition Classes
FileFunctions

Concrete Value Members

final def !=(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def !=(arg0: Any): Boolean

Definition Classes
Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def ==(arg0: Any): Boolean

Definition Classes
Any
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
def getCache(name: String): File

Definition Classes
VwDatasetFunctions
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def hashCode(): Int

Definition Classes
AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def kFold(vwDataset: Iterator[String], folds: Int, cacheBitSize: Int): Map[Int, (String, String)]

This method takes the VW input file specified in the class constructor and partitions the file into a training set and test set for every fold.
This method takes the VW input file specified in the class constructor and partitions the file into a training set and test set for every fold. The train and test set for every fold are then input into VW to generate cache files. These cache files are added to the SparkContext so that they'll be accessible on the executors. To keep track of the train and test set caches for every fold, a Map is used where the key is the fold number and the value is (trainingSetCachePath, testSetCachePath). These file names do NEED to be unique so that they do not collide with other file names. The entirety of this method runs on the driver. All VW input training / test set files as well as cache files are deleted upon JVM exit.
This strategy has the downside of duplicating the dataset across every node K times. An alternative approach is to train K cache files and train the regressor K - 1 times and test on the last test cache file.
vwDataset
folds
returns
a map representation where key is the fold number and value is (trainingSetFilename, testSetFilename)
def kFold(vwDataset: Iterable[String], folds: Int, cacheBitSize: Int): Map[Int, (String, String)]
def kFold(inputPath: String, folds: Int, cacheBitSize: Int): Map[Int, (String, String)]
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def save(inputIterator: Iterator[String]): String

Definition Classes
FileFunctions
def save(inputIterable: Iterable[String]): String

Definition Classes
FileFunctions
def save(inputPath: String): String

Definition Classes
FileFunctions
def saveAsCache(vwDatasetPath: String, vwCacheFilename: String, bitSize: Int): String

Definition Classes
VwDatasetFunctions
def saveAsCache(vwDatasetIterator: Iterator[String], vwCacheFilename: String, bitSize: Int): String

Definition Classes
VwDatasetFunctions
def saveAsCache(vwDatasetInputStream: InputStream, vwCacheFilename: String, bitSize: Int): String

Definition Classes
VwDatasetFunctions
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

trait VwCrossValidation extends VwDatasetFunctions

Abstract Value Members

abstract def get(name: String): File

abstract def save(file: File): String

Concrete Value Members

final def !=(arg0: AnyRef): Boolean

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: AnyRef): Boolean

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

def clone(): AnyRef

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

def getCache(name: String): File

final def getClass(): Class[_]

def hashCode(): Int

final def isInstanceOf[T0]: Boolean

def kFold(vwDataset: Iterator[String], folds: Int, cacheBitSize: Int): Map[Int, (String, String)]

def kFold(vwDataset: Iterable[String], folds: Int, cacheBitSize: Int): Map[Int, (String, String)]

def kFold(inputPath: String, folds: Int, cacheBitSize: Int): Map[Int, (String, String)]

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

def save(inputIterator: Iterator[String]): String

def save(inputIterable: Iterable[String]): String

def save(inputPath: String): String

def saveAsCache(vwDatasetPath: String, vwCacheFilename: String, bitSize: Int): String

def saveAsCache(vwDatasetIterator: Iterator[String], vwCacheFilename: String, bitSize: Int): String

def saveAsCache(vwDatasetInputStream: InputStream, vwCacheFilename: String, bitSize: Int): String

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from VwDatasetFunctions

Inherited from FileFunctions

Inherited from AnyRef

Inherited from Any

Ungrouped