hex

Class DataInfo

java.lang.Object

water.Iced<T>

water.Keyed<DataInfo>

hex.DataInfo

All Implemented Interfaces:

java.io.Externalizable, java.io.Serializable, java.lang.Cloneable, water.Freezable<DataInfo>

public class DataInfo
extends water.Keyed<DataInfo>

Created by tomasnykodym on 1/29/15. Provides higher level interface for accessing data row-wise. Performs on the fly auto-expansion of categorical variables (to 1 hot encoding) and standardization ( or normalize/demean/descale/none) of predictors and response. Supports sparse data, sparse columns can be transformed to sparse rows on the fly with some (significant) memory overhead, as the data of the whole chunk(s) will be copied.

See Also:

Serialized Form

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
class	DataInfo.Row
class	DataInfo.Rows
static class	DataInfo.TransformType

Field Summary

Fields

Modifier and Type	Field and Description
int[]	_activeCols
water.fvec.Frame	_adaptedFrame
int[][]	_catLvls
boolean[]	_catMissing
int[]	_catModes
int[]	_catOffsets
int	_cats
java.lang.String[]	_coefNames
boolean	_fold
boolean	_imputeMissing
boolean	_intercept
double[]	_normMul
double[]	_normRespMul
double[]	_normRespSub
double[]	_normSub
double[]	_numMeans
int	_nums
boolean	_offset
int	_outpus
int[]	_permutation
DataInfo.TransformType	_predictor_transform
DataInfo.TransformType	_response_transform
int	_responses
boolean	_skipMissing
boolean	_useAllFactorLevels
boolean	_valid
boolean	_weights

Fields inherited from class water.Keyed

_key

Constructor Summary

Constructors

Constructor and Description
DataInfo(water.fvec.Frame train, water.fvec.Frame valid, boolean useAllFactorLevels, DataInfo.TransformType predictor_transform, boolean skipMissing, boolean imputeMissing, boolean missingBucket)
DataInfo(water.fvec.Frame train, water.fvec.Frame valid, int nResponses, boolean useAllFactorLevels, DataInfo.TransformType predictor_transform, DataInfo.TransformType response_transform, boolean skipMissing, boolean imputeMissing, boolean missingBucket, boolean weight, boolean offset, boolean fold)
DataInfo(water.fvec.Frame train, water.fvec.Frame valid, int nResponses, boolean useAllFactorLevels, DataInfo.TransformType predictor_transform, DataInfo.TransformType response_transform, boolean skipMissing, boolean imputeMissing, boolean missingBucket, boolean weight, boolean offset, boolean fold, boolean intercept)

Method Summary

Methods

Modifier and Type	Method and Description
int[]	activeCols()
void	addOutput(java.lang.String name, water.fvec.Vec v)
void	addResponse(java.lang.String[] names, water.fvec.Vec[] vecs)
protected long	checksum_impl()
java.lang.String[]	coefNames()
DataInfo	deep_clone()
double[]	denormalizeBeta(double[] beta)
void	dropWeights()
DataInfo.Row	extractDenseRow(water.fvec.Chunk[] chunks, int rid, DataInfo.Row row)
DataInfo.Row	extractDenseRow(double[] vals, DataInfo.Row row)
DataInfo.Row[]	extractSparseRows(water.fvec.Chunk[] chunks) Extract (sparse) rows from given chunks.
DataInfo	filterExpandedColumns(int[] cols)
int	foldChunkId()
int	fullN()
int	getCategoricalId(int cid, int val)
water.fvec.Vec	getOffsetVec()
water.fvec.Vec	getOutputVec(int i)
water.fvec.Vec	getWeightsVec()
static int	imputeCat(water.fvec.Vec v)
int	largestCat()
int[]	mapNames(java.lang.String[] names)
DataInfo.Row	newDenseRow()
DataInfo.Row	newDenseRow(double[] numVals, long start)
int	numStart()
int	offsetChunkId()
int	outputChunkId()
int	outputChunkId(int n)
int	responseChunkId(int n)
DataInfo.Rows	rows(water.fvec.Chunk[] chks)
DataInfo.Rows	rows(water.fvec.Chunk[] chks, boolean sparse)
DataInfo	scoringInfo()
void	setPredictorTransform(DataInfo.TransformType t)
void	setResponse(java.lang.String name, water.fvec.Vec v)
void	setResponse(java.lang.String name, water.fvec.Vec v, int n)
void	setResponseTransform(DataInfo.TransformType t)
water.fvec.Vec	setWeights(java.lang.String name, water.fvec.Vec vec)
void	unScaleNumericals(double[] in, double[] out) Undo the standardization/normalization of numerical columns
void	updateWeightedSigmaAndMean(double[] sigmas, double[] mean)
void	updateWeightedSigmaAndMeanForResponse(double[] sigmas, double[] mean)
DataInfo	validDinfo(water.fvec.Frame valid)
int	weightChunkId()

Methods inherited from class water.Keyed

checksum, makeSchema, readAll_impl, readAll, remove_impl, remove, remove, remove, remove, writeAll_impl, writeAll

Methods inherited from class water.Iced

clone, copyOver, frozenType, read_impl, read, readExternal, readJSON_impl, readJSON, toJsonString, write_impl, write, writeExternal, writeJSON_impl, writeJSON

Methods inherited from class java.lang.Object

equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

_activeCols

public int[] _activeCols

_adaptedFrame

public water.fvec.Frame _adaptedFrame

_responses

public int _responses

_outpus

public int _outpus

_predictor_transform

public DataInfo.TransformType _predictor_transform

_response_transform

public DataInfo.TransformType _response_transform

_useAllFactorLevels

public boolean _useAllFactorLevels

_nums

public int _nums

_cats

public int _cats

_catOffsets

public int[] _catOffsets

_catMissing

public boolean[] _catMissing

_catModes

public int[] _catModes

_permutation

public int[] _permutation

_normMul

public double[] _normMul

_normSub

public double[] _normSub

_normRespMul

public double[] _normRespMul

_normRespSub

public double[] _normRespSub

_numMeans

public double[] _numMeans

_intercept

public boolean _intercept

_offset

public boolean _offset

_weights

public boolean _weights

_fold

public boolean _fold

_skipMissing

public final boolean _skipMissing

_imputeMissing

public final boolean _imputeMissing

_valid

public boolean _valid

_catLvls

public final int[][] _catLvls

_coefNames

public java.lang.String[] _coefNames

Constructor Detail

DataInfo

public DataInfo(water.fvec.Frame train,
        water.fvec.Frame valid,
        boolean useAllFactorLevels,
        DataInfo.TransformType predictor_transform,
        boolean skipMissing,
        boolean imputeMissing,
        boolean missingBucket)

DataInfo

public DataInfo(water.fvec.Frame train,
        water.fvec.Frame valid,
        int nResponses,
        boolean useAllFactorLevels,
        DataInfo.TransformType predictor_transform,
        DataInfo.TransformType response_transform,
        boolean skipMissing,
        boolean imputeMissing,
        boolean missingBucket,
        boolean weight,
        boolean offset,
        boolean fold)

DataInfo

public DataInfo(water.fvec.Frame train,
        water.fvec.Frame valid,
        int nResponses,
        boolean useAllFactorLevels,
        DataInfo.TransformType predictor_transform,
        DataInfo.TransformType response_transform,
        boolean skipMissing,
        boolean imputeMissing,
        boolean missingBucket,
        boolean weight,
        boolean offset,
        boolean fold,
        boolean intercept)

Method Detail

setWeights

public water.fvec.Vec setWeights(java.lang.String name,
                        water.fvec.Vec vec)

dropWeights

public void dropWeights()

activeCols

public int[] activeCols()

addResponse

public void addResponse(java.lang.String[] names,
               water.fvec.Vec[] vecs)

responseChunkId

public int responseChunkId(int n)

foldChunkId

public int foldChunkId()

offsetChunkId

public int offsetChunkId()

weightChunkId

public int weightChunkId()

outputChunkId

public int outputChunkId()

outputChunkId

public int outputChunkId(int n)

addOutput

public void addOutput(java.lang.String name,
             water.fvec.Vec v)

getOutputVec

public water.fvec.Vec getOutputVec(int i)

setResponse

public void setResponse(java.lang.String name,
               water.fvec.Vec v)

setResponse

public void setResponse(java.lang.String name,
               water.fvec.Vec v,
               int n)

checksum_impl

protected long checksum_impl()

Overrides:

checksum_impl in class water.Keyed<DataInfo>

deep_clone

public DataInfo deep_clone()

validDinfo

public DataInfo validDinfo(water.fvec.Frame valid)

scoringInfo

public DataInfo scoringInfo()

denormalizeBeta

public double[] denormalizeBeta(double[] beta)

imputeCat

public static int imputeCat(water.fvec.Vec v)

filterExpandedColumns

public DataInfo filterExpandedColumns(int[] cols)

updateWeightedSigmaAndMean

public void updateWeightedSigmaAndMean(double[] sigmas,
                              double[] mean)

updateWeightedSigmaAndMeanForResponse

public void updateWeightedSigmaAndMeanForResponse(double[] sigmas,
                                         double[] mean)

setPredictorTransform

public void setPredictorTransform(DataInfo.TransformType t)

setResponseTransform

public void setResponseTransform(DataInfo.TransformType t)

fullN

public final int fullN()

largestCat

public final int largestCat()

numStart

public final int numStart()

coefNames

public final java.lang.String[] coefNames()

mapNames

public int[] mapNames(java.lang.String[] names)

unScaleNumericals

public final void unScaleNumericals(double[] in,
                     double[] out)

Undo the standardization/normalization of numerical columns

Parameters:

in - input values

out - output values (can be the same as input)

getCategoricalId

public final int getCategoricalId(int cid,
                   int val)

extractDenseRow

public final DataInfo.Row extractDenseRow(double[] vals,
                           DataInfo.Row row)

extractDenseRow

public final DataInfo.Row extractDenseRow(water.fvec.Chunk[] chunks,
                           int rid,
                           DataInfo.Row row)

getWeightsVec

public water.fvec.Vec getWeightsVec()

getOffsetVec

public water.fvec.Vec getOffsetVec()

newDenseRow

public DataInfo.Row newDenseRow()

newDenseRow

public DataInfo.Row newDenseRow(double[] numVals,
                       long start)

rows

public DataInfo.Rows rows(water.fvec.Chunk[] chks)

rows

public DataInfo.Rows rows(water.fvec.Chunk[] chks,
                 boolean sparse)

extractSparseRows

public final DataInfo.Row[] extractSparseRows(water.fvec.Chunk[] chunks)

Extract (sparse) rows from given chunks. Note: 0 remains 0 - _normSub of DataInfo isn't used (mean shift during standarization is not reverted) - UNLESS offset is specified (for GLM only) Essentially turns the dataset 90 degrees.

Parameters:

chunks - - chunk of dataset

Returns:

array of sparse rows