Package com.logicalclocks.hsfs

Class TrainingDatasetBase

java.lang.Object

com.logicalclocks.hsfs.TrainingDatasetBase

Direct Known Subclasses:

TrainingDataset

public class TrainingDatasetBase
extends Object

Field Summary

Fields

Modifier and Type	Field	Description
protected Boolean	coalesce
protected DataFormat	dataFormat
protected DataSource	dataSource
protected String	description
protected Date	eventEndTime
protected Date	eventStartTime
protected FilterLogic	extraFilter
protected List<TrainingDatasetFeature>	features
protected FeatureStoreBase	featureStore
protected Integer	id
protected List<String>	label
protected String	location
protected static final org.slf4j.Logger	LOGGER
protected String	name
protected Long	seed
protected List<Split>	splits
protected StatisticsConfig	statisticsConfig
protected TrainingDatasetType	trainingDatasetType
protected String	trainSplit
protected String	type
protected Integer	version

Constructor Summary

Constructors

Constructor

Description

TrainingDatasetBase(Integer version, String description, DataFormat dataFormat, Boolean coalesce, StorageConnector storageConnector, String location, List<Split> splits, String trainSplit, Long seed, FeatureStoreBase featureStore, StatisticsConfig statisticsConfig, List<String> label, String eventStartTime, String eventEndTime, TrainingDatasetType trainingDatasetType, Float validationSize, Float testSize, String trainStart, String trainEnd, String validationStart, String validationEnd, String testStart, String testEnd, Integer timeSplitSize, FilterLogic extraFilterLogic, Filter extraFilter, DataSource dataSource)

Method Summary

Modifier and Type	Method	Description
void	addTag(String name, Object value)	Add name/value tag to the training dataset.
void	delete()	Delete training dataset and all associated metadata.
void	deleteTag(String name)	Delete a tag of the training dataset.
List<String>	getLabel()
Statistics	getStatistics()	Get the last statistics commit for the training dataset.
Statistics	getStatistics(String commitTime)	Get the statistics of a specific commit time for the training dataset.
Object	getTag(String name)	Get a single tag value of the training dataset.
Map<String,Object>	getTags()	Get all tags of the training dataset.
TrainingDatasetType	getTrainingDatasetType(DataSource dataSource)
void	setLabel(List<String> label)
void	setTimeSeriesSplits(Integer timeSplitSize, String trainStart, String trainEnd, String valStart, String valEnd, String testStart, String testEnd)
void	setValTestSplit(Float valSize, Float testSize)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

id

protected Integer id

name

protected String name

version

protected Integer version

description

protected String description

coalesce

protected Boolean coalesce

trainingDatasetType

protected TrainingDatasetType trainingDatasetType

features

protected List<TrainingDatasetFeature> features

featureStore

protected FeatureStoreBase featureStore

location

protected String location

seed

protected Long seed

splits

protected List<Split> splits

trainSplit

protected String trainSplit

label

protected List<String> label

eventStartTime

protected Date eventStartTime

eventEndTime

protected Date eventEndTime

extraFilter

protected FilterLogic extraFilter

dataFormat

protected DataFormat dataFormat

dataSource

protected DataSource dataSource

statisticsConfig

protected StatisticsConfig statisticsConfig

type

protected String type

LOGGER

protected static final org.slf4j.Logger LOGGER

Constructor Details

TrainingDatasetBase

public TrainingDatasetBase(Integer version,
 String description,
 DataFormat dataFormat,
 Boolean coalesce,
 StorageConnector storageConnector,
 String location,
 List<Split> splits,
 String trainSplit,
 Long seed,
 FeatureStoreBase featureStore,
 StatisticsConfig statisticsConfig,
 List<String> label,
 String eventStartTime,
 String eventEndTime,
 TrainingDatasetType trainingDatasetType,
 Float validationSize,
 Float testSize,
 String trainStart,
 String trainEnd,
 String validationStart,
 String validationEnd,
 String testStart,
 String testEnd,
 Integer timeSplitSize,
 FilterLogic extraFilterLogic,
 Filter extraFilter,
 DataSource dataSource)
                    throws FeatureStoreException,
ParseException

Throws:

FeatureStoreException

ParseException

Method Details

setTimeSeriesSplits

public void setTimeSeriesSplits(Integer timeSplitSize,
 String trainStart,
 String trainEnd,
 String valStart,
 String valEnd,
 String testStart,
 String testEnd)
                         throws FeatureStoreException,
ParseException

Throws:

FeatureStoreException

ParseException

setValTestSplit

public void setValTestSplit(Float valSize,
 Float testSize)

getLabel

public List<String> getLabel()

setLabel

public void setLabel(List<String> label)

getTrainingDatasetType

public TrainingDatasetType getTrainingDatasetType(DataSource dataSource)

getStatistics

public Statistics getStatistics()
                         throws FeatureStoreException,
IOException

Get the last statistics commit for the training dataset.

Returns:

statistics object of latest commit

Throws:

FeatureStoreException

IOException

getStatistics

public Statistics getStatistics(String commitTime)
                         throws FeatureStoreException,
IOException

Get the statistics of a specific commit time for the training dataset.

Parameters:

commitTime - commit time in the format "YYYYMMDDhhmmss"

Returns:

statistics object for the commit time

Throws:

FeatureStoreException

IOException

addTag

public void addTag(String name,
 Object value)
            throws FeatureStoreException,
IOException

Add name/value tag to the training dataset.

Parameters:

name - name of the tag

value - value of the tag. The value of a tag can be any valid json - primitives, arrays or json objects

Throws:

FeatureStoreException

IOException

getTags

public Map<String,Object> getTags()
                           throws FeatureStoreException,
IOException

Get all tags of the training dataset.

Returns:

a map of tag name and values. The value of a tag can be any valid json - primitives, arrays or json objects

Throws:

FeatureStoreException

IOException

getTag

public Object getTag(String name)
              throws FeatureStoreException,
IOException

Get a single tag value of the training dataset.

Parameters:

name - name of the tag

Returns:

The value of a tag can be any valid json - primitives, arrays or json objects

Throws:

FeatureStoreException

IOException

deleteTag

public void deleteTag(String name)
               throws FeatureStoreException,
IOException

Delete a tag of the training dataset.

Parameters:

name - name of the tag to be deleted

Throws:

FeatureStoreException

IOException

delete

public void delete()
            throws FeatureStoreException,
IOException

Delete training dataset and all associated metadata. Note that this operation drops only files which were materialized in HopsFS. If you used a Storage Connector for a cloud storage such as S3, the data will not be deleted, but you will not be able to track it anymore from the Feature Store. This operation drops all metadata associated with this version of the training dataset and the materialized data in HopsFS.

Throws:

FeatureStoreException

IOException