com.logicalclocks.hsfs.spark.engine

Class FeatureViewEngine

java.lang.Object

com.logicalclocks.hsfs.engine.FeatureViewEngineBase<Query,FeatureView,FeatureStore,StreamFeatureGroup,org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>>

com.logicalclocks.hsfs.spark.engine.FeatureViewEngine

public class FeatureViewEngine
extends FeatureViewEngineBase<Query,FeatureView,FeatureStore,StreamFeatureGroup,org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>>

Field Summary

Fields inherited from class com.logicalclocks.hsfs.engine.FeatureViewEngineBase

AMBIGUOUS_LABEL_ERROR, featureViewApi, LABEL_NOT_EXIST_ERROR, LOGGER, tagsApi

Constructor Summary

Constructors

Constructor and Description
FeatureViewEngine()

Method Summary

Modifier and Type	Method and Description
Statistics	computeStatistics(FeatureView featureView, TrainingDataset trainingDataset, org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>[] datasets)
protected Map<String,org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>>	convertSplitDatasetsToMap(List<Split> splits, org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>[] datasets)
TrainingDatasetBundle	createTrainingDataset(FeatureView featureView, TrainingDataset trainingDataset, Map<String,String> userWriteOptions)
FeatureView	get(FeatureStore featureStore, String name, Integer version)
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>	getBatchData(FeatureView featureView, Date startTime, Date endTime, Map<String,String> readOptions, Integer trainingDataVersion)
Query	getBatchQuery(FeatureView featureView, Date startTime, Date endTime, Boolean withLabels, Integer trainingDataVersion)
String	getBatchQueryString(FeatureView featureView, Date startTime, Date endTime, Integer trainingDataVersion)
FeatureView	getOrCreateFeatureView(FeatureStore featureStore, String name, Integer version, Query query, String description, List<String> labels)
TrainingDatasetBundle	getTrainingDataset(FeatureView featureView, Integer trainingDatasetVersion, List<String> requestedSplits, Map<String,String> userReadOptions)
TrainingDatasetBundle	getTrainingDataset(FeatureView featureView, TrainingDataset trainingDataset, List<String> requestedSplits, Map<String,String> userReadOptions)
TrainingDatasetBundle	getTrainingDataset(FeatureView featureView, TrainingDataset trainingDataset, Map<String,String> userReadOptions)
void	recreateTrainingDataset(FeatureView featureView, Integer version, Map<String,String> userWriteOptions)
FeatureView	update(FeatureView featureView)
void	writeTrainingDataset(FeatureView featureView, TrainingDataset trainingDataset, Map<String,String> userWriteOptions)

Methods inherited from class com.logicalclocks.hsfs.engine.FeatureViewEngineBase

addTag, addTag, delete, delete, deleteTag, deleteTag, deleteTrainingData, deleteTrainingData, deleteTrainingDatasetOnly, deleteTrainingDatasetOnly, get, get, getBatchQuery, getEndTime, getStartTime, getTag, getTag, getTags, getTags, makeLabelFeatures, save, setEventTime

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

FeatureViewEngine

public FeatureViewEngine()

Method Detail

update

public FeatureView update(FeatureView featureView)
                   throws FeatureStoreException,
                          IOException

Specified by:

update in class FeatureViewEngineBase<Query,FeatureView,FeatureStore,StreamFeatureGroup,org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>>

Throws:

FeatureStoreException

IOException

get

public FeatureView get(FeatureStore featureStore,
                       String name,
                       Integer version)
                throws FeatureStoreException,
                       IOException

Specified by:

get in class FeatureViewEngineBase<Query,FeatureView,FeatureStore,StreamFeatureGroup,org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>>

Throws:

FeatureStoreException

IOException

createTrainingDataset

public TrainingDatasetBundle createTrainingDataset(FeatureView featureView,
                                                   TrainingDataset trainingDataset,
                                                   Map<String,String> userWriteOptions)
                                            throws IOException,
                                                   FeatureStoreException

Throws:

IOException

FeatureStoreException

writeTrainingDataset

public void writeTrainingDataset(FeatureView featureView,
                                 TrainingDataset trainingDataset,
                                 Map<String,String> userWriteOptions)
                          throws IOException,
                                 FeatureStoreException

Throws:

IOException

FeatureStoreException

getTrainingDataset

public TrainingDatasetBundle getTrainingDataset(FeatureView featureView,
                                                Integer trainingDatasetVersion,
                                                List<String> requestedSplits,
                                                Map<String,String> userReadOptions)
                                         throws IOException,
                                                FeatureStoreException,
                                                ParseException

Throws:

IOException

FeatureStoreException

ParseException

getTrainingDataset

public TrainingDatasetBundle getTrainingDataset(FeatureView featureView,
                                                TrainingDataset trainingDataset,
                                                Map<String,String> userReadOptions)
                                         throws IOException,
                                                FeatureStoreException

Throws:

IOException

FeatureStoreException

getTrainingDataset

public TrainingDatasetBundle getTrainingDataset(FeatureView featureView,
                                                TrainingDataset trainingDataset,
                                                List<String> requestedSplits,
                                                Map<String,String> userReadOptions)
                                         throws IOException,
                                                FeatureStoreException

Throws:

IOException

FeatureStoreException

computeStatistics

public Statistics computeStatistics(FeatureView featureView,
                                    TrainingDataset trainingDataset,
                                    org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>[] datasets)
                             throws FeatureStoreException,
                                    IOException

Throws:

FeatureStoreException

IOException

convertSplitDatasetsToMap

protected Map<String,org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>> convertSplitDatasetsToMap(List<Split> splits,
                                                                                                       org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>[] datasets)

recreateTrainingDataset

public void recreateTrainingDataset(FeatureView featureView,
                                    Integer version,
                                    Map<String,String> userWriteOptions)
                             throws IOException,
                                    FeatureStoreException

Throws:

IOException

FeatureStoreException

getBatchQueryString

public String getBatchQueryString(FeatureView featureView,
                                  Date startTime,
                                  Date endTime,
                                  Integer trainingDataVersion)
                           throws FeatureStoreException,
                                  IOException

Specified by:

getBatchQueryString in class FeatureViewEngineBase<Query,FeatureView,FeatureStore,StreamFeatureGroup,org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>>

Throws:

FeatureStoreException

IOException

getBatchData

public org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> getBatchData(FeatureView featureView,
                                                                           Date startTime,
                                                                           Date endTime,
                                                                           Map<String,String> readOptions,
                                                                           Integer trainingDataVersion)
                                                                    throws FeatureStoreException,
                                                                           IOException

Specified by:

getBatchData in class FeatureViewEngineBase<Query,FeatureView,FeatureStore,StreamFeatureGroup,org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>>

Throws:

FeatureStoreException

IOException

getBatchQuery

public Query getBatchQuery(FeatureView featureView,
                           Date startTime,
                           Date endTime,
                           Boolean withLabels,
                           Integer trainingDataVersion)
                    throws FeatureStoreException,
                           IOException

Specified by:

getBatchQuery in class FeatureViewEngineBase<Query,FeatureView,FeatureStore,StreamFeatureGroup,org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>>

Throws:

FeatureStoreException

IOException

getOrCreateFeatureView

public FeatureView getOrCreateFeatureView(FeatureStore featureStore,
                                          String name,
                                          Integer version,
                                          Query query,
                                          String description,
                                          List<String> labels)
                                   throws FeatureStoreException,
                                          IOException

Specified by:

getOrCreateFeatureView in class FeatureViewEngineBase<Query,FeatureView,FeatureStore,StreamFeatureGroup,org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>>

Throws:

FeatureStoreException

IOException