com.logicalclocks.hsfs.spark.engine

Class TrainingDatasetEngine

java.lang.Object

com.logicalclocks.hsfs.spark.engine.TrainingDatasetEngine

public class TrainingDatasetEngine
extends Object

Constructor Summary

Constructors

Constructor and Description
TrainingDatasetEngine()

Method Summary

Modifier and Type	Method and Description
void	addTag(TrainingDataset trainingDataset, String name, Object value)
void	delete(TrainingDataset trainingDataset)
void	deleteTag(TrainingDataset trainingDataset, String name)
String	getQuery(TrainingDataset trainingDataset, Storage storage, boolean withLabel, boolean isHiveQuery)
Object	getTag(TrainingDataset trainingDataset, String name)
Map<String,Object>	getTags(TrainingDataset trainingDataset)
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>	read(TrainingDataset trainingDataset, String split, Map<String,String> providedOptions)
TrainingDataset	save(TrainingDataset trainingDataset, Query query, Map<String,String> userWriteOptions, List<String> labels) Make a REST call to Hopsworks to create the metadata and write the data on the File System.
void	updateStatisticsConfig(TrainingDataset trainingDataset)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

TrainingDatasetEngine

public TrainingDatasetEngine()

Method Detail

save

public TrainingDataset save(TrainingDataset trainingDataset,
                            Query query,
                            Map<String,String> userWriteOptions,
                            List<String> labels)
                     throws FeatureStoreException,
                            IOException

Make a REST call to Hopsworks to create the metadata and write the data on the File System.

Parameters:

trainingDataset - Training Dataset metadata object

query - Query Object

userWriteOptions - Additional write options as key-value pairs, defaults to empty Map

Returns:

Training Dataset metadata object

Throws:

FeatureStoreException - FeatureStoreException

IOException - IOException

read

public org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> read(TrainingDataset trainingDataset,
                                                                   String split,
                                                                   Map<String,String> providedOptions)
                                                            throws FeatureStoreException,
                                                                   IOException

Throws:

FeatureStoreException

IOException

addTag

public void addTag(TrainingDataset trainingDataset,
                   String name,
                   Object value)
            throws FeatureStoreException,
                   IOException

Throws:

FeatureStoreException

IOException

getTags

public Map<String,Object> getTags(TrainingDataset trainingDataset)
                           throws FeatureStoreException,
                                  IOException

Throws:

FeatureStoreException

IOException

getTag

public Object getTag(TrainingDataset trainingDataset,
                     String name)
              throws FeatureStoreException,
                     IOException

Throws:

FeatureStoreException

IOException

deleteTag

public void deleteTag(TrainingDataset trainingDataset,
                      String name)
               throws FeatureStoreException,
                      IOException

Throws:

FeatureStoreException

IOException

getQuery

public String getQuery(TrainingDataset trainingDataset,
                       Storage storage,
                       boolean withLabel,
                       boolean isHiveQuery)
                throws FeatureStoreException,
                       IOException

Throws:

FeatureStoreException

IOException

updateStatisticsConfig

public void updateStatisticsConfig(TrainingDataset trainingDataset)
                            throws FeatureStoreException,
                                   IOException

Throws:

FeatureStoreException

IOException

delete

public void delete(TrainingDataset trainingDataset)
            throws FeatureStoreException,
                   IOException

Throws:

FeatureStoreException

IOException