com.logicalclocks.hsfs.spark.engine

Class StatisticsEngine

java.lang.Object

com.logicalclocks.hsfs.spark.engine.StatisticsEngine

public class StatisticsEngine
extends Object

Constructor Summary

Constructors

Constructor and Description
StatisticsEngine(EntityEndpointType entityType)

Method Summary

Modifier and Type	Method and Description
Statistics	computeAndSaveSplitStatistics(FeatureView featureView, TrainingDataset trainingDataset, Map<String,org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>> splitDatasets)
Statistics	computeAndSaveSplitStatistics(TrainingDataset trainingDataset)
Statistics	computeSplitStatistics(TrainingDataset trainingDataset)
Statistics	computeSplitStatistics(TrainingDataset trainingDataset, Map<String,org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>> splitDatasets)
Statistics	computeStatistics(FeatureGroupBase featureGroup, org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataFrame, Long commitId)
Statistics	computeStatistics(FeatureView featureView, TrainingDataset trainingDataset, org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataFrame)
Statistics	computeStatistics(TrainingDataset trainingDataset, org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataFrame)
Statistics	get(FeatureGroupBase featureGroup, String commitTime)
Statistics	get(TrainingDataset trainingDataset, String commitTime)
Statistics	getLast(FeatureGroupBase featureGroup)
Statistics	getLast(TrainingDataset trainingDataset)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

StatisticsEngine

public StatisticsEngine(EntityEndpointType entityType)

Method Detail

computeStatistics

public Statistics computeStatistics(TrainingDataset trainingDataset,
                                    org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataFrame)
                             throws IOException,
                                    FeatureStoreException

Throws:

IOException

FeatureStoreException

computeStatistics

public Statistics computeStatistics(FeatureView featureView,
                                    TrainingDataset trainingDataset,
                                    org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataFrame)
                             throws FeatureStoreException,
                                    IOException

Throws:

FeatureStoreException

IOException

computeStatistics

public Statistics computeStatistics(FeatureGroupBase featureGroup,
                                    org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataFrame,
                                    Long commitId)
                             throws FeatureStoreException,
                                    IOException

Throws:

FeatureStoreException

IOException

computeAndSaveSplitStatistics

public Statistics computeAndSaveSplitStatistics(TrainingDataset trainingDataset)
                                         throws FeatureStoreException,
                                                IOException

Throws:

FeatureStoreException

IOException

computeAndSaveSplitStatistics

public Statistics computeAndSaveSplitStatistics(FeatureView featureView,
                                                TrainingDataset trainingDataset,
                                                Map<String,org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>> splitDatasets)
                                         throws FeatureStoreException,
                                                IOException

Throws:

FeatureStoreException

IOException

computeSplitStatistics

public Statistics computeSplitStatistics(TrainingDataset trainingDataset)
                                  throws FeatureStoreException,
                                         IOException

Throws:

FeatureStoreException

IOException

computeSplitStatistics

public Statistics computeSplitStatistics(TrainingDataset trainingDataset,
                                         Map<String,org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>> splitDatasets)

get

public Statistics get(FeatureGroupBase featureGroup,
                      String commitTime)
               throws FeatureStoreException,
                      IOException

Throws:

FeatureStoreException

IOException

get

public Statistics get(TrainingDataset trainingDataset,
                      String commitTime)
               throws FeatureStoreException,
                      IOException

Throws:

FeatureStoreException

IOException

getLast

public Statistics getLast(FeatureGroupBase featureGroup)
                   throws FeatureStoreException,
                          IOException

Throws:

FeatureStoreException

IOException

getLast

public Statistics getLast(TrainingDataset trainingDataset)
                   throws FeatureStoreException,
                          IOException

Throws:

FeatureStoreException

IOException