com.logicalclocks.hsfs

Class FeatureGroupBase<T>

java.lang.Object

com.logicalclocks.hsfs.FeatureGroupBase<T>

Direct Known Subclasses:

ExternalFeatureGroup, FeatureGroup, FeatureGroupBaseForApi, StreamFeatureGroup, StreamFeatureGroup, StreamFeatureGroup

public abstract class FeatureGroupBase<T>
extends Object

Field Summary

Fields

Modifier and Type	Field and Description
protected Date	created
protected User	creator
protected DeltaStreamerJobConf	deltaStreamerJobConf
protected Boolean	deprecated
protected String	description
protected String	eventTime
protected List<String>	expectationsNames
protected FeatureGroupEngineBase	featureGroupEngineBase
protected List<Feature>	features
protected FeatureStoreBase	featureStore
protected String	hudiPrecombineKey
protected Integer	id
protected String	location
protected static org.slf4j.Logger	LOGGER
protected String	name
protected String	notificationTopicName
protected OnlineConfig	onlineConfig
protected Boolean	onlineEnabled
protected OnlineIngestionApi	onlineIngestionApi
protected String	onlineTopicName
protected List<String>	partitionKeys
protected String	path
protected List<String>	primaryKeys
protected List<String>	statisticColumns
protected StatisticsConfig	statisticsConfig
protected StorageConnector	storageConnector
protected Subject	subject
protected TimeTravelFormat	timeTravelFormat
protected String	topicName
protected String	type
protected FeatureGroupUtils	utils
protected Integer	version

Constructor Summary

Constructors

Constructor and Description
FeatureGroupBase()

Method Summary

Modifier and Type	Method and Description
void	addTag(String name, Object value) Add name/value tag to the feature group.
abstract void	appendFeatures(Feature features) Append a single feature to the schema of the feature group.
abstract void	appendFeatures(List<Feature> features) Append features to the schema of the feature group.
abstract QueryBase	asOf(String wallclockTime)
abstract QueryBase	asOf(String wallclockTime, String excludeUntil)
void	checkDeprecated()
abstract void	commitDeleteRecord(T featureData)
abstract void	commitDeleteRecord(T featureData, Map<String,String> writeOptions)
abstract Map<Long,Map<String,String>>	commitDetails()
abstract Map<Long,Map<String,String>>	commitDetails(Integer limit)
abstract Map<Long,Map<String,String>>	commitDetails(String wallclockTime)
abstract Map<Long,Map<String,String>>	commitDetails(String wallclockTime, Integer limit)
abstract Statistics	computeStatistics() Recompute the statistics for the feature group and save them to the feature store.
abstract Statistics	computeStatistics(String wallclockTime)
void	delete()
void	deleteTag(String name) Delete a tag of the feature group.
String	getAvroSchema()
List<String>	getComplexFeatures()
org.apache.avro.Schema	getDeserializedAvroSchema()
org.apache.avro.Schema	getDeserializedEncodedAvroSchema()
String	getEncodedAvroSchema()
Feature	getFeature(String name) Retrieve a feature of the feature group by name.
String	getFeatureAvroSchema(String featureName)
OnlineIngestion	getLatestOnlineIngestion()
OnlineIngestion	getOnlineIngestion(Integer id)
List<String>	getPrimaryKeys()
abstract Statistics	getStatistics() Get the last statistics commit for the feature group.
Subject	getSubject()
Object	getTag(String name) Get a single tag value of the feature group.
Map<String,Object>	getTags() Get all tags of the feature group.
abstract void	insert(T featureData)
abstract void	insert(T featureData, boolean overwrite)
abstract void	insert(T featureData, boolean overwrite, Map<String,String> writeOptions)
abstract void	insert(T featureData, boolean overwrite, Map<String,String> writeOptions, JobConfiguration jobConfiguration)
abstract void	insert(T featureData, HudiOperationType operation)
abstract void	insert(T featureData, JobConfiguration jobConfiguration)
abstract void	insert(T featureData, Map<String,String> writeOptions)
abstract void	insert(T featureData, Storage storage)
abstract void	insert(T featureData, Storage storage, boolean overwrite)
abstract void	insert(T featureData, Storage storage, boolean overwrite, HudiOperationType operation, Map<String,String> writeOptions)
abstract Object	insertStream(T featureData)
abstract Object	insertStream(T featureData, Map<String,String> writeOptions)
abstract Object	insertStream(T featureData, String queryName)
abstract Object	insertStream(T featureData, String queryName, Map<String,String> writeOptions)
abstract Object	insertStream(T featureData, String queryName, String outputMode)
abstract Object	insertStream(T featureData, String queryName, String outputMode, boolean awaitTermination, Long timeout)
abstract Object	insertStream(T featureData, String queryName, String outputMode, boolean awaitTermination, Long timeout, String checkpointLocation)
abstract Object	insertStream(T featureData, String queryName, String outputMode, boolean awaitTermination, Long timeout, String checkpointLocation, Map<String,String> writeOptions)
abstract Object	insertStream(T featureData, String queryName, String outputMode, boolean awaitTermination, Long timeout, String checkpointLocation, Map<String,String> writeOptions, JobConfiguration jobConfiguration)
abstract Object	insertStream(T featureData, String queryName, String outputMode, boolean awaitTermination, String checkpointLocation)
abstract Object	insertStream(T featureData, String queryName, String outputMode, String checkpointLocation)
abstract T	read()
abstract T	read(boolean online)
abstract T	read(boolean online, Map<String,String> readOptions)
abstract T	read(Map<String,String> readOptions)
abstract T	read(String wallclockTime)
abstract T	read(String wallclockTime, Map<String,String> readOptions)
abstract QueryBase	select(List<String> features)
abstract QueryBase	selectAll()
abstract QueryBase	selectExcept(List<String> features)
abstract QueryBase	selectExceptFeatures(List<Feature> features)
abstract QueryBase	selectFeatures(List<Feature> features)
void	setDeprecated(Boolean deprecated)
abstract void	show(int numRows)
abstract void	show(int numRows, boolean online)
void	unloadSubject()
void	updateDeprecated() Deprecate the feature group.
void	updateDeprecated(Boolean deprecate) Deprecate the feature group.
void	updateDescription(String description) Update the description of the feature group.
void	updateFeatureDescription(String featureName, String description) Update the description of a single feature.
abstract void	updateFeatures(Feature feature) Update the metadata of multiple features.
abstract void	updateFeatures(List<Feature> features) Update the metadata of multiple features.
void	updateNotificationTopicName(String notificationTopicName) Update the notification topic name of the feature group.
void	updateStatisticsConfig() Update the statistics configuration of the feature group.

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

id

protected Integer id

featureStore

protected FeatureStoreBase featureStore

type

protected String type

name

protected String name

version

protected Integer version

description

protected String description

primaryKeys

protected List<String> primaryKeys

features

protected List<Feature> features

eventTime

protected String eventTime

created

protected Date created

creator

protected User creator

statisticsConfig

protected StatisticsConfig statisticsConfig

expectationsNames

protected List<String> expectationsNames

location

protected String location

timeTravelFormat

protected TimeTravelFormat timeTravelFormat

onlineEnabled

protected Boolean onlineEnabled

onlineTopicName

protected String onlineTopicName

topicName

protected String topicName

notificationTopicName

protected String notificationTopicName

statisticColumns

protected List<String> statisticColumns

deltaStreamerJobConf

protected DeltaStreamerJobConf deltaStreamerJobConf

deprecated

protected Boolean deprecated

onlineConfig

protected OnlineConfig onlineConfig

storageConnector

protected StorageConnector storageConnector

path

protected String path

partitionKeys

protected List<String> partitionKeys

hudiPrecombineKey

protected String hudiPrecombineKey

subject

protected Subject subject

featureGroupEngineBase

protected FeatureGroupEngineBase featureGroupEngineBase

utils

protected FeatureGroupUtils utils

onlineIngestionApi

protected OnlineIngestionApi onlineIngestionApi

LOGGER

protected static final org.slf4j.Logger LOGGER

Constructor Detail

FeatureGroupBase

public FeatureGroupBase()

Method Detail

setDeprecated

public void setDeprecated(Boolean deprecated)

checkDeprecated

public void checkDeprecated()

delete

public void delete()
            throws FeatureStoreException,
                   IOException

Throws:

FeatureStoreException

IOException

addTag

public void addTag(String name,
                   Object value)
            throws FeatureStoreException,
                   IOException

Add name/value tag to the feature group.

Parameters:

name - name of the tag

value - value of the tag. The value of a tag can be any valid json - primitives, arrays or json objects.

Throws:

FeatureStoreException - FeatureStoreException

IOException - IOException

getTags

public Map<String,Object> getTags()
                           throws FeatureStoreException,
                                  IOException

Get all tags of the feature group.

Returns:

map of tag name and values. The value of a tag can be any valid json - primitives, arrays or json objects

Throws:

FeatureStoreException - FeatureStoreException

IOException - IOException

getTag

public Object getTag(String name)
              throws FeatureStoreException,
                     IOException

Get a single tag value of the feature group.

Parameters:

name - name of tha tag

Returns:

The value of a tag can be any valid json - primitives, arrays or json objects

Throws:

FeatureStoreException - FeatureStoreException

IOException - IOException

deleteTag

public void deleteTag(String name)
               throws FeatureStoreException,
                      IOException

Delete a tag of the feature group.

Parameters:

name - name of the tag to be deleted

Throws:

FeatureStoreException - FeatureStoreException

IOException - IOException

updateDescription

public void updateDescription(String description)
                       throws FeatureStoreException,
                              IOException

Update the description of the feature group.

Parameters:

description - feature group description.

Throws:

FeatureStoreException - FeatureStoreException

IOException - IOException

updateNotificationTopicName

public void updateNotificationTopicName(String notificationTopicName)
                                 throws FeatureStoreException,
                                        IOException

Update the notification topic name of the feature group.

Parameters:

notificationTopicName - feature group notification topic name.

Throws:

FeatureStoreException - FeatureStoreException

IOException - IOException

updateDeprecated

public void updateDeprecated()
                      throws FeatureStoreException,
                             IOException

Deprecate the feature group.

Throws:

FeatureStoreException - FeatureStoreException

IOException - IOException

updateDeprecated

public void updateDeprecated(Boolean deprecate)
                      throws FeatureStoreException,
                             IOException

Deprecate the feature group.

Parameters:

deprecate - identifies if feature group should be deprecated.

Throws:

FeatureStoreException - FeatureStoreException

IOException - IOException

updateFeatureDescription

public void updateFeatureDescription(String featureName,
                                     String description)
                              throws FeatureStoreException,
                                     IOException

Update the description of a single feature.

Parameters:

featureName - Name of the feature

description - Description of the feature

Throws:

FeatureStoreException - FeatureStoreException

IOException - IOException

read

public abstract T read()
                throws FeatureStoreException,
                       IOException

Throws:

FeatureStoreException

IOException

read

public abstract T read(boolean online)
                throws FeatureStoreException,
                       IOException

Throws:

FeatureStoreException

IOException

read

public abstract T read(Map<String,String> readOptions)
                throws FeatureStoreException,
                       IOException

Throws:

FeatureStoreException

IOException

read

public abstract T read(boolean online,
                       Map<String,String> readOptions)
                throws FeatureStoreException,
                       IOException

Throws:

FeatureStoreException

IOException

read

public abstract T read(String wallclockTime)
                throws FeatureStoreException,
                       IOException,
                       ParseException

Throws:

FeatureStoreException

IOException

ParseException

read

public abstract T read(String wallclockTime,
                       Map<String,String> readOptions)
                throws FeatureStoreException,
                       IOException,
                       ParseException

Throws:

FeatureStoreException

IOException

ParseException

asOf

public abstract QueryBase asOf(String wallclockTime)
                        throws FeatureStoreException,
                               ParseException

Throws:

FeatureStoreException

ParseException

asOf

public abstract QueryBase asOf(String wallclockTime,
                               String excludeUntil)
                        throws FeatureStoreException,
                               ParseException

Throws:

FeatureStoreException

ParseException

show

public abstract void show(int numRows)
                   throws FeatureStoreException,
                          IOException

Throws:

FeatureStoreException

IOException

show

public abstract void show(int numRows,
                          boolean online)
                   throws FeatureStoreException,
                          IOException

Throws:

FeatureStoreException

IOException

insert

public abstract void insert(T featureData)
                     throws IOException,
                            FeatureStoreException,
                            ParseException

Throws:

IOException

FeatureStoreException

ParseException

insert

public abstract void insert(T featureData,
                            Map<String,String> writeOptions)
                     throws FeatureStoreException,
                            IOException,
                            ParseException

Throws:

FeatureStoreException

IOException

ParseException

insert

public abstract void insert(T featureData,
                            Storage storage)
                     throws IOException,
                            FeatureStoreException,
                            ParseException

Throws:

IOException

FeatureStoreException

ParseException

insert

public abstract void insert(T featureData,
                            boolean overwrite)
                     throws IOException,
                            FeatureStoreException,
                            ParseException

Throws:

IOException

FeatureStoreException

ParseException

insert

public abstract void insert(T featureData,
                            Storage storage,
                            boolean overwrite)
                     throws IOException,
                            FeatureStoreException,
                            ParseException

Throws:

IOException

FeatureStoreException

ParseException

insert

public abstract void insert(T featureData,
                            boolean overwrite,
                            Map<String,String> writeOptions)
                     throws FeatureStoreException,
                            IOException,
                            ParseException

Throws:

FeatureStoreException

IOException

ParseException

insert

public abstract void insert(T featureData,
                            HudiOperationType operation)
                     throws FeatureStoreException,
                            IOException,
                            ParseException

Throws:

FeatureStoreException

IOException

ParseException

insert

public abstract void insert(T featureData,
                            Storage storage,
                            boolean overwrite,
                            HudiOperationType operation,
                            Map<String,String> writeOptions)
                     throws FeatureStoreException,
                            IOException,
                            ParseException

Throws:

FeatureStoreException

IOException

ParseException

insert

public abstract void insert(T featureData,
                            JobConfiguration jobConfiguration)
                     throws FeatureStoreException,
                            IOException,
                            ParseException

Throws:

FeatureStoreException

IOException

ParseException

insert

public abstract void insert(T featureData,
                            boolean overwrite,
                            Map<String,String> writeOptions,
                            JobConfiguration jobConfiguration)
                     throws FeatureStoreException,
                            IOException,
                            ParseException

Throws:

FeatureStoreException

IOException

ParseException

commitDeleteRecord

public abstract void commitDeleteRecord(T featureData)
                                 throws FeatureStoreException,
                                        IOException,
                                        ParseException

Throws:

FeatureStoreException

IOException

ParseException

commitDeleteRecord

public abstract void commitDeleteRecord(T featureData,
                                        Map<String,String> writeOptions)
                                 throws FeatureStoreException,
                                        IOException,
                                        ParseException

Throws:

FeatureStoreException

IOException

ParseException

commitDetails

public abstract Map<Long,Map<String,String>> commitDetails()
                                                    throws IOException,
                                                           FeatureStoreException,
                                                           ParseException

Throws:

IOException

FeatureStoreException

ParseException

commitDetails

public abstract Map<Long,Map<String,String>> commitDetails(Integer limit)
                                                    throws IOException,
                                                           FeatureStoreException,
                                                           ParseException

Throws:

IOException

FeatureStoreException

ParseException

commitDetails

public abstract Map<Long,Map<String,String>> commitDetails(String wallclockTime)
                                                    throws IOException,
                                                           FeatureStoreException,
                                                           ParseException

Throws:

IOException

FeatureStoreException

ParseException

commitDetails

public abstract Map<Long,Map<String,String>> commitDetails(String wallclockTime,
                                                           Integer limit)
                                                    throws IOException,
                                                           FeatureStoreException,
                                                           ParseException

Throws:

IOException

FeatureStoreException

ParseException

selectFeatures

public abstract QueryBase selectFeatures(List<Feature> features)

select

public abstract QueryBase select(List<String> features)

selectAll

public abstract QueryBase selectAll()

selectExceptFeatures

public abstract QueryBase selectExceptFeatures(List<Feature> features)

selectExcept

public abstract QueryBase selectExcept(List<String> features)

insertStream

public abstract Object insertStream(T featureData)
                             throws Exception

Throws:

Exception

insertStream

public abstract Object insertStream(T featureData,
                                    String queryName)
                             throws Exception

Throws:

Exception

insertStream

public abstract Object insertStream(T featureData,
                                    Map<String,String> writeOptions)
                             throws Exception

Throws:

Exception

insertStream

public abstract Object insertStream(T featureData,
                                    String queryName,
                                    Map<String,String> writeOptions)
                             throws Exception

Throws:

Exception

insertStream

public abstract Object insertStream(T featureData,
                                    String queryName,
                                    String outputMode)
                             throws Exception

Throws:

Exception

insertStream

public abstract Object insertStream(T featureData,
                                    String queryName,
                                    String outputMode,
                                    String checkpointLocation)
                             throws Exception

Throws:

Exception

insertStream

public abstract Object insertStream(T featureData,
                                    String queryName,
                                    String outputMode,
                                    boolean awaitTermination,
                                    Long timeout)
                             throws Exception

Throws:

Exception

insertStream

public abstract Object insertStream(T featureData,
                                    String queryName,
                                    String outputMode,
                                    boolean awaitTermination,
                                    Long timeout,
                                    String checkpointLocation)
                             throws Exception

Throws:

Exception

insertStream

public abstract Object insertStream(T featureData,
                                    String queryName,
                                    String outputMode,
                                    boolean awaitTermination,
                                    Long timeout,
                                    String checkpointLocation,
                                    Map<String,String> writeOptions)
                             throws Exception

Throws:

Exception

insertStream

public abstract Object insertStream(T featureData,
                                    String queryName,
                                    String outputMode,
                                    boolean awaitTermination,
                                    String checkpointLocation)
                             throws Exception

Throws:

Exception

insertStream

public abstract Object insertStream(T featureData,
                                    String queryName,
                                    String outputMode,
                                    boolean awaitTermination,
                                    Long timeout,
                                    String checkpointLocation,
                                    Map<String,String> writeOptions,
                                    JobConfiguration jobConfiguration)
                             throws Exception

Throws:

Exception

updateFeatures

public abstract void updateFeatures(List<Feature> features)
                             throws FeatureStoreException,
                                    IOException,
                                    ParseException

Update the metadata of multiple features. Currently only feature description updates are supported.

Parameters:

features - List of Feature metadata objects

Throws:

FeatureStoreException - FeatureStoreException

IOException - IOException

ParseException - ParseException

updateFeatures

public abstract void updateFeatures(Feature feature)
                             throws FeatureStoreException,
                                    IOException,
                                    ParseException

Update the metadata of multiple features. Currently only feature description updates are supported.

Parameters:

feature - Feature metadata object

Throws:

FeatureStoreException - FeatureStoreException

IOException - IOException

ParseException - ParseException

appendFeatures

public abstract void appendFeatures(List<Feature> features)
                             throws FeatureStoreException,
                                    IOException,
                                    ParseException

Append features to the schema of the feature group. It is only possible to append features to a feature group. Removing features is considered a breaking change.

Parameters:

features - list of Feature metadata objects

Throws:

FeatureStoreException - FeatureStoreException

IOException - IOException

ParseException - ParseException

appendFeatures

public abstract void appendFeatures(Feature features)
                             throws FeatureStoreException,
                                    IOException,
                                    ParseException

Append a single feature to the schema of the feature group. It is only possible to append features to a feature group. Removing features is considered a breaking change.

Parameters:

features - List of Feature metadata objects

Throws:

FeatureStoreException - FeatureStoreException

IOException - IOException

ParseException - ParseException

updateStatisticsConfig

public void updateStatisticsConfig()
                            throws FeatureStoreException,
                                   IOException

Update the statistics configuration of the feature group. Change the `enabled`, `histograms`, `correlations` or `columns` attributes and persist the changes by calling this method.

Throws:

FeatureStoreException - FeatureStoreException

IOException - IOException

computeStatistics

public abstract Statistics computeStatistics()
                                      throws FeatureStoreException,
                                             IOException,
                                             ParseException

Recompute the statistics for the feature group and save them to the feature store.

Returns:

statistics object of computed statistics

Throws:

FeatureStoreException - FeatureStoreException

IOException - IOException

ParseException

computeStatistics

public abstract Statistics computeStatistics(String wallclockTime)
                                      throws FeatureStoreException,
                                             IOException,
                                             ParseException

Throws:

FeatureStoreException

IOException

ParseException

getStatistics

public abstract Statistics getStatistics()
                                  throws FeatureStoreException,
                                         IOException

Get the last statistics commit for the feature group.

Returns:

statistics object of latest commit

Throws:

FeatureStoreException - FeatureStoreException

IOException - IOException

getSubject

public Subject getSubject()
                   throws FeatureStoreException,
                          IOException

Throws:

FeatureStoreException

IOException

unloadSubject

public void unloadSubject()

getFeature

public Feature getFeature(String name)
                   throws FeatureStoreException

Retrieve a feature of the feature group by name.

Parameters:

name - feature name

Returns:

Feature metadata object

Throws:

FeatureStoreException - FeatureStoreException

getPrimaryKeys

public List<String> getPrimaryKeys()

getComplexFeatures

public List<String> getComplexFeatures()

getAvroSchema

public String getAvroSchema()
                     throws FeatureStoreException,
                            IOException

Throws:

FeatureStoreException

IOException

getFeatureAvroSchema

public String getFeatureAvroSchema(String featureName)
                            throws FeatureStoreException,
                                   IOException

Throws:

FeatureStoreException

IOException

getEncodedAvroSchema

public String getEncodedAvroSchema()
                            throws FeatureStoreException,
                                   IOException

Throws:

FeatureStoreException

IOException

getDeserializedEncodedAvroSchema

public org.apache.avro.Schema getDeserializedEncodedAvroSchema()
                                                        throws FeatureStoreException,
                                                               IOException

Throws:

FeatureStoreException

IOException

getDeserializedAvroSchema

public org.apache.avro.Schema getDeserializedAvroSchema()
                                                 throws FeatureStoreException,
                                                        IOException

Throws:

FeatureStoreException

IOException

getLatestOnlineIngestion

public OnlineIngestion getLatestOnlineIngestion()
                                         throws FeatureStoreException,
                                                IOException

Throws:

FeatureStoreException

IOException

getOnlineIngestion

public OnlineIngestion getOnlineIngestion(Integer id)
                                   throws FeatureStoreException,
                                          IOException

Throws:

FeatureStoreException

IOException