Package com.logicalclocks.hsfs.engine

Class FeatureGroupUtils

java.lang.Object

com.logicalclocks.hsfs.engine.FeatureGroupUtils

public class FeatureGroupUtils
extends Object

Constructor Summary

Constructors

Constructor	Description
FeatureGroupUtils()

Method Summary

Modifier and Type	Method	Description
Map<Long,Map<String,String>>	getCommitDetails(FeatureGroupBase featureGroup, String wallclockTime, Integer limit)
List<String>	getComplexFeatures(List<Feature> features)
String	getDatasetType(String path)
static Date	getDateFromDateString(String inputDate)
org.apache.avro.Schema	getDeserializedAvroSchema(String avroSchema)
org.apache.avro.Schema	getDeserializedEncodedAvroSchema(org.apache.avro.Schema schema, List<String> complexFeatures)
String	getEncodedAvroSchema(org.apache.avro.Schema schema, List<String> complexFeatures)
String	getFeatureAvroSchema(String featureName, org.apache.avro.Schema schema)
String	getFgName(FeatureGroupBase featureGroup)
static Map<String,byte[]>	getHeaders(FeatureGroupBase featureGroup, Long numEntries)
static Map<String,byte[]>	getHeaders(FeatureGroupBase featureGroup, Long numEntries, Map<String,String> options)
static Map<String,byte[]>	getHeaders(FeatureGroupBase featureGroup, Long numEntries, Map<String,String> options, String operation)
static Map<String,byte[]>	getHeaders(FeatureGroupBase featureGroup, Long numEntries, Map<String,String> options, String operation, Storage storage)	Kafka headers for the records of one write to the online topic.
String	getHiveServerConnection(FeatureGroupBase featureGroup, String connectionString)
scala.collection.Seq<String>	getPartitionColumns(FeatureGroupBase offlineFeatureGroup)
scala.collection.Seq<String>	getPrimaryColumns(FeatureGroupBase offlineFeatureGroup)
Subject	getSubject(FeatureGroupBase featureGroup)
String	getTableName(FeatureGroupBase offlineFeatureGroup)
static Long	getTimeStampFromDateString(String inputDate)
String	timeStampToHudiFormat(Long commitedOnTimeStamp)
void	verifyAttributeKeyNames(FeatureGroupBase featureGroup, List<String> partitionKeyNames, String precombineKeyName)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Details

FeatureGroupUtils

public FeatureGroupUtils()

Method Details

getTableName

public String getTableName(FeatureGroupBase offlineFeatureGroup)

getPartitionColumns

public scala.collection.Seq<String> getPartitionColumns(FeatureGroupBase offlineFeatureGroup)

getPrimaryColumns

public scala.collection.Seq<String> getPrimaryColumns(FeatureGroupBase offlineFeatureGroup)

getFgName

public String getFgName(FeatureGroupBase featureGroup)

getHiveServerConnection

public String getHiveServerConnection(FeatureGroupBase featureGroup,
 String connectionString)
                               throws IOException,
FeatureStoreException

Throws:

IOException

FeatureStoreException

getDateFromDateString

public static Date getDateFromDateString(String inputDate)
                                  throws FeatureStoreException,
ParseException

Throws:

FeatureStoreException

ParseException

getTimeStampFromDateString

public static Long getTimeStampFromDateString(String inputDate)
                                       throws FeatureStoreException,
ParseException

Throws:

FeatureStoreException

ParseException

timeStampToHudiFormat

public String timeStampToHudiFormat(Long commitedOnTimeStamp)

getCommitDetails

public Map<Long,Map<String,String>> getCommitDetails(FeatureGroupBase featureGroup,
 String wallclockTime,
 Integer limit)
                                              throws FeatureStoreException,
IOException,
ParseException

Throws:

FeatureStoreException

IOException

ParseException

getComplexFeatures

public List<String> getComplexFeatures(List<Feature> features)

getFeatureAvroSchema

public String getFeatureAvroSchema(String featureName,
 org.apache.avro.Schema schema)
                            throws FeatureStoreException,
IOException

Throws:

FeatureStoreException

IOException

getDeserializedEncodedAvroSchema

public org.apache.avro.Schema getDeserializedEncodedAvroSchema(org.apache.avro.Schema schema,
 List<String> complexFeatures)
                                                        throws FeatureStoreException,
IOException

Throws:

FeatureStoreException

IOException

getEncodedAvroSchema

public String getEncodedAvroSchema(org.apache.avro.Schema schema,
 List<String> complexFeatures)
                            throws FeatureStoreException,
IOException

Throws:

FeatureStoreException

IOException

getDeserializedAvroSchema

public org.apache.avro.Schema getDeserializedAvroSchema(String avroSchema)
                                                 throws FeatureStoreException,
IOException

Throws:

FeatureStoreException

IOException

verifyAttributeKeyNames

public void verifyAttributeKeyNames(FeatureGroupBase featureGroup,
 List<String> partitionKeyNames,
 String precombineKeyName)
                             throws FeatureStoreException

Throws:

FeatureStoreException

getSubject

public Subject getSubject(FeatureGroupBase featureGroup)
                   throws FeatureStoreException,
IOException

Throws:

FeatureStoreException

IOException

getDatasetType

public String getDatasetType(String path)

getHeaders

public static Map<String,byte[]> getHeaders(FeatureGroupBase featureGroup,
 Long numEntries)
                                     throws FeatureStoreException,
IOException

Throws:

FeatureStoreException

IOException

getHeaders

public static Map<String,byte[]> getHeaders(FeatureGroupBase featureGroup,
 Long numEntries,
 Map<String,String> options)
                                     throws FeatureStoreException,
IOException

Throws:

FeatureStoreException

IOException

getHeaders

public static Map<String,byte[]> getHeaders(FeatureGroupBase featureGroup,
 Long numEntries,
 Map<String,String> options,
 String operation)
                                     throws FeatureStoreException,
IOException

Throws:

FeatureStoreException

IOException

getHeaders

public static Map<String,byte[]> getHeaders(FeatureGroupBase featureGroup,
 Long numEntries,
 Map<String,String> options,
 String operation,
 Storage storage)
                                     throws FeatureStoreException,
IOException

Kafka headers for the records of one write to the online topic.

Parameters:

featureGroup - FeatureGroupBase Feature Group hsfs metadata object

numEntries - number of records this write produces, reported by the online ingestion

options - options map; supported keys under "online_ingestion_options.*"

operation - "delete" for an online delete tombstone, null for an upsert

storage - which consumer of the topic is meant to ingest these records: Storage.ONLINE for OnlineFS alone, Storage.OFFLINE for the offline materialization job alone, null when both read them

Throws:

FeatureStoreException - If Client is not connected to Hopsworks

IOException - Generic IO exception.