| Modifier and Type | Method and Description |
|---|---|
void |
FeatureGroupBase.addTag(String name,
Object value)
Add name/value tag to the feature group.
|
abstract void |
TrainingDatasetBase.addTag(String name,
Object value)
Add name/value tag to the training dataset.
|
abstract Statistics |
TrainingDatasetBase.computeStatistics()
Recompute the statistics for the entire training dataset and save them to the feature store.
|
void |
FeatureGroupBase.delete() |
abstract void |
TrainingDatasetBase.delete()
Delete training dataset and all associated metadata.
|
void |
FeatureGroupBase.deleteTag(String name)
Delete a tag of the feature group.
|
abstract void |
TrainingDatasetBase.deleteTag(String name)
Delete a tag of the training dataset.
|
StorageConnector.AdlsConnector |
FeatureStoreBase.getAdlsConnector(String name)
Get a previously created Adls compliant storage connector from the feature store.
|
String |
FeatureGroupBase.getAvroSchema() |
StorageConnector.BigqueryConnector |
FeatureStoreBase.getBigqueryConnector(String name)
Get a previously created BigQuery compliant storage connector from the feature store.
|
org.apache.avro.Schema |
FeatureGroupBase.getDeserializedAvroSchema() |
org.apache.avro.Schema |
FeatureGroupBase.getDeserializedEncodedAvroSchema() |
String |
FeatureGroupBase.getEncodedAvroSchema() |
Feature |
FeatureGroupBase.getFeature(String name)
Retrieve a feature of the feature group by name.
|
String |
FeatureGroupBase.getFeatureAvroSchema(String featureName) |
abstract Object |
HopsworksConnectionBase.getFeatureStore() |
FeatureStore |
HopsworksConnection.getFeatureStore()
Retrieve the project feature store.
|
abstract Object |
HopsworksConnectionBase.getFeatureStore(String name) |
FeatureStore |
HopsworksConnection.getFeatureStore(String name)
Retrieve a feature store based on name.
|
List<Object> |
FeatureViewBase.getFeatureVector(Map<String,Object> entry)
Returns assembled feature vector from online feature store.
|
List<Object> |
FeatureViewBase.getFeatureVector(Map<String,Object> entry,
boolean external)
Returns assembled feature vector from online feature store.
|
List<List<Object>> |
FeatureViewBase.getFeatureVectors(Map<String,List<Object>> entry)
Returns assembled feature vectors in batches from online feature store.
|
List<List<Object>> |
FeatureViewBase.getFeatureVectors(Map<String,List<Object>> entry,
boolean external)
Returns assembled feature vectors in batches from online feature store.
|
abstract Object |
FeatureStoreBase.getFeatureView(String name) |
FeatureView |
FeatureStore.getFeatureView(String name)
Get a feature view object from the selected feature store.
|
abstract Object |
FeatureStoreBase.getFeatureView(@NonNull String name,
@NonNull Integer version) |
FeatureView |
FeatureStore.getFeatureView(@NonNull String name,
@NonNull Integer version)
Get a feature view object from the selected feature store.
|
StorageConnector.GcsConnector |
FeatureStoreBase.getGcsConnector(String name)
Get a previously created Gcs compliant storage connector from the feature store.
|
StorageConnector.HopsFsConnector |
FeatureStoreBase.getHopsFsConnector(String name)
Get a previously created HopsFs compliant storage connector from the feature store.
|
StorageConnector.JdbcConnector |
FeatureStoreBase.getJdbcConnector(String name)
Get a previously created JDBC compliant storage connector from the feature store.
|
StorageConnector.KafkaConnector |
FeatureStoreBase.getKafkaConnector(String name)
Get a previously created Kafka compliant storage connector from the feature store.
|
StorageConnector.JdbcConnector |
FeatureStoreBase.getOnlineStorageConnector()
Get a previously created JDBC compliant storage connector from the feature store
to connect to the online feature store.
|
abstract FeatureGroupBase |
FeatureStoreBase.getOrCreateStreamFeatureGroup(@NonNull String name,
Integer version,
String description,
Boolean onlineEnabled,
TimeTravelFormat timeTravelFormat,
List<String> primaryKeys,
List<String> partitionKeys,
String eventTime,
String hudiPrecombineKey,
List<Feature> features,
StatisticsConfig statisticsConfig,
StorageConnector storageConnector,
String path)
Get a feature group metadata object or create a new one if it doesn't exists.
|
StreamFeatureGroup |
FeatureStore.getOrCreateStreamFeatureGroup(@NonNull String name,
Integer version,
String description,
Boolean onlineEnabled,
TimeTravelFormat timeTravelFormat,
List<String> primaryKeys,
List<String> partitionKeys,
String eventTime,
String hudiPrecombineKey,
List<Feature> features,
StatisticsConfig statisticsConfig,
StorageConnector storageConnector,
String path) |
abstract String |
StorageConnector.getPath(String subPath) |
HashSet<String> |
FeatureViewBase.getPrimaryKeys()
Set of primary key names that is used as keys in input dict object for `get_serving_vector` method.
|
Project |
HopsworksConnectionBase.getProject() |
String |
TrainingDatasetBase.getQuery() |
String |
TrainingDatasetBase.getQuery(boolean withLabel) |
String |
TrainingDatasetBase.getQuery(Storage storage) |
abstract String |
TrainingDatasetBase.getQuery(Storage storage,
boolean withLabel) |
StorageConnector.RedshiftConnector |
FeatureStoreBase.getRedshiftConnector(String name)
Get a previously created Redshift compliant storage connector from the feature store.
|
StorageConnector.S3Connector |
FeatureStoreBase.getS3Connector(String name)
Get a previously created S3 compliant storage connector from the feature store.
|
StorageConnector.SnowflakeConnector |
FeatureStoreBase.getSnowflakeConnector(String name)
Get a previously created Snowflake compliant storage connector from the feature store.
|
abstract Statistics |
TrainingDatasetBase.getStatistics()
Get the last statistics commit for the training dataset.
|
abstract Statistics |
TrainingDatasetBase.getStatistics(String commitTime)
Get the statistics of a specific commit time for the training dataset.
|
StorageConnector |
FeatureStoreBase.getStorageConnector(String name)
Get a previously created storage connector from the feature store.
|
abstract Object |
FeatureStoreBase.getStreamFeatureGroup(String name)
Get a feature group object with default version `1` from the feature store.
|
StreamFeatureGroup |
FeatureStore.getStreamFeatureGroup(String name)
Get a stream feature group object from the feature store.
|
abstract Object |
FeatureStoreBase.getStreamFeatureGroup(@NonNull String name,
@NonNull Integer version)
Get a feature group object from the feature store.
|
StreamFeatureGroup |
FeatureStore.getStreamFeatureGroup(@NonNull String name,
@NonNull Integer version)
Get a stream feature group object from the feature store.
|
Subject |
FeatureGroupBase.getSubject() |
Object |
FeatureGroupBase.getTag(String name)
Get a single tag value of the feature group.
|
abstract Object |
TrainingDatasetBase.getTag(String name)
Get a single tag value of the training dataset.
|
Map<String,Object> |
FeatureGroupBase.getTags()
Get all tags of the feature group.
|
abstract Map<String,Object> |
TrainingDatasetBase.getTags()
Get all tags of the training dataset.
|
void |
FeatureViewBase.initServing()
Initialise feature view to retrieve feature vector from online feature store.
|
void |
FeatureViewBase.initServing(Boolean batch,
Boolean external)
Initialise feature view to retrieve feature vector from online feature store.
|
Map<String,String> |
StorageConnector.KafkaConnector.kafkaOptions() |
abstract <T> T |
TrainingDatasetBase.read()
Read the content of the training dataset.
|
abstract Object |
TrainingDatasetBase.read(String split,
Map<String,String> readOptions)
Read a single split from the training dataset.
|
StorageConnector |
StorageConnector.refetch() |
void |
StreamFeatureGroup.save()
Save the feature group metadata on Hopsworks.
|
void |
StreamFeatureGroup.save(Map<String,String> writeOptions,
JobConfiguration materializationJobConfiguration)
Save the feature group metadata on Hopsworks.
|
void |
TrainingDatasetBase.setTimeSeriesSplits(Integer timeSplitSize,
String trainStart,
String trainEnd,
String valStart,
String valEnd,
String testStart,
String testEnd) |
abstract void |
TrainingDatasetBase.show(int numRows)
Show numRows from the training dataset (across all splits).
|
abstract Map<String,String> |
StorageConnector.sparkOptions() |
Map<String,String> |
StorageConnector.KafkaConnector.sparkOptions() |
void |
StorageConnector.S3Connector.update() |
void |
StorageConnector.RedshiftConnector.update() |
void |
StorageConnector.JdbcConnector.update() |
void |
FeatureGroupBase.updateDeprecated()
Deprecate the feature group.
|
void |
FeatureGroupBase.updateDeprecated(Boolean deprecate)
Deprecate the feature group.
|
void |
FeatureGroupBase.updateDescription(String description)
Update the description of the feature group.
|
void |
FeatureGroupBase.updateFeatureDescription(String featureName,
String description)
Update the description of a single feature.
|
void |
FeatureGroupBase.updateNotificationTopicName(String notificationTopicName)
Update the notification topic name of the feature group.
|
void |
FeatureGroupBase.updateStatisticsConfig()
Update the statistics configuration of the feature group.
|
abstract void |
TrainingDatasetBase.updateStatisticsConfig()
Update the statistics configuration of the training dataset.
|
protected void |
FeatureViewBase.validateTrainTestSplit(Float testSize,
String trainEnd,
String testStart) |
protected void |
FeatureViewBase.validateTrainValidationTestSplit(Float validationSize,
Float testSize,
String trainEnd,
String validationStart,
String validationEnd,
String testStart) |
| Constructor and Description |
|---|
Feature(String name,
String type,
Boolean primary,
Boolean partition) |
Feature(String name,
String type,
String onlineType,
Boolean primary,
Boolean partition,
String defaultValue,
String description) |
HopsworksConnection(String host,
int port,
String project,
software.amazon.awssdk.regions.Region region,
SecretStore secretStore,
boolean hostnameVerification,
String trustStorePath,
String certPath,
String apiKeyFilePath,
String apiKeyValue) |
| Modifier and Type | Method and Description |
|---|---|
FeatureStore |
HopsworksConnection.getFeatureStore()
Retrieve the project feature store.
|
FeatureStore |
HopsworksConnection.getFeatureStore(String name)
Retrieve a feature store based on name.
|
FeatureView |
FeatureStore.getFeatureView(String name)
Get a feature view object from the selected feature store.
|
FeatureView |
FeatureStore.getFeatureView(@NonNull String name,
@NonNull Integer version)
Get a feature view object from the selected feature store.
|
StreamFeatureGroup |
FeatureStore.getOrCreateStreamFeatureGroup(@NonNull String name,
Integer version,
String description,
Boolean onlineEnabled,
TimeTravelFormat timeTravelFormat,
List<String> primaryKeys,
List<String> partitionKeys,
String eventTime,
String hudiPrecombineKey,
List<Feature> features,
StatisticsConfig statisticsConfig,
StorageConnector storageConnector,
String path) |
Object |
FeatureStore.getStreamFeatureGroup(String name)
Get a stream feature group object from the feature store.
|
StreamFeatureGroup |
FeatureStore.getStreamFeatureGroup(@NonNull String name,
@NonNull Integer version)
Get a stream feature group object from the feature store.
|
void |
StreamFeatureGroup.save()
Save the feature group metadata on Hopsworks.
|
void |
StreamFeatureGroup.save(Map<String,String> writeOptions,
JobConfiguration materializationJobConfiguration)
Save the feature group metadata on Hopsworks.
|
| Constructor and Description |
|---|
HopsworksConnection(String host,
int port,
String project,
software.amazon.awssdk.regions.Region region,
SecretStore secretStore,
boolean hostnameVerification,
String trustStorePath,
String certPath,
String apiKeyFilePath,
String apiKeyValue) |
| Modifier and Type | Method and Description |
|---|---|
String |
BeamEngine.addFile(String filePath) |
FeatureView |
FeatureViewEngine.get(FeatureStore featureStore,
String name,
Integer version) |
static BeamEngine |
BeamEngine.getInstance() |
Map<String,String> |
BeamEngine.getKafkaConfig(FeatureGroupBase featureGroup,
Map<String,String> writeOptions) |
StreamFeatureGroup |
FeatureGroupEngine.getOrCreateFeatureGroup(FeatureStore featureStore,
@NonNull String name,
Integer version,
String description,
Boolean onlineEnabled,
TimeTravelFormat timeTravelFormat,
List<String> primaryKeys,
List<String> partitionKeys,
String eventTime,
String hudiPrecombineKey,
List<Feature> features,
StatisticsConfig statisticsConfig,
StorageConnector storageConnector,
String path) |
StreamFeatureGroup |
FeatureGroupEngine.getStreamFeatureGroup(FeatureStore featureStore,
String fgName,
Integer fgVersion) |
List<StreamFeatureGroup> |
FeatureGroupEngine.getStreamFeatureGroups(FeatureStore featureStore,
String fgName) |
BeamProducer |
BeamEngine.insertStream(StreamFeatureGroup streamFeatureGroup,
Map<String,String> writeOptions) |
void |
FeatureGroupEngine.save(StreamFeatureGroup featureGroup,
List<String> partitionKeys,
String precombineKeyName,
Map<String,String> writeOptions,
JobConfiguration materializationJobConfiguration) |
FeatureView |
FeatureViewEngine.update(FeatureView featureView) |
| Constructor and Description |
|---|
BeamProducer(String topic,
Map<String,String> properties,
org.apache.avro.Schema schema,
org.apache.avro.Schema encodedSchema,
Map<String,org.apache.avro.Schema> deserializedComplexFeatureSchemas,
List<String> primaryKeys,
StreamFeatureGroup streamFeatureGroup) |
| Modifier and Type | Method and Description |
|---|---|
T |
QueryBase.asOf(String wallclockTime)
Perform time travel on the given Query.
|
T |
QueryBase.asOf(String wallclockTime,
String excludeUntil)
Perform time travel on the given Query.
|
String |
FsQueryBase.getStorageQuery(Storage storage) |
T |
QueryBase.pullChanges(String wallclockStartTime,
String wallclockEndTime)
Deprecated.
use asOf(wallclockEndTime, wallclockStartTime) instead
|
abstract void |
FsQueryBase.registerHudiFeatureGroups(Map<String,String> readOptions) |
abstract void |
FsQueryBase.registerOnDemandFeatureGroups() |
| Modifier and Type | Method and Description |
|---|---|
String |
Engine.addFile(String filePath) |
abstract String |
EngineBase.addFile(String filePath) |
void |
FeatureGroupEngineBase.addTag(FeatureGroupBase featureGroupBase,
String name,
Object value) |
void |
FeatureViewEngineBase.addTag(FeatureViewBase featureViewBase,
String name,
Object value) |
void |
FeatureViewEngineBase.addTag(FeatureViewBase featureViewBase,
String name,
Object value,
Integer trainingDataVersion) |
void |
FeatureGroupEngineBase.delete(FeatureGroupBase featureGroupBase) |
void |
FeatureViewEngineBase.delete(T3 featureStoreBase,
String name) |
void |
FeatureViewEngineBase.delete(T3 featureStoreBase,
String name,
Integer version) |
void |
FeatureGroupEngineBase.deleteTag(FeatureGroupBase featureGroupBase,
String name) |
void |
FeatureViewEngineBase.deleteTag(FeatureViewBase featureViewBase,
String name) |
void |
FeatureViewEngineBase.deleteTag(FeatureViewBase featureViewBase,
String name,
Integer trainingDataVersion) |
void |
FeatureViewEngineBase.deleteTrainingData(T2 featureView) |
void |
FeatureViewEngineBase.deleteTrainingData(T2 featureView,
Integer trainingDataVersion) |
void |
FeatureViewEngineBase.deleteTrainingDatasetOnly(T2 featureView) |
void |
FeatureViewEngineBase.deleteTrainingDatasetOnly(T2 featureView,
Integer trainingDataVersion) |
FeatureView |
FeatureViewEngine.get(FeatureStore featureStore,
String name,
Integer version) |
List<FeatureViewBase> |
FeatureViewEngineBase.get(T3 featureStoreBase,
String name) |
abstract T2 |
FeatureViewEngineBase.get(T3 featureStore,
String name,
Integer version) |
T2 |
FeatureViewEngineBase.get(T3 featureStoreBase,
String name,
Integer version,
Class<T2> fvType) |
T1 |
FeatureViewEngineBase.getBatchQuery(T2 featureView,
Date startTime,
Date endTime,
Boolean withLabels,
Integer trainingDataVersion,
Class<T1> queryType) |
Map<Long,Map<String,String>> |
FeatureGroupUtils.getCommitDetails(FeatureGroupBase featureGroup,
String wallclockTime,
Integer limit) |
Map<String,org.apache.avro.io.DatumReader<Object>> |
VectorServer.getComplexFeatureSchemas(List<TrainingDatasetFeature> features) |
static Date |
FeatureGroupUtils.getDateFromDateString(String inputDate) |
org.apache.avro.Schema |
FeatureGroupUtils.getDeserializedAvroSchema(String avroSchema) |
org.apache.avro.Schema |
FeatureGroupUtils.getDeserializedEncodedAvroSchema(org.apache.avro.Schema schema,
List<String> complexFeatures) |
String |
FeatureGroupUtils.getEncodedAvroSchema(org.apache.avro.Schema schema,
List<String> complexFeatures) |
String |
FeatureGroupUtils.getFeatureAvroSchema(String featureName,
org.apache.avro.Schema schema) |
List<Object> |
VectorServer.getFeatureVector(FeatureViewBase featureViewBase,
Map<String,Object> entry) |
List<Object> |
VectorServer.getFeatureVector(FeatureViewBase featureViewBase,
Map<String,Object> entry,
boolean external) |
List<Object> |
VectorServer.getFeatureVector(Map<String,Object> entry) |
List<List<Object>> |
VectorServer.getFeatureVectors(FeatureViewBase featureViewBase,
Map<String,List<Object>> entry,
boolean external) |
List<List<Object>> |
VectorServer.getFeatureVectors(Map<String,List<Object>> entry) |
String |
FeatureGroupUtils.getHiveServerConnection(FeatureGroupBase featureGroup,
String connectionString) |
static Engine |
Engine.getInstance() |
Map<String,String> |
Engine.getKafkaConfig(FeatureGroupBase featureGroup,
Map<String,String> writeOptions) |
abstract Map<String,String> |
EngineBase.getKafkaConfig(FeatureGroupBase featureGroup,
Map<String,String> writeOptions) |
StreamFeatureGroup |
FeatureGroupEngine.getOrCreateFeatureGroup(FeatureStore featureStore,
@NonNull String name,
Integer version,
String description,
Boolean onlineEnabled,
TimeTravelFormat timeTravelFormat,
List<String> primaryKeys,
List<String> partitionKeys,
String eventTime,
String hudiPrecombineKey,
List<Feature> features,
StatisticsConfig statisticsConfig,
StorageConnector storageConnector,
String path) |
StreamFeatureGroup |
FeatureGroupEngine.getStreamFeatureGroup(FeatureStore featureStore,
String fgName,
Integer fgVersion) |
List<StreamFeatureGroup> |
FeatureGroupEngine.getStreamFeatureGroups(FeatureStore featureStore,
String fgName) |
Subject |
FeatureGroupUtils.getSubject(FeatureGroupBase featureGroup) |
Object |
FeatureGroupEngineBase.getTag(FeatureGroupBase featureGroupBase,
String name) |
Object |
FeatureViewEngineBase.getTag(FeatureViewBase featureViewBase,
String name) |
Object |
FeatureViewEngineBase.getTag(FeatureViewBase featureViewBase,
String name,
Integer trainingDataVersion) |
Map<String,Object> |
FeatureGroupEngineBase.getTags(FeatureGroupBase featureGroupBase) |
Map<String,Object> |
FeatureViewEngineBase.getTags(FeatureViewBase featureViewBase) |
Map<String,Object> |
FeatureViewEngineBase.getTags(FeatureViewBase featureViewBase,
Integer trainingDataVersion) |
static Long |
FeatureGroupUtils.getTimeStampFromDateString(String inputDate) |
void |
VectorServer.initPreparedStatement(FeatureStoreBase featureStoreBase,
List<TrainingDatasetFeature> features,
List<ServingPreparedStatement> servingPreparedStatements,
boolean batch,
boolean external) |
void |
VectorServer.initPreparedStatement(FeatureViewBase featureViewBase,
boolean batch) |
void |
VectorServer.initPreparedStatement(FeatureViewBase featureViewBase,
boolean batch,
boolean external) |
void |
VectorServer.initServing(FeatureViewBase featureViewBase,
boolean batch) |
void |
VectorServer.initServing(FeatureViewBase featureViewBase,
boolean batch,
boolean external) |
static List<TrainingDatasetFeature> |
FeatureViewEngineBase.makeLabelFeatures(QueryBase query,
List<String> labels) |
void |
FeatureGroupEngine.save(StreamFeatureGroup featureGroup,
List<String> partitionKeys,
String precombineKeyName,
Map<String,String> writeOptions,
JobConfiguration materializationJobConfiguration) |
T2 |
FeatureViewEngineBase.save(T2 featureViewBase,
Class<T2> fvType) |
void |
CodeEngine.saveCode(FeatureGroupBase featureGroup) |
void |
CodeEngine.saveCode(TrainingDatasetBase trainingDatasetBase) |
protected <T extends FeatureGroupBase> |
FeatureGroupEngineBase.saveExtennalFeatureGroupMetaData(T externalFeatureGroup,
Class<T> fgClass) |
void |
VectorServer.setupHikariPool(FeatureStoreBase featureStoreBase,
Boolean external) |
FeatureView |
FeatureViewEngine.update(FeatureView featureView) |
abstract T2 |
FeatureViewEngineBase.update(T2 featureView) |
<T extends FeatureGroupBase> |
FeatureGroupEngineBase.updateDeprecated(FeatureGroupBase featureGroup,
Boolean deprecate,
Class<T> fgClass) |
<T extends FeatureGroupBase> |
FeatureGroupEngineBase.updateDescription(FeatureGroupBase featureGroup,
String description,
Class<T> fgClass) |
<T extends FeatureGroupBase> |
FeatureGroupEngineBase.updateFeatures(FeatureGroupBase featureGroup,
List<Feature> features,
Class<T> fgClass) |
<T extends FeatureGroupBase> |
FeatureGroupEngineBase.updateNotificationTopicName(FeatureGroupBase featureGroup,
String notificationTopicName,
Class<T> fgClass) |
<T extends FeatureGroupBase> |
FeatureGroupEngineBase.updateStatisticsConfig(FeatureGroupBase featureGroup,
Class<T> fgClass) |
void |
FeatureGroupUtils.verifyAttributeKeyNames(FeatureGroupBase featureGroup,
List<String> partitionKeyNames,
String precombineKeyName) |
List<T> |
Engine.writeStream(StreamFeatureGroup streamFeatureGroup,
List<T> featureData,
Map<String,String> writeOptions) |
| Modifier and Type | Method and Description |
|---|---|
FeatureStore |
HopsworksConnection.getFeatureStore()
Retrieve the project feature store.
|
FeatureStore |
HopsworksConnection.getFeatureStore(String name)
Retrieve a feature store based on name.
|
FeatureView |
FeatureStore.getFeatureView(String name)
Get a feature view object from the selected feature store.
|
FeatureView |
FeatureStore.getFeatureView(@NonNull String name,
@NonNull Integer version)
Get a feature view object from the selected feature store.
|
StreamFeatureGroup |
FeatureStore.getOrCreateStreamFeatureGroup(@NonNull String name,
Integer version,
String description,
Boolean onlineEnabled,
TimeTravelFormat timeTravelFormat,
List<String> primaryKeys,
List<String> partitionKeys,
String eventTime,
String hudiPrecombineKey,
List<Feature> features,
StatisticsConfig statisticsConfig,
StorageConnector storageConnector,
String path) |
StreamFeatureGroup |
FeatureStore.getStreamFeatureGroup(String name)
Get a stream feature group object from the feature store.
|
StreamFeatureGroup |
FeatureStore.getStreamFeatureGroup(@NonNull String name,
@NonNull Integer version)
Get a stream feature group object from the feature store.
|
void |
StreamFeatureGroup.save()
Save the feature group metadata on Hopsworks.
|
void |
StreamFeatureGroup.save(Map<String,String> writeOptions,
JobConfiguration materializationJobConfiguration)
Save the feature group metadata on Hopsworks.
|
| Constructor and Description |
|---|
HopsworksConnection(String host,
int port,
String project,
software.amazon.awssdk.regions.Region region,
SecretStore secretStore,
boolean hostnameVerification,
String trustStorePath,
String certPath,
String apiKeyFilePath,
String apiKeyValue) |
| Modifier and Type | Method and Description |
|---|---|
FeatureView |
FeatureViewEngine.get(FeatureStore featureStore,
String name,
Integer version) |
Map<String,String> |
FlinkEngine.getKafkaConfig(FeatureGroupBase featureGroup,
Map<String,String> writeOptions) |
StreamFeatureGroup |
FeatureGroupEngine.getOrCreateFeatureGroup(FeatureStore featureStore,
@NonNull String name,
Integer version,
String description,
Boolean onlineEnabled,
TimeTravelFormat timeTravelFormat,
List<String> primaryKeys,
List<String> partitionKeys,
String eventTime,
String hudiPrecombineKey,
List<Feature> features,
StatisticsConfig statisticsConfig,
StorageConnector storageConnector,
String path) |
StreamFeatureGroup |
FeatureGroupEngine.getStreamFeatureGroup(FeatureStore featureStore,
String fgName,
Integer fgVersion) |
List<StreamFeatureGroup> |
FeatureGroupEngine.getStreamFeatureGroups(FeatureStore featureStore,
String fgName) |
void |
FeatureGroupEngine.save(StreamFeatureGroup featureGroup,
List<String> partitionKeys,
String precombineKeyName,
Map<String,String> writeOptions,
JobConfiguration materializationJobConfiguration) |
FeatureView |
FeatureViewEngine.update(FeatureView featureView) |
org.apache.flink.streaming.api.datastream.DataStreamSink<?> |
FlinkEngine.writeDataStream(StreamFeatureGroup streamFeatureGroup,
org.apache.flink.streaming.api.datastream.DataStream<?> dataStream,
Map<String,String> writeOptions) |
| Modifier and Type | Method and Description |
|---|---|
void |
TagsApi.add(FeatureGroupBase featureGroupBase,
String name,
Object value) |
void |
TagsApi.add(FeatureViewBase featureViewBase,
Integer trainingDatasetVersion,
String name,
Object value) |
void |
TagsApi.add(FeatureViewBase featureViewBase,
String name,
Object value) |
void |
TagsApi.add(TrainingDatasetBase trainingDatasetBase,
String name,
Object value) |
<T> FsQueryBase |
QueryConstructorApi.constructQuery(FeatureStoreBase featureStoreBase,
QueryBase queryBase,
Class<T> fsQueryType) |
<T extends TrainingDatasetBase> |
FeatureViewApi.createTrainingData(String featureViewName,
Integer featureViewVersion,
TrainingDatasetBase trainingData,
Class<T> tdType) |
TrainingDatasetBase |
TrainingDatasetApi.createTrainingDataset(TrainingDatasetBase trainingDatasetBase) |
void |
FeatureGroupApi.delete(FeatureGroupBase featureGroupBase) |
void |
FeatureViewApi.delete(FeatureStoreBase featureStoreBase,
String name) |
void |
FeatureViewApi.delete(FeatureStoreBase featureStoreBase,
String name,
Integer version) |
void |
TrainingDatasetApi.delete(TrainingDatasetBase trainingDatasetBase) |
void |
FeatureGroupApi.deleteContent(FeatureGroupBase featureGroup) |
void |
TagsApi.deleteTag(FeatureGroupBase featureGroup,
String name) |
void |
TagsApi.deleteTag(FeatureViewBase featureViewBase,
Integer trainingDatasetVersion,
String name) |
void |
TagsApi.deleteTag(FeatureViewBase featureViewBase,
String name) |
void |
TagsApi.deleteTag(TrainingDatasetBase trainingDatasetBase,
String name) |
void |
FeatureViewApi.deleteTrainingData(FeatureStoreBase featureStoreBase,
String name,
Integer version) |
void |
FeatureViewApi.deleteTrainingData(FeatureStoreBase featureStoreBase,
String featureViewName,
Integer featureViewVersion,
Integer trainingDataVersion) |
void |
FeatureViewApi.deleteTrainingDatasetOnly(FeatureStoreBase featureStoreBase,
String name,
Integer version) |
void |
FeatureViewApi.deleteTrainingDatasetOnly(FeatureStoreBase featureStoreBase,
String name,
Integer version,
Integer trainingDataVersion) |
FeatureGroupCommit |
FeatureGroupApi.featureGroupCommit(FeatureGroupBase featureGroup,
FeatureGroupCommit featureGroupCommit) |
Map<String,Object> |
TagsApi.get(FeatureGroupBase featureGroupBase) |
Statistics |
StatisticsApi.get(FeatureGroupBase featureGroup,
String commitTime) |
Object |
TagsApi.get(FeatureGroupBase featureGroupBase,
String name) |
List<FeatureViewBase> |
FeatureViewApi.get(FeatureStoreBase featureStoreBase,
String name) |
List<TrainingDatasetBase> |
TrainingDatasetApi.get(FeatureStoreBase featureStoreBase,
String tdName,
Integer tdVersion) |
<T extends FeatureViewBase> |
FeatureViewApi.get(FeatureStoreBase featureStoreBase,
String name,
Integer version,
Class<T> fvType) |
Map<String,Object> |
TagsApi.get(FeatureViewBase featureViewBase) |
Map<String,Object> |
TagsApi.get(FeatureViewBase featureViewBase,
Integer trainingDatasetVersion) |
Object |
TagsApi.get(FeatureViewBase featureViewBase,
Integer trainingDatasetVersion,
String name) |
Object |
TagsApi.get(FeatureViewBase featureViewBase,
String name) |
<U> U |
StorageConnectorApi.get(Integer featureStoreId,
String name,
Class<U> storageConnectorType) |
Optional<Variable> |
VariablesApi.get(String variableName) |
Project |
ProjectApi.get(String name) |
<T> T |
FeatureStoreApi.get(String name,
Class<T> fsType) |
Map<String,Object> |
TagsApi.get(TrainingDatasetBase trainingDatasetBase) |
Statistics |
StatisticsApi.get(TrainingDatasetBase trainingDatasetBase,
String commitTime) |
Object |
TagsApi.get(TrainingDatasetBase trainingDatasetBase,
String name) |
protected String |
HopsworksExternalClient.getAssumedRole() |
<T extends QueryBase> |
FeatureViewApi.getBatchQuery(FeatureStoreBase featureStoreBase,
String name,
Integer version,
Long startTime,
Long endTime,
Boolean withLabels,
Integer trainingDataVersion,
Class<T> queryType) |
<T> T |
StorageConnectorApi.getByName(FeatureStoreBase featureStoreBase,
String name,
Class<T> storageConnectorType) |
List<FeatureGroupCommit> |
FeatureGroupApi.getCommitDetails(FeatureGroupBase featureGroupBase,
Long wallclockTimestamp,
Integer limit) |
Credentials |
HopsworksClient.getCredentials() |
static HopsworksClient |
HopsworksClient.getInstance() |
<T extends FeatureGroupBase> |
FeatureGroupApi.getInternal(FeatureStoreBase featureStoreBase,
String fgName,
Integer fgVersion,
Class<T[]> fgType) |
StorageConnector.KafkaConnector |
StorageConnectorApi.getKafkaStorageConnector(FeatureStoreBase featureStoreBase,
boolean external) |
Statistics |
StatisticsApi.getLast(FeatureGroupBase featureGroup) |
Statistics |
StatisticsApi.getLast(TrainingDatasetBase trainingDatasetBase) |
<T> T |
StorageConnectorApi.getOnlineStorageConnector(FeatureStoreBase featureStoreBase,
Class<T> storageConnectorType) |
FsQueryBase |
TrainingDatasetApi.getQuery(TrainingDatasetBase trainingDatasetBase,
boolean withLabel,
boolean isHiveQuery) |
List<ServingPreparedStatement> |
FeatureViewApi.getServingPreparedStatement(FeatureViewBase featureViewBase,
boolean batch) |
List<ServingPreparedStatement> |
TrainingDatasetApi.getServingPreparedStatement(TrainingDatasetBase trainingDatasetBase,
boolean batch) |
Subject |
KafkaApi.getSubject(FeatureStoreBase featureStoreBase,
String subjectName) |
Subject |
KafkaApi.getSubject(FeatureStoreBase featureStoreBase,
String subjectName,
String subjectVersion) |
<T extends TrainingDatasetBase> |
FeatureViewApi.getTrainingData(FeatureStoreBase featureStoreBase,
String featureViewName,
Integer featureViewVersion,
Integer trainingDataVersion,
Class<T> tdType) |
TrainingDatasetBase |
TrainingDatasetApi.getTrainingDataset(FeatureStoreBase featureStoreBase,
String tdName,
Integer tdVersion) |
List<TransformationFunctionAttached> |
FeatureViewApi.getTransformationFunctions(FeatureViewBase featureViewBase) |
List<TransformationFunctionAttached> |
TrainingDatasetApi.getTransformationFunctions(TrainingDatasetBase trainingDatasetBase) |
<T> T |
HopsworksClient.handleRequest(org.apache.http.HttpRequest request) |
<T> T |
HopsworksClient.handleRequest(org.apache.http.HttpRequest request,
Class<T> cls) |
<T> T |
HopsworksInternalClient.handleRequest(org.apache.http.HttpRequest request,
org.apache.http.client.ResponseHandler<T> responseHandler) |
<T> T |
HopsworksClient.handleRequest(org.apache.http.HttpRequest request,
org.apache.http.client.ResponseHandler<T> responseHandler) |
<T> T |
HopsworksHttpClient.handleRequest(org.apache.http.HttpRequest request,
org.apache.http.client.ResponseHandler<T> responseHandler) |
void |
CodeApi.post(FeatureGroupBase featureGroup,
Code code,
String entityId,
Code.RunType type,
String browserHostName) |
Statistics |
StatisticsApi.post(FeatureGroupBase featureGroup,
Statistics statistics) |
Statistics |
StatisticsApi.post(FeatureViewBase featureViewBase,
Integer trainingDataVersion,
Statistics statistics) |
void |
CodeApi.post(TrainingDatasetBase trainingDatasetBase,
Code code,
String entityId,
Code.RunType type,
String browserHostName) |
Statistics |
StatisticsApi.post(TrainingDatasetBase trainingDatasetBase,
Statistics statistics) |
String |
HopsworksExternalClient.readApiKey(SecretStore secretStore,
software.amazon.awssdk.regions.Region region,
String apiKeyFilepath)
Read API key.
|
protected String |
HopsworksExternalClient.readApiKeyParamStore(software.amazon.awssdk.regions.Region region,
String secretKey) |
protected String |
HopsworksExternalClient.readApiKeySecretManager(software.amazon.awssdk.regions.Region region,
String secretKey) |
static byte[] |
DatasetApi.readContent(String path,
String datasetType) |
void |
HopsworksInternalClient.refreshJwt() |
<U extends FeatureGroupBase> |
FeatureGroupApi.save(FeatureGroupBase featureGroup,
Class<U> fgType) |
<T extends FeatureViewBase> |
FeatureViewApi.save(FeatureViewBase featureViewBase,
Class<T> fvType) |
<U extends FeatureGroupBase> |
FeatureGroupApi.saveFeatureGroupMetaData(FeatureGroupBase featureGroup,
List<String> partitionKeys,
String hudiPrecombineKey,
Map<String,String> writeOptions,
JobConfiguration jobConfiguration,
Class<U> fgType) |
<U extends FeatureGroupBase> |
FeatureGroupApi.saveInternal(FeatureGroupBase featureGroupBase,
org.apache.http.entity.StringEntity entity,
Class<U> fgType) |
static HopsworksClient |
HopsworksClient.setupHopsworksClient(String host,
int port,
software.amazon.awssdk.regions.Region region,
SecretStore secretStore,
boolean hostnameVerification,
String trustStorePath,
String apiKeyFilePath,
String apiKeyValue) |
<T> T |
FeatureViewApi.update(FeatureViewBase featureViewBase,
Class<T> fvType) |
<T extends FeatureGroupBase> |
FeatureGroupApi.updateMetadata(FeatureGroupBase featureGroup,
String queryParameter,
Class<T> fgType) |
<T extends FeatureGroupBase> |
FeatureGroupApi.updateMetadata(FeatureGroupBase featureGroup,
String queryParameter,
Object value,
Class<T> fgType) |
TrainingDatasetBase |
TrainingDatasetApi.updateMetadata(TrainingDatasetBase trainingDatasetBase,
String queryParameter) |
| Constructor and Description |
|---|
HopsworksInternalClient() |
| Modifier and Type | Method and Description |
|---|---|
void |
TrainingDataset.addTag(String name,
Object value)
Add name/value tag to the training dataset.
|
void |
FeatureView.addTag(String name,
Object value)
Add name/value tag to the feature view.
|
void |
FeatureView.addTrainingDatasetTag(Integer version,
String name,
Object value)
Add name/value tag to the training dataset.
|
void |
ExternalFeatureGroup.appendFeatures(Feature features) |
void |
StreamFeatureGroup.appendFeatures(Feature features)
Append a single feature to the schema of the stream feature group.
|
void |
FeatureGroup.appendFeatures(Feature features)
Append a single feature to the schema of the feature group.
|
void |
ExternalFeatureGroup.appendFeatures(List<Feature> features) |
void |
StreamFeatureGroup.appendFeatures(List<Feature> features)
Append features to the schema of the stream feature group.
|
void |
FeatureGroup.appendFeatures(List<Feature> features)
Append features to the schema of the feature group.
|
Query |
StreamFeatureGroup.asOf(String wallclockTime)
Get Query object to retrieve all features of the group at a point in the past.
|
Query |
FeatureGroup.asOf(String wallclockTime)
Get Query object to retrieve all features of the group at a point in the past.
|
Query |
StreamFeatureGroup.asOf(String wallclockTime,
String excludeUntil)
Get Query object to retrieve all features of the group at a point in the past.
|
Query |
FeatureGroup.asOf(String wallclockTime,
String excludeUntil)
Get Query object to retrieve all features of the group at a point in the past.
|
FeatureView |
FeatureView.FeatureViewBuilder.build() |
void |
FeatureView.clean(FeatureStore featureStore,
String featureViewName,
Integer featureViewVersion)
Delete the feature view and all associated metadata and training data.
|
void |
StreamFeatureGroup.commitDeleteRecord(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData)
Drops records present in the provided DataFrame and commits it as update to this Stream Feature group.
|
void |
FeatureGroup.commitDeleteRecord(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData)
Drops records present in the provided DataFrame and commits it as update to this Feature group.
|
void |
StreamFeatureGroup.commitDeleteRecord(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Map<String,String> writeOptions)
Drops records present in the provided DataFrame and commits it as update to this Stream Feature group.
|
void |
FeatureGroup.commitDeleteRecord(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Map<String,String> writeOptions)
Drops records present in the provided DataFrame and commits it as update to this Feature group.
|
Map<Long,Map<String,String>> |
StreamFeatureGroup.commitDetails()
Retrieves commit timeline for this stream feature group.
|
Map<Long,Map<String,String>> |
FeatureGroup.commitDetails()
Retrieves commit timeline for this feature group.
|
Map<Long,Map<String,String>> |
StreamFeatureGroup.commitDetails(Integer limit)
/**
Retrieves commit timeline for this stream feature group.
|
Map<Long,Map<String,String>> |
FeatureGroup.commitDetails(Integer limit)
Retrieves commit timeline for this feature group.
|
Map<Long,Map<String,String>> |
StreamFeatureGroup.commitDetails(String wallclockTime)
Return commit details as of specific point in time.
|
Map<Long,Map<String,String>> |
FeatureGroup.commitDetails(String wallclockTime)
Return commit details as of specific point in time.
|
Map<Long,Map<String,String>> |
StreamFeatureGroup.commitDetails(String wallclockTime,
Integer limit)
Return commit details as of specific point in time.
|
Map<Long,Map<String,String>> |
FeatureGroup.commitDetails(String wallclockTime,
Integer limit)
Return commit details as of specific point in time.
|
Statistics |
ExternalFeatureGroup.computeStatistics() |
Statistics |
TrainingDataset.computeStatistics()
Recompute the statistics for the entire training dataset and save them to the feature store.
|
Statistics |
StreamFeatureGroup.computeStatistics()
Recompute the statistics for the stream feature group and save them to the feature store.
|
Statistics |
FeatureGroup.computeStatistics()
Recompute the statistics for the feature group and save them to the feature store.
|
Statistics |
StreamFeatureGroup.computeStatistics(String wallclockTime)
Recompute the statistics for the feature group and save them to the feature store.
|
Statistics |
FeatureGroup.computeStatistics(String wallclockTime)
Recompute the statistics for the feature group and save them to the feature store.
|
Integer |
FeatureView.createTrainingData(String startTime,
String endTime,
String description,
DataFormat dataFormat)
Create the metadata for a training dataset and save the corresponding training data into `location`.
|
Integer |
FeatureView.createTrainingData(String startTime,
String endTime,
String description,
DataFormat dataFormat,
Boolean coalesce,
StorageConnector storageConnector,
String location,
Long seed,
StatisticsConfig statisticsConfig,
Map<String,String> writeOptions,
FilterLogic extraFilterLogic,
Filter extraFilter)
Create the metadata for a training dataset and save the corresponding training data into `location`.
|
Integer |
FeatureView.createTrainTestSplit(Float testSize,
String trainStart,
String trainEnd,
String testStart,
String testEnd,
String description,
DataFormat dataFormat)
Create the metadata for a training dataset and save the corresponding training data into `location`.
|
Integer |
FeatureView.createTrainTestSplit(Float testSize,
String trainStart,
String trainEnd,
String testStart,
String testEnd,
String description,
DataFormat dataFormat,
Boolean coalesce,
StorageConnector storageConnector,
String location,
Long seed,
StatisticsConfig statisticsConfig,
Map<String,String> writeOptions,
FilterLogic extraFilterLogic,
Filter extraFilter)
Create the metadata for a training dataset and save the corresponding training data into `location`.
|
Integer |
FeatureView.createTrainValidationTestSplit(Float validationSize,
Float testSize,
String trainStart,
String trainEnd,
String validationStart,
String validationEnd,
String testStart,
String testEnd,
String description,
DataFormat dataFormat)
Create the metadata for a training dataset and save the corresponding training data into `location`.
|
Integer |
FeatureView.createTrainValidationTestSplit(Float validationSize,
Float testSize,
String trainStart,
String trainEnd,
String validationStart,
String validationEnd,
String testStart,
String testEnd,
String description,
DataFormat dataFormat,
Boolean coalesce,
StorageConnector storageConnector,
String location,
Long seed,
StatisticsConfig statisticsConfig,
Map<String,String> writeOptions,
FilterLogic extraFilterLogic,
Filter extraFilter)
Create the metadata for a training dataset and save the corresponding training data into `location`.
|
void |
TrainingDataset.delete()
Delete training dataset and all associated metadata.
|
void |
FeatureView.delete()
Delete current feature view, all associated metadata and training data.
|
void |
FeatureView.deleteAllTrainingDatasets()
Delete all training datasets.
|
void |
TrainingDataset.deleteTag(String name)
Delete a tag of the training dataset.
|
void |
FeatureView.deleteTag(String name)
Delete a tag of the feature view.
|
void |
FeatureView.deleteTrainingDataset(Integer version)
Delete a training dataset.
|
void |
FeatureView.deleteTrainingDatasetTag(Integer version,
String name)
Delete a tag of the training dataset.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
FeatureView.getBatchData()
Get all data from the feature view as a batch from the offline feature store.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
FeatureView.getBatchData(String startTime,
String endTime)
Get a batch of data from an event time interval from the offline feature store.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
FeatureView.getBatchData(String startTime,
String endTime,
Map<String,String> readOptions)
Get a batch of data from an event time interval from the offline feature store.
|
String |
FeatureView.getBatchQuery()
Get a query string of the batch query.
|
String |
FeatureView.getBatchQuery(String startTime,
String endTime)
Get a query string of the batch query.
|
ExternalFeatureGroup |
FeatureStore.getExternalFeatureGroup(String name)
Get an external feature group object with default version `1` from the feature store.
|
ExternalFeatureGroup |
FeatureStore.getExternalFeatureGroup(@NonNull String name,
@NonNull Integer version)
Get an external feature group object from the feature store.
|
scala.collection.Seq<ExternalFeatureGroup> |
FeatureStore.getExternalFeatureGroups(@NonNull String name)
Get a list of all versions of an external feature group from the feature store.
|
FeatureGroup |
FeatureStore.getFeatureGroup(String name)
Get a feature group object with default version `1` from the feature store.
|
FeatureGroup |
FeatureStore.getFeatureGroup(@NonNull String name,
@NonNull Integer version)
Get a feature group object from the feature store.
|
scala.collection.Seq<FeatureGroup> |
FeatureStore.getFeatureGroups(@NonNull String name)
Get a list of all versions of a feature group from the feature store.
|
FeatureStore |
HopsworksConnection.getFeatureStore()
Retrieve the project feature store.
|
FeatureStore |
HopsworksConnection.getFeatureStore(String name)
Retrieve a feature store based on name.
|
FeatureView |
FeatureStore.getFeatureView(String name)
Get a feature view object with the default version `1` from the selected feature store.
|
FeatureView |
FeatureStore.getFeatureView(@NonNull String name,
@NonNull Integer version)
Get a feature view object from the selected feature store.
|
ExternalFeatureGroup |
FeatureStore.getOnDemandFeatureGroup(String name)
Deprecated.
|
ExternalFeatureGroup |
FeatureStore.getOnDemandFeatureGroup(@NonNull String name,
@NonNull Integer version)
Deprecated.
|
scala.collection.Seq<ExternalFeatureGroup> |
FeatureStore.getOnDemandFeatureGroups(@NonNull String name)
Deprecated.
|
FeatureGroup |
FeatureStore.getOrCreateFeatureGroup(String name,
Integer version)
Get feature group metadata object or create a new one if it doesn't exist.
|
FeatureGroup |
FeatureStore.getOrCreateFeatureGroup(String name,
Integer version,
List<String> primaryKeys,
boolean onlineEnabled,
String eventTime)
Get feature group metadata object or create a new one if it doesn't exist.
|
FeatureGroup |
FeatureStore.getOrCreateFeatureGroup(String name,
Integer version,
List<String> primaryKeys,
List<String> partitionKeys,
boolean onlineEnabled,
String eventTime)
Get feature group metadata object or create a new one if it doesn't exist.
|
FeatureGroup |
FeatureStore.getOrCreateFeatureGroup(String name,
Integer version,
String description,
List<String> primaryKeys,
List<String> partitionKeys,
String hudiPrecombineKey,
boolean onlineEnabled,
TimeTravelFormat timeTravelFormat,
StatisticsConfig statisticsConfig,
String topicName,
String notificationTopicName,
String eventTime)
Get feature group metadata object or create a new one if it doesn't exist.
|
FeatureView |
FeatureStore.getOrCreateFeatureView(String name,
Query query,
Integer version)
Get feature view metadata object or create a new one if it doesn't exist.
|
FeatureView |
FeatureStore.getOrCreateFeatureView(String name,
Query query,
Integer version,
String description,
List<String> labels)
Get feature view metadata object or create a new one if it doesn't exist.
|
StreamFeatureGroup |
FeatureStore.getOrCreateStreamFeatureGroup(String name,
Integer version)
Get stream feature group metadata object or create a new one if it doesn't exist.
|
StreamFeatureGroup |
FeatureStore.getOrCreateStreamFeatureGroup(String name,
Integer version,
List<String> primaryKeys,
boolean onlineEnabled,
String eventTime)
Get stream feature group metadata object or create a new one if it doesn't exist.
|
StreamFeatureGroup |
FeatureStore.getOrCreateStreamFeatureGroup(String name,
Integer version,
List<String> primaryKeys,
List<String> partitionKeys,
boolean onlineEnabled,
String eventTime)
Get stream feature group metadata object or create a new one if it doesn't exist.
|
StreamFeatureGroup |
FeatureStore.getOrCreateStreamFeatureGroup(@NonNull String name,
Integer version,
String description,
Boolean onlineEnabled,
TimeTravelFormat timeTravelFormat,
List<String> primaryKeys,
List<String> partitionKeys,
String eventTime,
String hudiPrecombineKey,
List<Feature> features,
StatisticsConfig statisticsConfig,
StorageConnector storageConnector,
String path) |
StreamFeatureGroup |
FeatureStore.getOrCreateStreamFeatureGroup(String name,
Integer version,
String description,
List<String> primaryKeys,
List<String> partitionKeys,
String hudiPrecombineKey,
boolean onlineEnabled,
StatisticsConfig statisticsConfig,
String eventTime,
TimeTravelFormat timeTravelFormat)
Get stream feature group metadata object or create a new one if it doesn't exist.
|
HashSet<String> |
FeatureView.getPrimaryKeys()
Get set of primary key names that is used as keys in input dict object for `getServingVector` method.
|
String |
TrainingDataset.getQuery(Storage storage,
boolean withLabel) |
Statistics |
ExternalFeatureGroup.getStatistics() |
Statistics |
TrainingDataset.getStatistics()
Get the last statistics commit for the training dataset.
|
Statistics |
StreamFeatureGroup.getStatistics() |
Statistics |
FeatureGroup.getStatistics()
Get the last statistics commit for the feature group.
|
Statistics |
TrainingDataset.getStatistics(String commitTime)
Get the statistics of a specific commit time for the training dataset.
|
StreamFeatureGroup |
FeatureStore.getStreamFeatureGroup(String name)
Get a stream feature group object with default version `1` from the feature store.
|
StreamFeatureGroup |
FeatureStore.getStreamFeatureGroup(@NonNull String name,
@NonNull Integer version)
Get a stream feature group object from the feature store.
|
Object |
TrainingDataset.getTag(String name)
Get a single tag value of the training dataset.
|
Object |
FeatureView.getTag(String name)
Get a single tag value of the feature view.
|
Map<String,Object> |
TrainingDataset.getTags()
Get all tags of the training dataset.
|
Map<String,Object> |
FeatureView.getTags()
Get all tags of the feature view.
|
List<org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>> |
FeatureView.getTrainingData(Integer version)
Get training data created by `featureView.createTrainingData` or `featureView.trainingData`.
|
List<org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>> |
FeatureView.getTrainingData(Integer version,
Map<String,String> readOptions)
Get training data created by `featureView.createTrainingData` or `featureView.trainingData`.
|
TrainingDataset |
FeatureStore.getTrainingDataset(String name)
Deprecated.
|
TrainingDataset |
FeatureStore.getTrainingDataset(@NonNull String name,
@NonNull Integer version)
Deprecated.
|
scala.collection.Seq<TrainingDataset> |
FeatureStore.getTrainingDatasets(@NonNull String name)
Deprecated.
|
Object |
FeatureView.getTrainingDatasetTag(Integer version,
String name)
Get a single tag value of the training dataset.
|
Map<String,Object> |
FeatureView.getTrainingDatasetTags(Integer version)
Get all tags of the training dataset.
|
List<org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>> |
FeatureView.getTrainTestSplit(Integer version)
Get training data created by `featureView.createTrainTestSplit` or `featureView.trainTestSplit`.
|
List<org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>> |
FeatureView.getTrainTestSplit(Integer version,
Map<String,String> readOptions)
Get training data created by `featureView.createTrainTestSplit` or `featureView.trainTestSplit`.
|
List<org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>> |
FeatureView.getTrainValidationTestSplit(Integer version)
Get training data created by `featureView.createTrainValidationTestSplit` or featureView.trainValidationTestSplit`.
|
List<org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>> |
FeatureView.getTrainValidationTestSplit(Integer version,
Map<String,String> readOptions)
Get training data created by `featureView.createTrainValidationTestSplit` or featureView.trainValidationTestSplit`.
|
void |
ExternalFeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData)
Incrementally insert data to the online storage of an external feature group.
|
void |
StreamFeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData)
Incrementally insert data to a stream feature group or overwrite all data contained in the feature group.
|
void |
FeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData)
Incrementally insert data to a feature group or overwrite all data contained in the feature group.
|
void |
StreamFeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
boolean overwrite)
Incrementally insert data to a stream feature group or overwrite all data contained in the feature group.
|
void |
FeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
boolean overwrite)
Incrementally insert data to a feature group or overwrite all data contained in the feature group.
|
void |
StreamFeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
boolean overwrite,
Map<String,String> writeOptions)
Incrementally insert data to a stream feature group or overwrite all data contained in the feature group.
|
void |
FeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
boolean overwrite,
Map<String,String> writeOptions)
Incrementally insert data to a feature group or overwrite all data contained in the feature group.
|
void |
StreamFeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
boolean overwrite,
Map<String,String> writeOptions,
JobConfiguration jobConfiguration)
Incrementally insert data to a stream feature group or overwrite all data contained in the feature group.
|
void |
FeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
boolean overwrite,
Map<String,String> writeOptions,
JobConfiguration jobConfiguration) |
void |
StreamFeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
HudiOperationType operation) |
void |
FeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
HudiOperationType operation)
Incrementally insert data to a feature group or overwrite all data contained in the feature group.
|
void |
StreamFeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
JobConfiguration jobConfiguration)
Incrementally insert data to a stream feature group or overwrite all data contained in the feature group.
|
void |
FeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
JobConfiguration jobConfiguration) |
void |
ExternalFeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Map<String,String> writeOptions)
Incrementally insert data to the online storage of an external feature group.
|
void |
StreamFeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Map<String,String> writeOptions)
Incrementally insert data to a stream feature group or overwrite all data contained in the feature group.
|
void |
FeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Map<String,String> writeOptions)
Incrementally insert data to a feature group or overwrite all data contained in the feature group.
|
void |
ExternalFeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Storage storage) |
void |
StreamFeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Storage storage) |
void |
FeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Storage storage)
Incrementally insert data to a feature group or overwrite all data contained in the feature group.
|
void |
StreamFeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Storage storage,
boolean overwrite) |
void |
FeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Storage storage,
boolean overwrite)
Incrementally insert data to a feature group or overwrite all data contained in the feature group.
|
void |
StreamFeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Storage storage,
boolean overwrite,
HudiOperationType operation,
Map<String,String> writeOptions) |
void |
FeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Storage storage,
boolean overwrite,
HudiOperationType operation,
Map<String,String> writeOptions)
Incrementally insert data to a feature group or overwrite all data contained in the feature group.
|
org.apache.spark.sql.streaming.StreamingQuery |
FeatureGroup.insertStream(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData)
Deprecated.
insertStream method is deprecated FeatureGroups. Full capability insertStream is available for StreamFeatureGroups.
|
org.apache.spark.sql.streaming.StreamingQuery |
FeatureGroup.insertStream(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Map<String,String> writeOptions)
Deprecated.
|
org.apache.spark.sql.streaming.StreamingQuery |
FeatureGroup.insertStream(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
String queryName)
Deprecated.
|
org.apache.spark.sql.streaming.StreamingQuery |
FeatureGroup.insertStream(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
String queryName,
Map<String,String> writeOptions) |
org.apache.spark.sql.streaming.StreamingQuery |
FeatureGroup.insertStream(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
String queryName,
String outputMode)
Deprecated.
|
org.apache.spark.sql.streaming.StreamingQuery |
FeatureGroup.insertStream(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
String queryName,
String outputMode,
boolean awaitTermination,
Long timeout)
Deprecated.
insertStream method is deprecated FeatureGroups. Full capability insertStream is available for StreamFeatureGroups.
|
org.apache.spark.sql.streaming.StreamingQuery |
FeatureGroup.insertStream(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
String queryName,
String outputMode,
boolean awaitTermination,
Long timeout,
String checkpointLocation)
Deprecated.
|
org.apache.spark.sql.streaming.StreamingQuery |
FeatureGroup.insertStream(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
String queryName,
String outputMode,
boolean awaitTermination,
Long timeout,
String checkpointLocation,
Map<String,String> writeOptions)
Deprecated.
|
Object |
FeatureGroup.insertStream(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
String queryName,
String outputMode,
boolean awaitTermination,
Long timeout,
String checkpointLocation,
Map<String,String> writeOptions,
JobConfiguration jobConfiguration) |
org.apache.spark.sql.streaming.StreamingQuery |
FeatureGroup.insertStream(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
String queryName,
String outputMode,
boolean awaitTermination,
String checkpointLocation)
Deprecated.
insertStream method is deprecated FeatureGroups. Full capability insertStream is available for StreamFeatureGroups.
|
org.apache.spark.sql.streaming.StreamingQuery |
FeatureGroup.insertStream(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
String queryName,
String outputMode,
String checkpointLocation)
Deprecated.
|
void |
FeatureView.purgeAllTrainingData()
Delete all training datasets in this feature view (data only).
|
void |
FeatureView.purgeTrainingData(Integer version)
Delete a training dataset (data only).
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
ExternalFeatureGroup.read() |
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
TrainingDataset.read()
Read the content of the training dataset.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
StreamFeatureGroup.read()
Reads the feature group from the offline storage as Spark DataFrame.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
FeatureGroup.read()
Reads the feature group from the offline storage as Spark DataFrame.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
ExternalFeatureGroup.read(boolean online) |
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
StreamFeatureGroup.read(boolean online)
Reads the stream feature group from the offline or online storage as Spark DataFrame.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
FeatureGroup.read(boolean online)
Reads the feature group from the offline or online storage as Spark DataFrame.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
ExternalFeatureGroup.read(boolean online,
Map<String,String> readOptions) |
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
StreamFeatureGroup.read(boolean online,
Map<String,String> readOptions)
Reads the stream feature group from the offline or online storage as Spark DataFrame.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
FeatureGroup.read(boolean online,
Map<String,String> readOptions)
Reads the feature group from the offline or online storage as Spark DataFrame.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
ExternalFeatureGroup.read(Map<String,String> readOptions) |
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
TrainingDataset.read(Map<String,String> readOptions)
Read the content of the training dataset.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
StreamFeatureGroup.read(Map<String,String> readOptions)
Reads the stream feature group from the offline storage as Spark DataFrame.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
FeatureGroup.read(Map<String,String> readOptions)
Reads the feature group from the offline storage as Spark DataFrame.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
TrainingDataset.read(String split)
Read all a single split from the training dataset.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
StreamFeatureGroup.read(String wallclockTime)
Reads stream Feature group into a dataframe at a specific point in time.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
FeatureGroup.read(String wallclockTime)
Reads Feature group into a dataframe at a specific point in time.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
TrainingDataset.read(String split,
Map<String,String> readOptions)
Read a single split from the training dataset.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
StreamFeatureGroup.read(String wallclockTime,
Map<String,String> readOptions)
Reads stream Feature group into a dataframe at a specific point in time.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
FeatureGroup.read(String wallclockTime,
Map<String,String> readOptions)
Reads Feature group into a dataframe at a specific point in time.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
StreamFeatureGroup.readChanges(String wallclockStartTime,
String wallclockEndTime)
Deprecated.
`readChanges` method is deprecated. Use `asOf(wallclockEndTime, wallclockStartTime).read()` instead.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
FeatureGroup.readChanges(String wallclockStartTime,
String wallclockEndTime)
Deprecated.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
StreamFeatureGroup.readChanges(String wallclockStartTime,
String wallclockEndTime,
Map<String,String> readOptions)
Deprecated.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
FeatureGroup.readChanges(String wallclockStartTime,
String wallclockEndTime,
Map<String,String> readOptions)
Deprecated.
|
void |
FeatureView.recreateTrainingDataset(Integer version,
Map<String,String> writeOptions)
Recreate a training dataset.
|
void |
ExternalFeatureGroup.save() |
void |
FeatureGroup.save(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData)
Deprecated.
|
void |
StreamFeatureGroup.save(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Map<String,String> writeOptions)
Deprecated.
|
void |
FeatureGroup.save(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Map<String,String> writeOptions)
Deprecated.
|
void |
StreamFeatureGroup.save(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Map<String,String> writeOptions,
JobConfiguration jobConfiguration)
Deprecated.
|
void |
TrainingDataset.save(Query query) |
void |
TrainingDataset.save(Query query,
Map<String,String> writeOptions)
Create the training dataset based on the content of the feature store query.
|
void |
ExternalFeatureGroup.show(int numRows) |
void |
TrainingDataset.show(int numRows)
Show numRows from the training dataset (across all splits).
|
void |
StreamFeatureGroup.show(int numRows)
Show the first `n` rows of the feature group.
|
void |
FeatureGroup.show(int numRows)
Show the first `n` rows of the feature group.
|
void |
ExternalFeatureGroup.show(int numRows,
boolean online) |
void |
StreamFeatureGroup.show(int numRows,
boolean online)
Show the first `n` rows of the feature group.
|
void |
FeatureGroup.show(int numRows,
boolean online)
Show the first `n` rows of the feature group.
|
List<org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>> |
FeatureView.trainingData(String startTime,
String endTime,
String description)
Create the metadata for a training dataset and get the corresponding training data from the offline feature store.
|
List<org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>> |
FeatureView.trainingData(String startTime,
String endTime,
String description,
Long seed,
StatisticsConfig statisticsConfig,
Map<String,String> readOptions,
FilterLogic extraFilterLogic,
Filter extraFilter)
Create the metadata for a training dataset and get the corresponding training data from the offline feature store.
|
List<org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>> |
FeatureView.trainTestSplit(Float testSize,
String trainStart,
String trainEnd,
String testStart,
String testEnd,
String description)
Create the metadata for a training dataset and get the corresponding training data from the offline feature store.
|
List<org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>> |
FeatureView.trainTestSplit(Float testSize,
String trainStart,
String trainEnd,
String testStart,
String testEnd,
String description,
Long seed,
StatisticsConfig statisticsConfig,
Map<String,String> readOptions,
FilterLogic extraFilterLogic,
Filter extraFilter)
Create the metadata for a training dataset and get the corresponding training data from the offline feature store.
|
List<org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>> |
FeatureView.trainValidationTestSplit(Float validationSize,
Float testSize,
String trainStart,
String trainEnd,
String validationStart,
String validationEnd,
String testStart,
String testEnd,
String description)
Create the metadata for a training dataset and get the corresponding training data from the offline feature store.
|
List<org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>> |
FeatureView.trainValidationTestSplit(Float validationSize,
Float testSize,
String trainStart,
String trainEnd,
String validationStart,
String validationEnd,
String testStart,
String testEnd,
String description,
Long seed,
StatisticsConfig statisticsConfig,
Map<String,String> readOptions,
FilterLogic extraFilterLogic,
Filter extraFilter)
Create the metadata for a training dataset and get the corresponding training data from the offline feature store.
|
FeatureView |
FeatureView.update(FeatureView other)
Update the description of the feature view.
|
void |
ExternalFeatureGroup.updateFeatures(Feature feature) |
void |
StreamFeatureGroup.updateFeatures(Feature feature)
Update the metadata of feature.
|
void |
FeatureGroup.updateFeatures(Feature feature)
Update the metadata of feature.
|
void |
ExternalFeatureGroup.updateFeatures(List<Feature> features) |
void |
StreamFeatureGroup.updateFeatures(List<Feature> features)
Update the metadata of multiple features.
|
void |
FeatureGroup.updateFeatures(List<Feature> features)
Update the metadata of multiple features.
|
void |
TrainingDataset.updateStatisticsConfig()
Update the statistics configuration of the training dataset.
|
| Constructor and Description |
|---|
HopsworksConnection(String host,
int port,
String project,
software.amazon.awssdk.regions.Region region,
SecretStore secretStore,
boolean hostnameVerification,
String trustStorePath,
String certPath,
String apiKeyFilePath,
String apiKeyValue) |
TrainingDataset(@NonNull String name,
Integer version,
String description,
DataFormat dataFormat,
Boolean coalesce,
StorageConnector storageConnector,
String location,
List<Split> splits,
String trainSplit,
Long seed,
FeatureStore featureStore,
StatisticsConfig statisticsConfig,
List<String> label,
String eventStartTime,
String eventEndTime,
TrainingDatasetType trainingDatasetType,
Float validationSize,
Float testSize,
String trainStart,
String trainEnd,
String validationStart,
String validationEnd,
String testStart,
String testEnd,
Integer timeSplitSize,
FilterLogic extraFilterLogic,
Filter extraFilter) |
| Modifier and Type | Method and Description |
|---|---|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
Query.read() |
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
Query.read(boolean online) |
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
Query.read(boolean online,
Map<String,String> readOptions) |
void |
FsQuery.registerHudiFeatureGroups(Map<String,String> readOptions) |
void |
FsQuery.registerOnDemandFeatureGroups() |
void |
Query.show(boolean online,
int numRows) |
void |
Query.show(int numRows) |
| Modifier and Type | Method and Description |
|---|---|
String |
SparkEngine.addFile(String filePath) |
void |
TrainingDatasetEngine.addTag(TrainingDataset trainingDataset,
String name,
Object value) |
<T extends FeatureGroupBase> |
FeatureGroupEngine.appendFeatures(FeatureGroupBase featureGroup,
List<Feature> features,
Class<T> fgClass) |
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
SparkEngine.castColumnType(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset,
List<TrainingDatasetFeature> features) |
FeatureGroupCommit |
FeatureGroupEngine.commitDelete(FeatureGroupBase featureGroupBase,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> genericDataset,
Map<String,String> writeOptions) |
Map<Long,Map<String,String>> |
FeatureGroupEngine.commitDetails(FeatureGroupBase featureGroupBase,
Integer limit) |
Map<Long,Map<String,String>> |
FeatureGroupEngine.commitDetailsByWallclockTime(FeatureGroupBase featureGroup,
String wallclockTime,
Integer limit) |
Statistics |
StatisticsEngine.computeAndSaveSplitStatistics(FeatureView featureView,
TrainingDataset trainingDataset,
Map<String,org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>> splitDatasets) |
Statistics |
StatisticsEngine.computeAndSaveSplitStatistics(TrainingDataset trainingDataset) |
Statistics |
StatisticsEngine.computeSplitStatistics(TrainingDataset trainingDataset) |
Statistics |
StatisticsEngine.computeStatistics(FeatureGroupBase featureGroup,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataFrame,
Long commitId) |
Statistics |
StatisticsEngine.computeStatistics(FeatureView featureView,
TrainingDataset trainingDataset,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataFrame) |
Statistics |
FeatureViewEngine.computeStatistics(FeatureView featureView,
TrainingDataset trainingDataset,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>[] datasets) |
Statistics |
StatisticsEngine.computeStatistics(TrainingDataset trainingDataset,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataFrame) |
String |
SparkEngine.constructCheckpointPath(FeatureGroupBase featureGroup,
String queryName,
String queryPrefix) |
TrainingDatasetBundle |
FeatureViewEngine.createTrainingDataset(FeatureView featureView,
TrainingDataset trainingDataset,
Map<String,String> userWriteOptions) |
void |
TrainingDatasetEngine.delete(TrainingDataset trainingDataset) |
void |
TrainingDatasetEngine.deleteTag(TrainingDataset trainingDataset,
String name) |
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
SparkEngine.encodeComplexFeatures(FeatureGroupBase featureGroupBase,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset)
Encodes all complex type features to binary using their avro type as schema.
|
Statistics |
StatisticsEngine.get(FeatureGroupBase featureGroup,
String commitTime) |
FeatureView |
FeatureViewEngine.get(FeatureStore featureStore,
String name,
Integer version) |
Statistics |
StatisticsEngine.get(TrainingDataset trainingDataset,
String commitTime) |
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
FeatureViewEngine.getBatchData(FeatureView featureView,
Date startTime,
Date endTime,
Map<String,String> readOptions,
Integer trainingDataVersion) |
Query |
FeatureViewEngine.getBatchQuery(FeatureView featureView,
Date startTime,
Date endTime,
Boolean withLabels,
Integer trainingDataVersion) |
String |
FeatureViewEngine.getBatchQueryString(FeatureView featureView,
Date startTime,
Date endTime,
Integer trainingDataVersion) |
ExternalFeatureGroup |
FeatureGroupEngine.getExternalFeatureGroup(FeatureStore featureStore,
String fgName,
Integer fgVersion) |
List<ExternalFeatureGroup> |
FeatureGroupEngine.getExternalFeatureGroups(FeatureStore featureStore,
String fgName) |
FeatureGroup |
FeatureGroupEngine.getFeatureGroup(FeatureStore featureStore,
String fgName,
Integer fgVersion) |
List<FeatureGroup> |
FeatureGroupEngine.getFeatureGroups(FeatureStore featureStore,
String fgName) |
Map<String,String> |
SparkEngine.getKafkaConfig(FeatureGroupBase featureGroup,
Map<String,String> writeOptions) |
Statistics |
StatisticsEngine.getLast(FeatureGroupBase featureGroup) |
Statistics |
StatisticsEngine.getLast(TrainingDataset trainingDataset) |
FeatureGroup |
FeatureGroupEngine.getOrCreateFeatureGroup(FeatureStore featureStore,
String name,
Integer version,
String description,
List<String> primaryKeys,
List<String> partitionKeys,
String hudiPrecombineKey,
boolean onlineEnabled,
TimeTravelFormat timeTravelFormat,
StatisticsConfig statisticsConfig,
String topicName,
String notificationTopicName,
String eventTime) |
FeatureView |
FeatureViewEngine.getOrCreateFeatureView(FeatureStore featureStore,
String name,
Integer version,
Query query,
String description,
List<String> labels) |
StreamFeatureGroup |
FeatureGroupEngine.getOrCreateStreamFeatureGroup(FeatureStore featureStore,
String name,
Integer version,
String description,
List<String> primaryKeys,
List<String> partitionKeys,
String hudiPrecombineKey,
boolean onlineEnabled,
StatisticsConfig statisticsConfig,
String eventTime,
TimeTravelFormat timeTravelFormat,
List<Feature> features,
StorageConnector storageConnector,
String path) |
String |
TrainingDatasetEngine.getQuery(TrainingDataset trainingDataset,
Storage storage,
boolean withLabel,
boolean isHiveQuery) |
StreamFeatureGroup |
FeatureGroupEngine.getStreamFeatureGroup(FeatureStore featureStore,
String fgName,
Integer fgVersion) |
List<StreamFeatureGroup> |
FeatureGroupEngine.getStreamFeatureGroups(FeatureStore featureStore,
String fgName) |
Object |
TrainingDatasetEngine.getTag(TrainingDataset trainingDataset,
String name) |
Map<String,Object> |
TrainingDatasetEngine.getTags(TrainingDataset trainingDataset) |
TrainingDatasetBundle |
FeatureViewEngine.getTrainingDataset(FeatureView featureView,
Integer trainingDatasetVersion,
List<String> requestedSplits,
Map<String,String> userReadOptions) |
TrainingDatasetBundle |
FeatureViewEngine.getTrainingDataset(FeatureView featureView,
TrainingDataset trainingDataset,
List<String> requestedSplits,
Map<String,String> userReadOptions) |
TrainingDatasetBundle |
FeatureViewEngine.getTrainingDataset(FeatureView featureView,
TrainingDataset trainingDataset,
Map<String,String> userReadOptions) |
void |
FeatureGroupEngine.insert(ExternalFeatureGroup externalFeatureGroup,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Map<String,String> writeOptions) |
void |
FeatureGroupEngine.insert(FeatureGroup featureGroup,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Storage storage,
HudiOperationType operation,
org.apache.spark.sql.SaveMode saveMode,
List<String> partitionKeys,
String hudiPrecombineKey,
Map<String,String> writeOptions) |
void |
FeatureGroupEngine.insert(StreamFeatureGroup streamFeatureGroup,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
org.apache.spark.sql.SaveMode saveMode,
List<String> partitionKeys,
String hudiPrecombineKey,
Map<String,String> writeOptions,
JobConfiguration jobConfiguration) |
org.apache.spark.sql.streaming.StreamingQuery |
FeatureGroupEngine.insertStream(FeatureGroup featureGroup,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
String queryName,
String outputMode,
boolean awaitTermination,
Long timeout,
String checkpointLocation,
List<String> partitionKeys,
String hudiPrecombineKey,
Map<String,String> writeOptions)
Deprecated.
|
List<Feature> |
SparkEngine.parseFeatureGroupSchema(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset,
TimeTravelFormat timeTravelFormat) |
List<TrainingDatasetFeature> |
TrainingDatasetUtils.parseTrainingDatasetSchema(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset) |
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
SparkEngine.read(StorageConnector storageConnector,
String dataFormat,
Map<String,String> readOptions,
String location) |
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
TrainingDatasetEngine.read(TrainingDataset trainingDataset,
String split,
Map<String,String> providedOptions) |
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
SparkEngine.readStream(StorageConnector storageConnector,
String dataFormat,
String messageFormat,
String schema,
Map<String,String> options,
boolean includeMetadata) |
void |
FeatureViewEngine.recreateTrainingDataset(FeatureView featureView,
Integer version,
Map<String,String> userWriteOptions) |
void |
SparkEngine.registerHudiTemporaryTable(FeatureGroupAlias featureGroupAlias,
Map<String,String> readOptions) |
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
SparkEngine.registerOnDemandTemporaryTable(ExternalFeatureGroup onDemandFeatureGroup,
String alias) |
FeatureGroup |
FeatureGroupEngine.save(FeatureGroup featureGroup,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset,
List<String> partitionKeys,
String hudiPrecombineKey,
Map<String,String> writeOptions)
Create the metadata and write the data to the online/offline feature store.
|
StreamFeatureGroup |
FeatureGroupEngine.save(StreamFeatureGroup featureGroup,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset,
List<String> partitionKeys,
String hudiPrecombineKey,
Map<String,String> writeOptions,
JobConfiguration sparkJobConfiguration)
Create the metadata and write the data to the online/offline feature store.
|
TrainingDataset |
TrainingDatasetEngine.save(TrainingDataset trainingDataset,
Query query,
Map<String,String> userWriteOptions,
List<String> labels)
Make a REST call to Hopsworks to create the metadata and write the data on the File System.
|
void |
FeatureGroupEngine.saveDataframe(FeatureGroup featureGroup,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset,
Storage storage,
HudiOperationType operation,
Map<String,String> offlineWriteOptions,
Map<String,String> onlineWriteOptions,
Integer validationId) |
ExternalFeatureGroup |
FeatureGroupEngine.saveExternalFeatureGroup(ExternalFeatureGroup externalFeatureGroup) |
FeatureGroup |
FeatureGroupEngine.saveFeatureGroupMetaData(FeatureGroup featureGroup,
List<String> partitionKeys,
String hudiPrecombineKey,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
boolean saveEmpty) |
StreamFeatureGroup |
FeatureGroupEngine.saveFeatureGroupMetaData(StreamFeatureGroup featureGroup,
List<String> partitionKeys,
String hudiPrecombineKey,
Map<String,String> writeOptions,
JobConfiguration sparkJobConfiguration,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData) |
static void |
TrainingDatasetUtils.setLabelFeature(List<TrainingDatasetFeature> features,
List<String> labels) |
void |
SparkEngine.setupConnectorHadoopConf(StorageConnector storageConnector) |
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>[] |
SparkEngine.splitDataset(TrainingDataset trainingDataset,
Query query,
Map<String,String> readOptions) |
void |
TrainingDatasetUtils.trainingDatasetSchemaMatch(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset,
List<TrainingDatasetFeature> features) |
FeatureView |
FeatureViewEngine.update(FeatureView featureView) |
void |
TrainingDatasetEngine.updateStatisticsConfig(TrainingDataset trainingDataset) |
void |
SparkEngine.validateSparkConfiguration() |
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>[] |
SparkEngine.write(TrainingDataset trainingDataset,
Query query,
Map<String,String> queryReadOptions,
Map<String,String> writeOptions,
org.apache.spark.sql.SaveMode saveMode)
Setup Spark to write the data on the File System.
|
void |
SparkEngine.writeEmptyDataframe(FeatureGroupBase featureGroup) |
void |
SparkEngine.writeOfflineDataframe(FeatureGroupBase featureGroup,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset,
HudiOperationType operation,
Map<String,String> writeOptions,
Integer validationId) |
void |
SparkEngine.writeOnlineDataframe(FeatureGroupBase featureGroupBase,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset,
String onlineTopicName,
Map<String,String> writeOptions)
Writes feature group dataframe to kafka for online-fs ingestion.
|
<S> org.apache.spark.sql.streaming.StreamingQuery |
SparkEngine.writeStreamDataframe(FeatureGroupBase featureGroupBase,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset,
String queryName,
String outputMode,
boolean awaitTermination,
Long timeout,
String checkpointLocation,
Map<String,String> writeOptions) |
void |
FeatureViewEngine.writeTrainingDataset(FeatureView featureView,
TrainingDataset trainingDataset,
Map<String,String> userWriteOptions) |
| Modifier and Type | Method and Description |
|---|---|
FeatureGroupCommit |
HudiEngine.deleteRecord(org.apache.spark.sql.SparkSession sparkSession,
FeatureGroupBase featureGroup,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> deleteDF,
Map<String,String> writeOptions) |
void |
HudiEngine.reconcileHudiSchema(org.apache.spark.sql.SparkSession sparkSession,
FeatureGroupAlias featureGroupAlias,
Map<String,String> hudiArgs) |
void |
HudiEngine.saveHudiFeatureGroup(org.apache.spark.sql.SparkSession sparkSession,
FeatureGroupBase featureGroup,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset,
HudiOperationType operation,
Map<String,String> writeOptions,
Integer validationId) |
| Modifier and Type | Method and Description |
|---|---|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
StorageConnectorUtils.read(StorageConnector.AdlsConnector connector,
String dataFormat,
Map<String,String> options,
String path)
Reads path into a spark dataframe using the AdlsConnector.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
StorageConnectorUtils.read(StorageConnector.BigqueryConnector connector,
String query,
Map<String,String> options,
String path)
Reads a query or a path into a spark dataframe using the sBigqueryConnector.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
StorageConnectorUtils.read(StorageConnector.GcsConnector connector,
String dataFormat,
Map<String,String> options,
String path)
Reads a path into a spark dataframe using the GcsConnector.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
StorageConnectorUtils.read(StorageConnector.HopsFsConnector connector,
String dataFormat,
Map<String,String> options,
String path)
Reads path into a spark dataframe using the HopsFsConnector.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
StorageConnectorUtils.read(StorageConnector.JdbcConnector connector,
String query)
Reads query into a spark dataframe using the JdbcConnector.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
StorageConnectorUtils.read(StorageConnector.RedshiftConnector connector,
String query)
Reads query into a spark dataframe using the RedshiftConnector.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
StorageConnectorUtils.read(StorageConnector.S3Connector connector,
String dataFormat,
Map<String,String> options,
String path)
Reads path into a spark dataframe using the S3Connector.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
StorageConnectorUtils.read(StorageConnector.SnowflakeConnector connector,
String query)
Reads query into a spark dataframe using the SnowflakeConnector.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
StorageConnectorUtils.read(StorageConnector connector,
String query,
String dataFormat,
Map<String,String> options,
String path)
Reads a query or a path into a spark dataframe using the storage connector.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
StorageConnectorUtils.readStream(StorageConnector.KafkaConnector connector,
String topic,
boolean topicPattern,
String messageFormat,
String schema,
Map<String,String> options,
boolean includeMetadata)
Reads stream into a spark dataframe using the kafka storage connector.
|
Copyright © 2025. All rights reserved.