| Modifier and Type | Method and Description |
|---|---|
void |
FeatureGroupBase.addTag(String name,
Object value)
Add name/value tag to the feature group.
|
abstract void |
TrainingDatasetBase.addTag(String name,
Object value)
Add name/value tag to the training dataset.
|
abstract void |
FeatureViewBase.addTag(String name,
Object value)
Add name/value tag to the feature view.
|
abstract void |
FeatureViewBase.addTrainingDatasetTag(Integer version,
String name,
Object value)
Add name/value tag to the training dataset.
|
abstract void |
FeatureGroupBase.appendFeatures(Feature features)
Append a single feature to the schema of the feature group.
|
void |
FeatureGroupBaseForApi.appendFeatures(Feature features) |
abstract void |
FeatureGroupBase.appendFeatures(List<Feature> features)
Append features to the schema of the feature group.
|
void |
FeatureGroupBaseForApi.appendFeatures(List<Feature> features) |
abstract QueryBase |
FeatureGroupBase.asOf(String wallclockTime) |
QueryBase |
FeatureGroupBaseForApi.asOf(String wallclockTime) |
abstract QueryBase |
FeatureGroupBase.asOf(String wallclockTime,
String excludeUntil) |
QueryBase |
FeatureGroupBaseForApi.asOf(String wallclockTime,
String excludeUntil) |
abstract void |
FeatureViewBase.clean(T3 featureStore,
String featureViewName,
Integer featureViewVersion) |
void |
FeatureGroupBaseForApi.commitDeleteRecord(FeatureGroupBaseForApi featureData) |
void |
FeatureGroupBaseForApi.commitDeleteRecord(FeatureGroupBaseForApi featureData,
Map<String,String> writeOptions) |
abstract void |
FeatureGroupBase.commitDeleteRecord(T featureData) |
abstract void |
FeatureGroupBase.commitDeleteRecord(T featureData,
Map<String,String> writeOptions) |
abstract Map<Long,Map<String,String>> |
FeatureGroupBase.commitDetails() |
Map<Long,Map<String,String>> |
FeatureGroupBaseForApi.commitDetails() |
abstract Map<Long,Map<String,String>> |
FeatureGroupBase.commitDetails(Integer limit) |
Map<Long,Map<String,String>> |
FeatureGroupBaseForApi.commitDetails(Integer limit) |
abstract Map<Long,Map<String,String>> |
FeatureGroupBase.commitDetails(String wallclockTime) |
Map<Long,Map<String,String>> |
FeatureGroupBaseForApi.commitDetails(String wallclockTime) |
abstract Map<Long,Map<String,String>> |
FeatureGroupBase.commitDetails(String wallclockTime,
Integer limit) |
Map<Long,Map<String,String>> |
FeatureGroupBaseForApi.commitDetails(String wallclockTime,
Integer limit) |
abstract Statistics |
FeatureGroupBase.computeStatistics()
Recompute the statistics for the feature group and save them to the feature store.
|
abstract Statistics |
TrainingDatasetBase.computeStatistics()
Recompute the statistics for the entire training dataset and save them to the feature store.
|
Statistics |
FeatureGroupBaseForApi.computeStatistics() |
abstract Statistics |
FeatureGroupBase.computeStatistics(String wallclockTime) |
Statistics |
FeatureGroupBaseForApi.computeStatistics(String wallclockTime) |
void |
FeatureGroupBase.delete() |
abstract void |
TrainingDatasetBase.delete()
Delete training dataset and all associated metadata.
|
abstract void |
FeatureViewBase.delete() |
abstract void |
FeatureViewBase.deleteAllTrainingDatasets() |
void |
FeatureGroupBase.deleteTag(String name)
Delete a tag of the feature group.
|
abstract void |
TrainingDatasetBase.deleteTag(String name)
Delete a tag of the training dataset.
|
abstract void |
FeatureViewBase.deleteTag(String name)
Delete a tag of the feature view.
|
abstract void |
FeatureViewBase.deleteTrainingDataset(Integer version) |
abstract void |
FeatureViewBase.deleteTrainingDatasetTag(Integer version,
String name)
Delete a tag of the training dataset.
|
abstract Object |
FeatureStoreBase.getAdlsConnector(String name) |
String |
FeatureGroupBase.getAvroSchema() |
abstract T5 |
FeatureViewBase.getBatchData() |
abstract T5 |
FeatureViewBase.getBatchData(String startTime,
String endTime) |
abstract T5 |
FeatureViewBase.getBatchData(String startTime,
String endTime,
Map<String,String> readOptions) |
abstract String |
FeatureViewBase.getBatchQuery() |
abstract String |
FeatureViewBase.getBatchQuery(String startTime,
String endTime) |
abstract Object |
FeatureStoreBase.getBigqueryConnector(String name) |
org.apache.avro.Schema |
FeatureGroupBase.getDeserializedAvroSchema() |
org.apache.avro.Schema |
FeatureGroupBase.getDeserializedEncodedAvroSchema() |
String |
FeatureGroupBase.getEncodedAvroSchema() |
abstract Object |
FeatureStoreBase.getExternalFeatureGroup(String name)
Get a external feature group object with default version `1` from the feature store.
|
abstract Object |
FeatureStoreBase.getExternalFeatureGroup(@NonNull String name,
@NonNull Integer version)
Get a external feature group object from the feature store.
|
abstract Object |
FeatureStoreBase.getExternalFeatureGroups(@NonNull String name) |
Feature |
FeatureGroupBase.getFeature(String name)
Retrieve a feature of the feature group by name.
|
String |
FeatureGroupBase.getFeatureAvroSchema(String featureName) |
abstract Object |
FeatureStoreBase.getFeatureGroups(@NonNull String name) |
abstract Object |
HopsworksConnectionBase.getFeatureStore() |
abstract Object |
HopsworksConnectionBase.getFeatureStore(String name) |
List<Object> |
FeatureViewBase.getFeatureVector(Map<String,Object> entry)
Returns assembled feature vector from online feature store.
|
List<Object> |
FeatureViewBase.getFeatureVector(Map<String,Object> entry,
boolean external)
Returns assembled feature vector from online feature store.
|
List<List<Object>> |
FeatureViewBase.getFeatureVectors(Map<String,List<Object>> entry)
Returns assembled feature vectors in batches from online feature store.
|
List<List<Object>> |
FeatureViewBase.getFeatureVectors(Map<String,List<Object>> entry,
boolean external)
Returns assembled feature vectors in batches from online feature store.
|
abstract Object |
FeatureStoreBase.getFeatureView(String name) |
abstract Object |
FeatureStoreBase.getFeatureView(@NonNull String name,
@NonNull Integer version) |
abstract Object |
FeatureStoreBase.getGcsConnector(String name) |
abstract Object |
FeatureStoreBase.getHopsFsConnector(String name) |
abstract Object |
FeatureStoreBase.getJdbcConnector(String name) |
abstract Object |
FeatureStoreBase.getKafkaConnector(String name) |
abstract Object |
FeatureStoreBase.getOnlineStorageConnector() |
abstract Object |
FeatureStoreBase.getOrCreateFeatureGroup(String name,
Integer version) |
abstract Object |
FeatureStoreBase.getOrCreateFeatureGroup(String name,
Integer version,
List<String> primaryKeys,
boolean onlineEnabled,
String eventTime) |
abstract Object |
FeatureStoreBase.getOrCreateFeatureGroup(String name,
Integer version,
List<String> primaryKeys,
List<String> partitionKeys,
boolean onlineEnabled,
String eventTime) |
abstract Object |
FeatureStoreBase.getOrCreateFeatureGroup(String name,
Integer version,
String description,
List<String> primaryKeys,
List<String> partitionKeys,
String hudiPrecombineKey,
boolean onlineEnabled,
TimeTravelFormat timeTravelFormat,
StatisticsConfig statisticsConfig,
String eventTime) |
abstract Object |
FeatureStoreBase.getOrCreateFeatureView(String name,
T2 query,
Integer version) |
abstract Object |
FeatureStoreBase.getOrCreateFeatureView(String name,
T2 query,
Integer version,
String description,
List<String> labels) |
abstract Object |
FeatureStoreBase.getOrCreateStreamFeatureGroup(String name,
Integer version) |
abstract Object |
FeatureStoreBase.getOrCreateStreamFeatureGroup(String name,
Integer version,
List<String> primaryKeys,
boolean onlineEnabled,
String eventTime) |
abstract Object |
FeatureStoreBase.getOrCreateStreamFeatureGroup(String name,
Integer version,
List<String> primaryKeys,
List<String> partitionKeys,
boolean onlineEnabled,
String eventTime) |
abstract Object |
FeatureStoreBase.getOrCreateStreamFeatureGroup(String name,
Integer version,
String description,
List<String> primaryKeys,
List<String> partitionKeys,
String hudiPrecombineKey,
boolean onlineEnabled,
StatisticsConfig statisticsConfig,
String eventTime) |
abstract String |
StorageConnector.getPath(String subPath) |
HashSet<String> |
FeatureViewBase.getPrimaryKeys()
Set of primary key names that is used as keys in input dict object for `get_serving_vector` method.
|
Project |
HopsworksConnectionBase.getProject() |
String |
TrainingDatasetBase.getQuery() |
String |
TrainingDatasetBase.getQuery(boolean withLabel) |
String |
TrainingDatasetBase.getQuery(Storage storage) |
abstract String |
TrainingDatasetBase.getQuery(Storage storage,
boolean withLabel) |
abstract Object |
FeatureStoreBase.getRedshiftConnector(String name) |
abstract Object |
FeatureStoreBase.getS3Connector(String name) |
HashSet<String> |
TrainingDatasetBase.getServingKeys()
Set of primary key names that is used as keys in input dict object for `get_serving_vector` method.
|
List<Object> |
TrainingDatasetBase.getServingVector(Map<String,Object> entry)
Retrieve feature vector from online feature store.
|
List<Object> |
TrainingDatasetBase.getServingVector(Map<String,Object> entry,
boolean external)
Retrieve feature vector from online feature store.
|
List<List<Object>> |
TrainingDatasetBase.getServingVectors(Map<String,List<Object>> entry) |
List<List<Object>> |
TrainingDatasetBase.getServingVectors(Map<String,List<Object>> entry,
boolean external) |
abstract Object |
FeatureStoreBase.getSnowflakeConnector(String name) |
abstract Statistics |
FeatureGroupBase.getStatistics()
Get the last statistics commit for the feature group.
|
abstract Statistics |
TrainingDatasetBase.getStatistics()
Get the last statistics commit for the training dataset.
|
Statistics |
FeatureGroupBaseForApi.getStatistics() |
abstract Statistics |
TrainingDatasetBase.getStatistics(String commitTime)
Get the statistics of a specific commit time for the training dataset.
|
abstract StorageConnector |
FeatureStoreBase.getStorageConnector(String name)
Get a list of all versions of an external feature group from the feature store.
|
abstract Object |
FeatureStoreBase.getStreamFeatureGroup(String name)
Get a feature group object with default version `1` from the feature store.
|
abstract Object |
FeatureStoreBase.getStreamFeatureGroup(@NonNull String name,
@NonNull Integer version)
Get a feature group object from the feature store.
|
Subject |
FeatureGroupBase.getSubject() |
Object |
FeatureGroupBase.getTag(String name)
Get a single tag value of the feature group.
|
abstract Object |
TrainingDatasetBase.getTag(String name)
Get a single tag value of the training dataset.
|
abstract Object |
FeatureViewBase.getTag(String name)
Get a single tag value of the feature view.
|
Map<String,Object> |
FeatureGroupBase.getTags()
Get all tags of the feature group.
|
abstract Map<String,Object> |
TrainingDatasetBase.getTags()
Get all tags of the training dataset.
|
abstract Map<String,Object> |
FeatureViewBase.getTags()
Get all tags of the feature view.
|
abstract Object |
FeatureViewBase.getTrainingData(Integer version,
Map<String,String> readOptions) |
abstract TrainingDatasetBase |
FeatureStoreBase.getTrainingDataset(String name)
Get a training dataset object with the default version `1` from the selected feature store.
|
abstract TrainingDatasetBase |
FeatureStoreBase.getTrainingDataset(@NonNull String name,
@NonNull Integer version)
Get a training dataset object from the selected feature store.
|
abstract Object |
FeatureStoreBase.getTrainingDatasets(@NonNull String name)
Get all versions of a training dataset object from the selected feature store.
|
abstract Object |
FeatureViewBase.getTrainingDatasetTag(Integer version,
String name)
Get a single tag value of the training dataset.
|
abstract Map<String,Object> |
FeatureViewBase.getTrainingDatasetTags(Integer version)
Get all tags of the training dataset.
|
abstract Object |
FeatureViewBase.getTrainTestSplit(Integer version,
Map<String,String> readOptions) |
abstract Object |
FeatureViewBase.getTrainValidationTestSplit(Integer version,
Map<String,String> readOptions) |
void |
TrainingDatasetBase.initPreparedStatement()
Initialise and cache parametrised prepared statement to retrieve feature vector from online feature store.
|
void |
TrainingDatasetBase.initPreparedStatement(boolean external)
Initialise and cache parametrised prepared statement to retrieve feature vector from online feature store.
|
void |
TrainingDatasetBase.initPreparedStatement(boolean external,
boolean batch)
Initialise and cache parametrised prepared statement to retrieve batch feature vectors from online feature store.
|
void |
FeatureViewBase.initServing()
Initialise feature view to retrieve feature vector from online feature store.
|
void |
FeatureViewBase.initServing(Boolean batch,
Boolean external)
Deprecated.
|
void |
FeatureGroupBaseForApi.insert(FeatureGroupBaseForApi featureData) |
void |
FeatureGroupBaseForApi.insert(FeatureGroupBaseForApi featureData,
boolean overwrite) |
void |
FeatureGroupBaseForApi.insert(FeatureGroupBaseForApi featureData,
boolean overwrite,
Map<String,String> writeOptions) |
void |
FeatureGroupBaseForApi.insert(FeatureGroupBaseForApi featureData,
boolean overwrite,
Map<String,String> writeOptions,
JobConfiguration jobConfiguration) |
void |
FeatureGroupBaseForApi.insert(FeatureGroupBaseForApi featureData,
HudiOperationType operation) |
void |
FeatureGroupBaseForApi.insert(FeatureGroupBaseForApi featureData,
JobConfiguration jobConfiguration) |
void |
FeatureGroupBaseForApi.insert(FeatureGroupBaseForApi featureData,
Map<String,String> writeOptions) |
void |
FeatureGroupBaseForApi.insert(FeatureGroupBaseForApi featureData,
Storage storage) |
void |
FeatureGroupBaseForApi.insert(FeatureGroupBaseForApi featureData,
Storage storage,
boolean overwrite) |
void |
FeatureGroupBaseForApi.insert(FeatureGroupBaseForApi featureData,
Storage storage,
boolean overwrite,
HudiOperationType operation,
Map<String,String> writeOptions) |
abstract void |
FeatureGroupBase.insert(T featureData) |
abstract void |
FeatureGroupBase.insert(T featureData,
boolean overwrite) |
abstract void |
FeatureGroupBase.insert(T featureData,
boolean overwrite,
Map<String,String> writeOptions) |
abstract void |
FeatureGroupBase.insert(T featureData,
boolean overwrite,
Map<String,String> writeOptions,
JobConfiguration jobConfiguration) |
abstract void |
FeatureGroupBase.insert(T featureData,
HudiOperationType operation) |
abstract void |
FeatureGroupBase.insert(T featureData,
JobConfiguration jobConfiguration) |
abstract void |
FeatureGroupBase.insert(T featureData,
Map<String,String> writeOptions) |
abstract void |
FeatureGroupBase.insert(T featureData,
Storage storage) |
abstract void |
FeatureGroupBase.insert(T featureData,
Storage storage,
boolean overwrite) |
abstract void |
FeatureGroupBase.insert(T featureData,
Storage storage,
boolean overwrite,
HudiOperationType operation,
Map<String,String> writeOptions) |
abstract void |
FeatureViewBase.purgeAllTrainingData() |
abstract void |
FeatureViewBase.purgeTrainingData(Integer version) |
abstract T |
FeatureGroupBase.read() |
FeatureGroupBaseForApi |
FeatureGroupBaseForApi.read() |
abstract <T> T |
TrainingDatasetBase.read()
Read the content of the training dataset.
|
abstract T |
FeatureGroupBase.read(boolean online) |
FeatureGroupBaseForApi |
FeatureGroupBaseForApi.read(boolean online) |
abstract T |
FeatureGroupBase.read(boolean online,
Map<String,String> readOptions) |
FeatureGroupBaseForApi |
FeatureGroupBaseForApi.read(boolean online,
Map<String,String> readOptions) |
abstract T |
FeatureGroupBase.read(Map<String,String> readOptions) |
FeatureGroupBaseForApi |
FeatureGroupBaseForApi.read(Map<String,String> readOptions) |
abstract T |
FeatureGroupBase.read(String wallclockTime) |
FeatureGroupBaseForApi |
FeatureGroupBaseForApi.read(String wallclockTime) |
abstract T |
FeatureGroupBase.read(String wallclockTime,
Map<String,String> readOptions) |
abstract Object |
TrainingDatasetBase.read(String split,
Map<String,String> readOptions)
Read a single split from the training dataset.
|
FeatureGroupBaseForApi |
FeatureGroupBaseForApi.read(String wallclockTime,
Map<String,String> readOptions) |
StorageConnector |
StorageConnector.refetch() |
void |
TrainingDatasetBase.setTimeSeriesSplits(Integer timeSplitSize,
String trainStart,
String trainEnd,
String valStart,
String valEnd,
String testStart,
String testEnd) |
abstract void |
FeatureGroupBase.show(int numRows) |
abstract void |
TrainingDatasetBase.show(int numRows)
Show numRows from the training dataset (across all splits).
|
void |
FeatureGroupBaseForApi.show(int numRows) |
abstract void |
FeatureGroupBase.show(int numRows,
boolean online) |
void |
FeatureGroupBaseForApi.show(int numRows,
boolean online) |
void |
StorageConnector.S3Connector.update() |
void |
StorageConnector.RedshiftConnector.update() |
void |
StorageConnector.JdbcConnector.update() |
abstract T |
FeatureViewBase.update(T other) |
void |
FeatureGroupBase.updateDescription(String description)
Update the description of the feature group.
|
void |
FeatureGroupBase.updateFeatureDescription(String featureName,
String description)
Update the description of a single feature.
|
abstract void |
FeatureGroupBase.updateFeatures(Feature feature)
Update the metadata of multiple features.
|
void |
FeatureGroupBaseForApi.updateFeatures(Feature feature) |
abstract void |
FeatureGroupBase.updateFeatures(List<Feature> features)
Update the metadata of multiple features.
|
void |
FeatureGroupBaseForApi.updateFeatures(List<Feature> features) |
void |
FeatureGroupBase.updateStatisticsConfig()
Update the statistics configuration of the feature group.
|
abstract void |
TrainingDatasetBase.updateStatisticsConfig()
Update the statistics configuration of the training dataset.
|
protected void |
FeatureViewBase.validateTrainTestSplit(Float testSize,
String trainEnd,
String testStart) |
protected void |
FeatureViewBase.validateTrainValidationTestSplit(Float validationSize,
Float testSize,
String trainEnd,
String validationStart,
String validationEnd,
String testStart) |
| Constructor and Description |
|---|
Feature(String name,
String type,
Boolean primary,
Boolean partition) |
Feature(String name,
String type,
String onlineType,
Boolean primary,
Boolean partition,
String defaultValue,
String description) |
| Modifier and Type | Method and Description |
|---|---|
T |
QueryBase.asOf(String wallclockTime)
Perform time travel on the given Query.
|
T |
QueryBase.asOf(String wallclockTime,
String excludeUntil)
Perform time travel on the given Query.
|
String |
FsQueryBase.getStorageQuery(Storage storage) |
T |
QueryBase.pullChanges(String wallclockStartTime,
String wallclockEndTime)
Deprecated.
use asOf(wallclockEndTime, wallclockStartTime) instead
|
abstract Object |
QueryBase.read() |
abstract Object |
QueryBase.read(boolean online) |
abstract Object |
QueryBase.read(boolean online,
Map<String,String> readOptions) |
abstract void |
FsQueryBase.registerHudiFeatureGroups(Map<String,String> readOptions) |
abstract void |
FsQueryBase.registerOnDemandFeatureGroups() |
abstract void |
QueryBase.show(boolean online,
int numRows) |
abstract void |
QueryBase.show(int numRows) |
| Modifier and Type | Method and Description |
|---|---|
void |
FeatureGroupEngineBase.addTag(FeatureGroupBase featureGroupBase,
String name,
Object value) |
void |
FeatureViewEngineBase.addTag(FeatureViewBase featureViewBase,
String name,
Object value) |
void |
FeatureViewEngineBase.addTag(FeatureViewBase featureViewBase,
String name,
Object value,
Integer trainingDataVersion) |
void |
FeatureGroupEngineBase.delete(FeatureGroupBase featureGroupBase) |
void |
FeatureViewEngineBase.delete(T3 featureStoreBase,
String name) |
void |
FeatureViewEngineBase.delete(T3 featureStoreBase,
String name,
Integer version) |
void |
FeatureGroupEngineBase.deleteTag(FeatureGroupBase featureGroupBase,
String name) |
void |
FeatureViewEngineBase.deleteTag(FeatureViewBase featureViewBase,
String name) |
void |
FeatureViewEngineBase.deleteTag(FeatureViewBase featureViewBase,
String name,
Integer trainingDataVersion) |
void |
FeatureViewEngineBase.deleteTrainingData(T2 featureView) |
void |
FeatureViewEngineBase.deleteTrainingData(T2 featureView,
Integer trainingDataVersion) |
void |
FeatureViewEngineBase.deleteTrainingDatasetOnly(T2 featureView) |
void |
FeatureViewEngineBase.deleteTrainingDatasetOnly(T2 featureView,
Integer trainingDataVersion) |
List<FeatureViewBase> |
FeatureViewEngineBase.get(T3 featureStoreBase,
String name) |
abstract T2 |
FeatureViewEngineBase.get(T3 featureStore,
String name,
Integer version) |
T2 |
FeatureViewEngineBase.get(T3 featureStoreBase,
String name,
Integer version,
Class<T2> fvType) |
String |
FeatureGroupUtils.getAvroSchema(FeatureGroupBase featureGroup,
FeatureStoreBase featureStoreBase) |
abstract T5 |
FeatureViewEngineBase.getBatchData(T2 featureView,
Date startTime,
Date endTime,
Map<String,String> readOptions,
Integer trainingDataVersion) |
abstract T1 |
FeatureViewEngineBase.getBatchQuery(T2 featureView,
Date startTime,
Date endTime,
Boolean withLabels,
Integer trainingDataVersion) |
T1 |
FeatureViewEngineBase.getBatchQuery(T2 featureView,
Date startTime,
Date endTime,
Boolean withLabels,
Integer trainingDataVersion,
Class<T1> queryType) |
abstract String |
FeatureViewEngineBase.getBatchQueryString(T2 featureView,
Date startTime,
Date endTime,
Integer trainingDataVersion) |
Map<Long,Map<String,String>> |
FeatureGroupUtils.getCommitDetails(FeatureGroupBase featureGroup,
String wallclockTime,
Integer limit) |
static Date |
FeatureGroupUtils.getDateFromDateString(String inputDate) |
org.apache.avro.Schema |
FeatureGroupUtils.getDeserializedAvroSchema(String avroSchema) |
org.apache.avro.Schema |
FeatureGroupUtils.getDeserializedEncodedAvroSchema(org.apache.avro.Schema schema,
List<String> complexFeatures) |
String |
FeatureGroupUtils.getEncodedAvroSchema(org.apache.avro.Schema schema,
List<String> complexFeatures) |
String |
FeatureGroupUtils.getFeatureAvroSchema(String featureName,
org.apache.avro.Schema schema) |
List<Object> |
VectorServer.getFeatureVector(FeatureViewBase featureViewBase,
Map<String,Object> entry) |
List<Object> |
VectorServer.getFeatureVector(FeatureViewBase featureViewBase,
Map<String,Object> entry,
boolean external) |
List<Object> |
VectorServer.getFeatureVector(TrainingDatasetBase trainingDatasetBase,
Map<String,Object> entry) |
List<Object> |
VectorServer.getFeatureVector(TrainingDatasetBase trainingDatasetBase,
Map<String,Object> entry,
boolean external) |
List<List<Object>> |
VectorServer.getFeatureVectors(FeatureViewBase featureViewBase,
Map<String,List<Object>> entry) |
List<List<Object>> |
VectorServer.getFeatureVectors(FeatureViewBase featureViewBase,
Map<String,List<Object>> entry,
boolean external) |
List<List<Object>> |
VectorServer.getFeatureVectors(TrainingDatasetBase trainingDatasetBase,
Map<String,List<Object>> entry) |
List<List<Object>> |
VectorServer.getFeatureVectors(TrainingDatasetBase trainingDatasetBase,
Map<String,List<Object>> entry,
boolean external) |
String |
FeatureGroupUtils.getHiveServerConnection(FeatureGroupBase featureGroup,
String connectionString) |
abstract T2 |
FeatureViewEngineBase.getOrCreateFeatureView(T3 featureStore,
String name,
Integer version,
T1 query,
String description,
List<String> labels) |
Subject |
FeatureGroupUtils.getSubject(FeatureGroupBase featureGroup) |
Object |
FeatureGroupEngineBase.getTag(FeatureGroupBase featureGroupBase,
String name) |
Object |
FeatureViewEngineBase.getTag(FeatureViewBase featureViewBase,
String name) |
Object |
FeatureViewEngineBase.getTag(FeatureViewBase featureViewBase,
String name,
Integer trainingDataVersion) |
Map<String,Object> |
FeatureGroupEngineBase.getTags(FeatureGroupBase featureGroupBase) |
Map<String,Object> |
FeatureViewEngineBase.getTags(FeatureViewBase featureViewBase) |
Map<String,Object> |
FeatureViewEngineBase.getTags(FeatureViewBase featureViewBase,
Integer trainingDataVersion) |
static Long |
FeatureGroupUtils.getTimeStampFromDateString(String inputDate) |
void |
VectorServer.initPreparedStatement(FeatureViewBase featureViewBase,
boolean batch) |
void |
VectorServer.initPreparedStatement(FeatureViewBase featureViewBase,
boolean batch,
boolean external) |
void |
VectorServer.initPreparedStatement(TrainingDatasetBase trainingDatasetBase,
boolean batch) |
void |
VectorServer.initPreparedStatement(TrainingDatasetBase trainingDatasetBase,
boolean batch,
boolean external) |
void |
VectorServer.initServing(FeatureViewBase featureViewBase,
boolean batch) |
void |
VectorServer.initServing(FeatureViewBase featureViewBase,
boolean batch,
boolean external) |
void |
VectorServer.initServing(TrainingDatasetBase trainingDatasetBase,
boolean batch) |
static List<TrainingDatasetFeature> |
FeatureViewEngineBase.makeLabelFeatures(QueryBase query,
List<String> labels) |
T2 |
FeatureViewEngineBase.save(T2 featureViewBase,
Class<T2> fvType) |
void |
CodeEngine.saveCode(FeatureGroupBase featureGroup) |
void |
CodeEngine.saveCode(TrainingDatasetBase trainingDatasetBase) |
protected <T extends FeatureGroupBase> |
FeatureGroupEngineBase.saveExtennalFeatureGroupMetaData(T externalFeatureGroup,
Class<T> fgClass) |
abstract T2 |
FeatureViewEngineBase.update(T2 featureView) |
<T extends FeatureGroupBase> |
FeatureGroupEngineBase.updateDescription(FeatureGroupBase featureGroup,
String description,
Class<T> fgClass) |
<T extends FeatureGroupBase> |
FeatureGroupEngineBase.updateFeatures(FeatureGroupBase featureGroup,
List<Feature> features,
Class<T> fgClass) |
<T extends FeatureGroupBase> |
FeatureGroupEngineBase.updateStatisticsConfig(FeatureGroupBase featureGroup,
Class<T> fgClass) |
void |
FeatureGroupUtils.verifyAttributeKeyNames(FeatureGroupBase featureGroup,
List<String> partitionKeyNames,
String precombineKeyName) |
| Modifier and Type | Method and Description |
|---|---|
void |
FeatureView.addTag(String s,
Object o) |
void |
FeatureView.addTrainingDatasetTag(Integer integer,
String s,
Object o) |
void |
StreamFeatureGroup.appendFeatures(Feature feature) |
void |
StreamFeatureGroup.appendFeatures(List<Feature> feature) |
QueryBase |
StreamFeatureGroup.asOf(String wallclockTime) |
QueryBase |
StreamFeatureGroup.asOf(String wallclockTime,
String excludeUntil) |
void |
FeatureView.clean(FeatureStore featureStore,
String s,
Integer integer) |
void |
StreamFeatureGroup.commitDeleteRecord(org.apache.flink.streaming.api.datastream.DataStream<?> featureData) |
void |
StreamFeatureGroup.commitDeleteRecord(org.apache.flink.streaming.api.datastream.DataStream<?> featureData,
Map<String,String> writeOptions) |
Map<Long,Map<String,String>> |
StreamFeatureGroup.commitDetails() |
Map<Long,Map<String,String>> |
StreamFeatureGroup.commitDetails(Integer integer) |
Map<Long,Map<String,String>> |
StreamFeatureGroup.commitDetails(String limit) |
Map<Long,Map<String,String>> |
StreamFeatureGroup.commitDetails(String wallclockTime,
Integer limit) |
Statistics |
StreamFeatureGroup.computeStatistics() |
Statistics |
StreamFeatureGroup.computeStatistics(String wallclockTime) |
void |
FeatureView.delete() |
void |
FeatureView.deleteAllTrainingDatasets() |
void |
FeatureView.deleteTag(String s) |
void |
FeatureView.deleteTrainingDataset(Integer integer) |
void |
FeatureView.deleteTrainingDatasetTag(Integer integer,
String s) |
Object |
FeatureStore.getAdlsConnector(String name) |
org.apache.flink.streaming.api.datastream.DataStream<Object> |
FeatureView.getBatchData() |
org.apache.flink.streaming.api.datastream.DataStream<Object> |
FeatureView.getBatchData(String s,
String s1) |
org.apache.flink.streaming.api.datastream.DataStream<Object> |
FeatureView.getBatchData(String s,
String s1,
Map<String,String> map) |
String |
FeatureView.getBatchQuery() |
String |
FeatureView.getBatchQuery(String s,
String s1) |
Object |
FeatureStore.getBigqueryConnector(String name) |
Object |
FeatureStore.getExternalFeatureGroup(String name) |
Object |
FeatureStore.getExternalFeatureGroup(@NonNull String name,
@NonNull Integer version) |
Object |
FeatureStore.getExternalFeatureGroups(@NonNull String name) |
Object |
FeatureStore.getFeatureGroups(@NonNull String name) |
FeatureStore |
HopsworksConnection.getFeatureStore()
Retrieve the project feature store.
|
FeatureStore |
HopsworksConnection.getFeatureStore(String name)
Retrieve a feature store based on name.
|
FeatureView |
FeatureStore.getFeatureView(String name)
Get a feature view object from the selected feature store.
|
FeatureView |
FeatureStore.getFeatureView(@NonNull String name,
@NonNull Integer version)
Get a feature view object from the selected feature store.
|
Object |
FeatureStore.getGcsConnector(String name) |
Object |
FeatureStore.getHopsFsConnector(String name) |
Object |
FeatureStore.getJdbcConnector(String name) |
Object |
FeatureStore.getKafkaConnector(String name) |
Object |
FeatureStore.getOnlineStorageConnector() |
Object |
FeatureStore.getOrCreateFeatureGroup(String name,
Integer version) |
Object |
FeatureStore.getOrCreateFeatureGroup(String name,
Integer integer,
List<String> primaryKeys,
boolean onlineEnabled,
String eventTime) |
Object |
FeatureStore.getOrCreateFeatureGroup(String name,
Integer version,
List<String> primaryKeys,
List<String> partitionKeys,
boolean onlineEnabled,
String eventTime) |
Object |
FeatureStore.getOrCreateFeatureGroup(String name,
Integer version,
String description,
List<String> primaryKeys,
List<String> partitionKeys,
String hudiPrecombineKey,
boolean onlineEnabled,
TimeTravelFormat timeTravelFormat,
StatisticsConfig statisticsConfig,
String eventTime) |
FeatureView |
FeatureStore.getOrCreateFeatureView(String name,
com.logicalclocks.hsfs.flink.constructor.Query query,
Integer version) |
FeatureView |
FeatureStore.getOrCreateFeatureView(String name,
com.logicalclocks.hsfs.flink.constructor.Query query,
Integer version,
String description,
List<String> labels) |
StreamFeatureGroup |
FeatureStore.getOrCreateStreamFeatureGroup(String name,
Integer version) |
StreamFeatureGroup |
FeatureStore.getOrCreateStreamFeatureGroup(String name,
Integer version,
List<String> primaryKeys,
boolean onlineEnabled,
String eventTime) |
StreamFeatureGroup |
FeatureStore.getOrCreateStreamFeatureGroup(String name,
Integer version,
List<String> primaryKeys,
List<String> partitionKeys,
boolean onlineEnabled,
String eventTime) |
StreamFeatureGroup |
FeatureStore.getOrCreateStreamFeatureGroup(String name,
Integer version,
String description,
List<String> primaryKeys,
List<String> partitionKeys,
String hudiPrecombineKey,
boolean onlineEnabled,
StatisticsConfig statisticsConfig,
String eventTime) |
Object |
FeatureStore.getRedshiftConnector(String name) |
Object |
FeatureStore.getS3Connector(String name) |
Object |
FeatureStore.getSnowflakeConnector(String name) |
Statistics |
StreamFeatureGroup.getStatistics() |
StorageConnector |
FeatureStore.getStorageConnector(String name) |
StreamFeatureGroup |
FeatureStore.getStreamFeatureGroup(String name)
Get a stream feature group object from the feature store.
|
StreamFeatureGroup |
FeatureStore.getStreamFeatureGroup(@NonNull String name,
@NonNull Integer version)
Get a stream feature group object from the feature store.
|
Object |
FeatureView.getTag(String s) |
Map<String,Object> |
FeatureView.getTags() |
Object |
FeatureView.getTrainingData(Integer integer,
Map<String,String> map) |
TrainingDatasetBase |
FeatureStore.getTrainingDataset(String name) |
TrainingDatasetBase |
FeatureStore.getTrainingDataset(@NonNull String name,
@NonNull Integer version) |
Object |
FeatureStore.getTrainingDatasets(@NonNull String name) |
Object |
FeatureView.getTrainingDatasetTag(Integer integer,
String s) |
Map<String,Object> |
FeatureView.getTrainingDatasetTags(Integer integer) |
Object |
FeatureView.getTrainTestSplit(Integer integer,
Map<String,String> map) |
Object |
FeatureView.getTrainValidationTestSplit(Integer integer,
Map<String,String> map) |
void |
StreamFeatureGroup.insert(org.apache.flink.streaming.api.datastream.DataStream<?> featureData) |
void |
StreamFeatureGroup.insert(org.apache.flink.streaming.api.datastream.DataStream<?> featureData,
boolean overwrite) |
void |
StreamFeatureGroup.insert(org.apache.flink.streaming.api.datastream.DataStream<?> featureData,
boolean online,
Map<String,String> writeOptions) |
void |
StreamFeatureGroup.insert(org.apache.flink.streaming.api.datastream.DataStream<?> featureData,
boolean online,
Map<String,String> writeOptions,
JobConfiguration jobConfiguration) |
void |
StreamFeatureGroup.insert(org.apache.flink.streaming.api.datastream.DataStream<?> featureData,
HudiOperationType hudiOperationType) |
void |
StreamFeatureGroup.insert(org.apache.flink.streaming.api.datastream.DataStream<?> featureData,
JobConfiguration jobConfiguration) |
void |
StreamFeatureGroup.insert(org.apache.flink.streaming.api.datastream.DataStream<?> featureData,
Map<String,String> writeOptions) |
void |
StreamFeatureGroup.insert(org.apache.flink.streaming.api.datastream.DataStream<?> featureData,
Storage storage) |
void |
StreamFeatureGroup.insert(org.apache.flink.streaming.api.datastream.DataStream<?> featureData,
Storage storage,
boolean overwrite) |
void |
StreamFeatureGroup.insert(org.apache.flink.streaming.api.datastream.DataStream<?> featureData,
Storage storage,
boolean online,
HudiOperationType hudiOperationType,
Map<String,String> writeOptions) |
void |
FeatureView.purgeAllTrainingData() |
void |
FeatureView.purgeTrainingData(Integer integer) |
org.apache.flink.streaming.api.datastream.DataStream<?> |
StreamFeatureGroup.read() |
org.apache.flink.streaming.api.datastream.DataStream<?> |
StreamFeatureGroup.read(boolean online) |
org.apache.flink.streaming.api.datastream.DataStream<?> |
StreamFeatureGroup.read(boolean online,
Map<String,String> readOptions) |
org.apache.flink.streaming.api.datastream.DataStream<?> |
StreamFeatureGroup.read(Map<String,String> readOptions) |
org.apache.flink.streaming.api.datastream.DataStream<?> |
StreamFeatureGroup.read(String wallclockTime) |
org.apache.flink.streaming.api.datastream.DataStream<?> |
StreamFeatureGroup.read(String wallclockTime,
Map<String,String> readOptions) |
void |
StreamFeatureGroup.show(int numRows) |
void |
StreamFeatureGroup.show(int numRows,
boolean online) |
FeatureView |
FeatureView.update(FeatureView featureView) |
void |
StreamFeatureGroup.updateFeatures(Feature feature) |
void |
StreamFeatureGroup.updateFeatures(List<Feature> feature) |
| Constructor and Description |
|---|
HopsworksConnection(String host,
int port,
String project,
software.amazon.awssdk.regions.Region region,
SecretStore secretStore,
boolean hostnameVerification,
String trustStorePath,
String certPath,
String apiKeyFilePath,
String apiKeyValue) |
| Modifier and Type | Method and Description |
|---|---|
FeatureView |
FeatureViewEngine.get(FeatureStore featureStore,
String name,
Integer version) |
org.apache.flink.streaming.api.datastream.DataStream |
FeatureViewEngine.getBatchData(FeatureView featureView,
Date startTime,
Date endTime,
Map<String,String> readOptions,
Integer trainingDataVersion) |
com.logicalclocks.hsfs.flink.constructor.Query |
FeatureViewEngine.getBatchQuery(FeatureView featureView,
Date date,
Date date1,
Boolean withLabels,
Integer integer) |
String |
FeatureViewEngine.getBatchQueryString(FeatureView featureView,
Date startTime,
Date endTime,
Integer trainingDataVersion) |
static FlinkEngine |
FlinkEngine.getInstance() |
FeatureView |
FeatureViewEngine.getOrCreateFeatureView(FeatureStore featureStore,
String name,
Integer version,
com.logicalclocks.hsfs.flink.constructor.Query query,
String description,
List<String> labels) |
StreamFeatureGroup |
FeatureGroupEngine.getStreamFeatureGroup(FeatureStore featureStore,
String fgName,
Integer fgVersion) |
List<StreamFeatureGroup> |
FeatureGroupEngine.getStreamFeatureGroups(FeatureStore featureStore,
String fgName) |
FeatureView |
FeatureViewEngine.update(FeatureView featureView) |
org.apache.flink.streaming.api.datastream.DataStreamSink<?> |
FlinkEngine.writeDataStream(StreamFeatureGroup streamFeatureGroup,
org.apache.flink.streaming.api.datastream.DataStream<?> dataStream,
Map<String,String> writeOptions) |
| Modifier and Type | Method and Description |
|---|---|
void |
TagsApi.add(FeatureGroupBase featureGroupBase,
String name,
Object value) |
void |
TagsApi.add(FeatureViewBase featureViewBase,
Integer trainingDatasetVersion,
String name,
Object value) |
void |
TagsApi.add(FeatureViewBase featureViewBase,
String name,
Object value) |
void |
TagsApi.add(TrainingDatasetBase trainingDatasetBase,
String name,
Object value) |
<T> FsQueryBase |
QueryConstructorApi.constructQuery(FeatureStoreBase featureStoreBase,
QueryBase queryBase,
Class<T> fsQueryType) |
<T extends TrainingDatasetBase> |
FeatureViewApi.createTrainingData(String featureViewName,
Integer featureViewVersion,
TrainingDatasetBase trainingData,
Class<T> tdType) |
TrainingDatasetBase |
TrainingDatasetApi.createTrainingDataset(TrainingDatasetBase trainingDatasetBase) |
void |
FeatureGroupApi.delete(FeatureGroupBase featureGroupBase) |
void |
FeatureViewApi.delete(FeatureStoreBase featureStoreBase,
String name) |
void |
FeatureViewApi.delete(FeatureStoreBase featureStoreBase,
String name,
Integer version) |
void |
TrainingDatasetApi.delete(TrainingDatasetBase trainingDatasetBase) |
void |
FeatureGroupApi.deleteContent(FeatureGroupBase featureGroup) |
void |
TagsApi.deleteTag(FeatureGroupBase featureGroup,
String name) |
void |
TagsApi.deleteTag(FeatureViewBase featureViewBase,
Integer trainingDatasetVersion,
String name) |
void |
TagsApi.deleteTag(FeatureViewBase featureViewBase,
String name) |
void |
TagsApi.deleteTag(TrainingDatasetBase trainingDatasetBase,
String name) |
void |
FeatureViewApi.deleteTrainingData(FeatureStoreBase featureStoreBase,
String name,
Integer version) |
void |
FeatureViewApi.deleteTrainingData(FeatureStoreBase featureStoreBase,
String featureViewName,
Integer featureViewVersion,
Integer trainingDataVersion) |
void |
FeatureViewApi.deleteTrainingDatasetOnly(FeatureStoreBase featureStoreBase,
String name,
Integer version) |
void |
FeatureViewApi.deleteTrainingDatasetOnly(FeatureStoreBase featureStoreBase,
String name,
Integer version,
Integer trainingDataVersion) |
FeatureGroupCommit |
FeatureGroupApi.featureGroupCommit(FeatureGroupBase featureGroup,
FeatureGroupCommit featureGroupCommit) |
Map<String,Object> |
TagsApi.get(FeatureGroupBase featureGroupBase) |
Object |
TagsApi.get(FeatureGroupBase featureGroupBase,
String name) |
Statistics |
StatisticsApi.get(FeatureGroupBase featureGroup,
String commitTime) |
List<FeatureViewBase> |
FeatureViewApi.get(FeatureStoreBase featureStoreBase,
String name) |
List<TrainingDatasetBase> |
TrainingDatasetApi.get(FeatureStoreBase featureStoreBase,
String tdName,
Integer tdVersion) |
<T extends FeatureViewBase> |
FeatureViewApi.get(FeatureStoreBase featureStoreBase,
String name,
Integer version,
Class<T> fvType) |
Map<String,Object> |
TagsApi.get(FeatureViewBase featureViewBase) |
Map<String,Object> |
TagsApi.get(FeatureViewBase featureViewBase,
Integer trainingDatasetVersion) |
Object |
TagsApi.get(FeatureViewBase featureViewBase,
Integer trainingDatasetVersion,
String name) |
Object |
TagsApi.get(FeatureViewBase featureViewBase,
String name) |
<U> U |
StorageConnectorApi.get(Integer featureStoreId,
String name,
Class<U> storageConnectorType) |
<T> T |
FeatureStoreApi.get(int projectId,
String name,
Class<T> fsType) |
Project |
ProjectApi.get(String name) |
Optional<Variable> |
VariablesApi.get(String variableName) |
Map<String,Object> |
TagsApi.get(TrainingDatasetBase trainingDatasetBase) |
Object |
TagsApi.get(TrainingDatasetBase trainingDatasetBase,
String name) |
Statistics |
StatisticsApi.get(TrainingDatasetBase trainingDatasetBase,
String commitTime) |
protected String |
HopsworksExternalClient.getAssumedRole() |
<T extends QueryBase> |
FeatureViewApi.getBatchQuery(FeatureStoreBase featureStoreBase,
String name,
Integer version,
Long startTime,
Long endTime,
Boolean withLabels,
Integer trainingDataVersion,
Class<T> queryType) |
List<String> |
KafkaApi.getBrokerEndpoints(FeatureStoreBase featureStoreBase) |
List<String> |
KafkaApi.getBrokerEndpoints(FeatureStoreBase featureStoreBase,
boolean externalListeners) |
<T> T |
StorageConnectorApi.getByName(FeatureStoreBase featureStoreBase,
String name,
Class<T> storageConnectorType) |
List<FeatureGroupCommit> |
FeatureGroupApi.getCommitDetails(FeatureGroupBase featureGroupBase,
Long wallclockTimestamp,
Integer limit) |
Credentials |
HopsworksClient.getCredentials(Project project) |
static HopsworksClient |
HopsworksClient.getInstance() |
<T> T |
FeatureGroupApi.getInternal(FeatureStoreBase featureStoreBase,
String fgName,
Integer fgVersion,
Class<T> fgType) |
Statistics |
StatisticsApi.getLast(FeatureGroupBase featureGroup) |
Statistics |
StatisticsApi.getLast(TrainingDatasetBase trainingDatasetBase) |
<T> T |
StorageConnectorApi.getOnlineStorageConnector(FeatureStoreBase featureStoreBase,
Class<T> storageConnectorType) |
FsQueryBase |
TrainingDatasetApi.getQuery(TrainingDatasetBase trainingDatasetBase,
boolean withLabel,
boolean isHiveQuery) |
List<ServingPreparedStatement> |
FeatureViewApi.getServingPreparedStatement(FeatureViewBase featureViewBase,
boolean batch) |
List<ServingPreparedStatement> |
TrainingDatasetApi.getServingPreparedStatement(TrainingDatasetBase trainingDatasetBase,
boolean batch) |
List<PartitionDetails> |
KafkaApi.getTopicDetails(FeatureStoreBase featureStoreBase,
String topicName) |
Subject |
KafkaApi.getTopicSubject(FeatureStoreBase featureStoreBase,
String topicName) |
<T extends TrainingDatasetBase> |
FeatureViewApi.getTrainingData(FeatureStoreBase featureStoreBase,
String featureViewName,
Integer featureViewVersion,
Integer trainingDataVersion,
Class<T> tdType) |
TrainingDatasetBase |
TrainingDatasetApi.getTrainingDataset(FeatureStoreBase featureStoreBase,
String tdName,
Integer tdVersion) |
List<TransformationFunctionAttached> |
FeatureViewApi.getTransformationFunctions(FeatureViewBase featureViewBase) |
List<TransformationFunctionAttached> |
TrainingDatasetApi.getTransformationFunctions(TrainingDatasetBase trainingDatasetBase) |
<T> T |
HopsworksClient.handleRequest(org.apache.http.HttpRequest request) |
<T> T |
HopsworksClient.handleRequest(org.apache.http.HttpRequest request,
Class<T> cls) |
<T> T |
HopsworksHttpClient.handleRequest(org.apache.http.HttpRequest request,
org.apache.http.client.ResponseHandler<T> responseHandler) |
<T> T |
HopsworksInternalClient.handleRequest(org.apache.http.HttpRequest request,
org.apache.http.client.ResponseHandler<T> responseHandler) |
<T> T |
HopsworksClient.handleRequest(org.apache.http.HttpRequest request,
org.apache.http.client.ResponseHandler<T> responseHandler) |
void |
CodeApi.post(FeatureGroupBase featureGroup,
Code code,
String entityId,
Code.RunType type,
String browserHostName) |
Statistics |
StatisticsApi.post(FeatureGroupBase featureGroup,
Statistics statistics) |
Statistics |
StatisticsApi.post(FeatureViewBase featureViewBase,
Integer trainingDataVersion,
Statistics statistics) |
void |
CodeApi.post(TrainingDatasetBase trainingDatasetBase,
Code code,
String entityId,
Code.RunType type,
String browserHostName) |
Statistics |
StatisticsApi.post(TrainingDatasetBase trainingDatasetBase,
Statistics statistics) |
String |
HopsworksExternalClient.readApiKey(SecretStore secretStore,
software.amazon.awssdk.regions.Region region,
String apiKeyFilepath)
Read API key.
|
protected String |
HopsworksExternalClient.readApiKeyParamStore(software.amazon.awssdk.regions.Region region,
String secretKey) |
protected String |
HopsworksExternalClient.readApiKeySecretManager(software.amazon.awssdk.regions.Region region,
String secretKey) |
void |
HopsworksInternalClient.refreshJwt() |
<U extends FeatureGroupBase> |
FeatureGroupApi.save(FeatureGroupBase featureGroup,
Class<U> fgType) |
<T extends FeatureViewBase> |
FeatureViewApi.save(FeatureViewBase featureViewBase,
Class<T> fvType) |
<U extends FeatureGroupBase> |
FeatureGroupApi.saveFeatureGroupMetaData(FeatureGroupBase featureGroup,
List<String> partitionKeys,
String hudiPrecombineKey,
Map<String,String> writeOptions,
JobConfiguration jobConfiguration,
Class<U> fgType) |
<U extends FeatureGroupBase> |
FeatureGroupApi.saveInternal(FeatureGroupBase featureGroupBase,
org.apache.http.entity.StringEntity entity,
Class<U> fgType) |
static HopsworksClient |
HopsworksClient.setupHopsworksClient(String host,
int port,
software.amazon.awssdk.regions.Region region,
SecretStore secretStore,
boolean hostnameVerification,
String trustStorePath,
String apiKeyFilePath,
String apiKeyValue) |
<T> T |
FeatureViewApi.update(FeatureViewBase featureViewBase,
Class<T> fvType) |
<T extends FeatureGroupBase> |
FeatureGroupApi.updateMetadata(FeatureGroupBase featureGroup,
String queryParameter,
Class<T> fgType) |
<T extends FeatureGroupBase> |
FeatureGroupApi.updateMetadata(FeatureGroupBase featureGroup,
String queryParameter,
Object value,
Class<T> fgType) |
TrainingDatasetBase |
TrainingDatasetApi.updateMetadata(TrainingDatasetBase trainingDatasetBase,
String queryParameter) |
| Constructor and Description |
|---|
HopsworksInternalClient() |
| Modifier and Type | Method and Description |
|---|---|
void |
TrainingDataset.addTag(String name,
Object value)
Add name/value tag to the training dataset.
|
void |
FeatureView.addTag(String name,
Object value)
Add name/value tag to the feature view.
|
void |
FeatureView.addTrainingDatasetTag(Integer version,
String name,
Object value)
Add name/value tag to the training dataset.
|
void |
ExternalFeatureGroup.appendFeatures(Feature features) |
void |
StreamFeatureGroup.appendFeatures(Feature features)
Append a single feature to the schema of the stream feature group.
|
void |
FeatureGroup.appendFeatures(Feature features)
Append a single feature to the schema of the feature group.
|
void |
ExternalFeatureGroup.appendFeatures(List<Feature> features) |
void |
StreamFeatureGroup.appendFeatures(List<Feature> features)
Append features to the schema of the stream feature group.
|
void |
FeatureGroup.appendFeatures(List<Feature> features)
Append features to the schema of the feature group.
|
QueryBase |
ExternalFeatureGroup.asOf(String wallclockTime) |
Query |
StreamFeatureGroup.asOf(String wallclockTime)
Get Query object to retrieve all features of the group at a point in the past.
|
Query |
FeatureGroup.asOf(String wallclockTime)
Get Query object to retrieve all features of the group at a point in the past.
|
QueryBase |
ExternalFeatureGroup.asOf(String wallclockTime,
String excludeUntil) |
Query |
StreamFeatureGroup.asOf(String wallclockTime,
String excludeUntil)
Get Query object to retrieve all features of the group at a point in the past.
|
Query |
FeatureGroup.asOf(String wallclockTime,
String excludeUntil)
Get Query object to retrieve all features of the group at a point in the past.
|
FeatureView |
FeatureView.FeatureViewBuilder.build() |
void |
FeatureView.clean(FeatureStore featureStore,
String featureViewName,
Integer featureViewVersion)
Delete the feature view and all associated metadata and training data.
|
void |
ExternalFeatureGroup.commitDeleteRecord(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData) |
void |
StreamFeatureGroup.commitDeleteRecord(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData)
Drops records present in the provided DataFrame and commits it as update to this Stream Feature group.
|
void |
FeatureGroup.commitDeleteRecord(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData)
Drops records present in the provided DataFrame and commits it as update to this Feature group.
|
void |
ExternalFeatureGroup.commitDeleteRecord(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Map<String,String> writeOptions) |
void |
StreamFeatureGroup.commitDeleteRecord(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Map<String,String> writeOptions)
Drops records present in the provided DataFrame and commits it as update to this Stream Feature group.
|
void |
FeatureGroup.commitDeleteRecord(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Map<String,String> writeOptions)
Drops records present in the provided DataFrame and commits it as update to this Feature group.
|
Map<Long,Map<String,String>> |
ExternalFeatureGroup.commitDetails() |
Map<Long,Map<String,String>> |
StreamFeatureGroup.commitDetails()
Retrieves commit timeline for this stream feature group.
|
Map<Long,Map<String,String>> |
FeatureGroup.commitDetails()
Retrieves commit timeline for this feature group.
|
Map<Long,Map<String,String>> |
ExternalFeatureGroup.commitDetails(Integer limit) |
Map<Long,Map<String,String>> |
StreamFeatureGroup.commitDetails(Integer limit)
/**
Retrieves commit timeline for this stream feature group.
|
Map<Long,Map<String,String>> |
FeatureGroup.commitDetails(Integer limit)
Retrieves commit timeline for this feature group.
|
Map<Long,Map<String,String>> |
ExternalFeatureGroup.commitDetails(String wallclockTime) |
Map<Long,Map<String,String>> |
StreamFeatureGroup.commitDetails(String wallclockTime)
Return commit details as of specific point in time.
|
Map<Long,Map<String,String>> |
FeatureGroup.commitDetails(String wallclockTime)
Return commit details as of specific point in time.
|
Map<Long,Map<String,String>> |
ExternalFeatureGroup.commitDetails(String wallclockTime,
Integer limit) |
Map<Long,Map<String,String>> |
StreamFeatureGroup.commitDetails(String wallclockTime,
Integer limit)
Return commit details as of specific point in time.
|
Map<Long,Map<String,String>> |
FeatureGroup.commitDetails(String wallclockTime,
Integer limit)
Return commit details as of specific point in time.
|
Statistics |
TrainingDataset.computeStatistics()
Recompute the statistics for the entire training dataset and save them to the feature store.
|
Statistics |
ExternalFeatureGroup.computeStatistics() |
Statistics |
StreamFeatureGroup.computeStatistics()
Recompute the statistics for the stream feature group and save them to the feature store.
|
Statistics |
FeatureGroup.computeStatistics()
Recompute the statistics for the feature group and save them to the feature store.
|
Statistics |
ExternalFeatureGroup.computeStatistics(String wallclockTime) |
Statistics |
StreamFeatureGroup.computeStatistics(String wallclockTime)
Recompute the statistics for the feature group and save them to the feature store.
|
Statistics |
FeatureGroup.computeStatistics(String wallclockTime)
Recompute the statistics for the feature group and save them to the feature store.
|
Integer |
FeatureView.createTrainingData(String startTime,
String endTime,
String description,
DataFormat dataFormat)
Create the metadata for a training dataset and save the corresponding training data into `location`.
|
Integer |
FeatureView.createTrainingData(String startTime,
String endTime,
String description,
DataFormat dataFormat,
Boolean coalesce,
StorageConnector storageConnector,
String location,
Long seed,
StatisticsConfig statisticsConfig,
Map<String,String> writeOptions,
FilterLogic extraFilterLogic,
Filter extraFilter)
Create the metadata for a training dataset and save the corresponding training data into `location`.
|
Integer |
FeatureView.createTrainTestSplit(Float testSize,
String trainStart,
String trainEnd,
String testStart,
String testEnd,
String description,
DataFormat dataFormat)
Create the metadata for a training dataset and save the corresponding training data into `location`.
|
Integer |
FeatureView.createTrainTestSplit(Float testSize,
String trainStart,
String trainEnd,
String testStart,
String testEnd,
String description,
DataFormat dataFormat,
Boolean coalesce,
StorageConnector storageConnector,
String location,
Long seed,
StatisticsConfig statisticsConfig,
Map<String,String> writeOptions,
FilterLogic extraFilterLogic,
Filter extraFilter)
Create the metadata for a training dataset and save the corresponding training data into `location`.
|
Integer |
FeatureView.createTrainValidationTestSplit(Float validationSize,
Float testSize,
String trainStart,
String trainEnd,
String validationStart,
String validationEnd,
String testStart,
String testEnd,
String description,
DataFormat dataFormat)
Create the metadata for a training dataset and save the corresponding training data into `location`.
|
Integer |
FeatureView.createTrainValidationTestSplit(Float validationSize,
Float testSize,
String trainStart,
String trainEnd,
String validationStart,
String validationEnd,
String testStart,
String testEnd,
String description,
DataFormat dataFormat,
Boolean coalesce,
StorageConnector storageConnector,
String location,
Long seed,
StatisticsConfig statisticsConfig,
Map<String,String> writeOptions,
FilterLogic extraFilterLogic,
Filter extraFilter)
Create the metadata for a training dataset and save the corresponding training data into `location`.
|
void |
TrainingDataset.delete()
Delete training dataset and all associated metadata.
|
void |
FeatureView.delete()
Delete current feature view, all associated metadata and training data.
|
void |
FeatureView.deleteAllTrainingDatasets()
Delete all training datasets.
|
void |
TrainingDataset.deleteTag(String name)
Delete a tag of the training dataset.
|
void |
FeatureView.deleteTag(String name)
Delete a tag of the feature view.
|
void |
FeatureView.deleteTrainingDataset(Integer version)
Delete a training dataset.
|
void |
FeatureView.deleteTrainingDatasetTag(Integer version,
String name)
Delete a tag of the training dataset.
|
StorageConnector.AdlsConnector |
FeatureStore.getAdlsConnector(String name)
Get a previously created Adls compliant storage connector from the feature store.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
FeatureView.getBatchData()
Get all data from the feature view as a batch from the offline feature store.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
FeatureView.getBatchData(String startTime,
String endTime)
Get a batch of data from an event time interval from the offline feature store.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
FeatureView.getBatchData(String startTime,
String endTime,
Map<String,String> readOptions)
Get a batch of data from an event time interval from the offline feature store.
|
String |
FeatureView.getBatchQuery()
Get a query string of the batch query.
|
String |
FeatureView.getBatchQuery(String startTime,
String endTime)
Get a query string of the batch query.
|
StorageConnector.BigqueryConnector |
FeatureStore.getBigqueryConnector(String name)
Get a previously created BigQuery compliant storage connector from the feature store.
|
ExternalFeatureGroup |
FeatureStore.getExternalFeatureGroup(String name)
Get an external feature group object with default version `1` from the feature store.
|
ExternalFeatureGroup |
FeatureStore.getExternalFeatureGroup(@NonNull String name,
@NonNull Integer version)
Get an external feature group object from the feature store.
|
scala.collection.Seq<ExternalFeatureGroup> |
FeatureStore.getExternalFeatureGroups(@NonNull String name)
Get a list of all versions of an external feature group from the feature store.
|
FeatureGroup |
FeatureStore.getFeatureGroup(String name)
Get a feature group object with default version `1` from the feature store.
|
FeatureGroup |
FeatureStore.getFeatureGroup(@NonNull String name,
@NonNull Integer version)
Get a feature group object from the feature store.
|
scala.collection.Seq<FeatureGroup> |
FeatureStore.getFeatureGroups(@NonNull String name)
Get a list of all versions of a feature group from the feature store.
|
FeatureStore |
HopsworksConnection.getFeatureStore()
Retrieve the project feature store.
|
FeatureStore |
HopsworksConnection.getFeatureStore(String name)
Retrieve a feature store based on name.
|
FeatureView |
FeatureStore.getFeatureView(String name)
Get a feature view object with the default version `1` from the selected feature store.
|
FeatureView |
FeatureStore.getFeatureView(@NonNull String name,
@NonNull Integer version)
Get a feature view object from the selected feature store.
|
StorageConnector.GcsConnector |
FeatureStore.getGcsConnector(String name)
Get a previously created Gcs compliant storage connector from the feature store.
|
StorageConnector.HopsFsConnector |
FeatureStore.getHopsFsConnector(String name)
Get a previously created HopsFs compliant storage connector from the feature store.
|
StorageConnector.JdbcConnector |
FeatureStore.getJdbcConnector(String name)
Get a previously created JDBC compliant storage connector from the feature store.
|
StorageConnector.KafkaConnector |
FeatureStore.getKafkaConnector(String name)
Get a previously created Kafka compliant storage connector from the feature store.
|
ExternalFeatureGroup |
FeatureStore.getOnDemandFeatureGroup(String name)
Deprecated.
|
ExternalFeatureGroup |
FeatureStore.getOnDemandFeatureGroup(@NonNull String name,
@NonNull Integer version)
Deprecated.
|
scala.collection.Seq<ExternalFeatureGroup> |
FeatureStore.getOnDemandFeatureGroups(@NonNull String name)
Deprecated.
|
StorageConnector.JdbcConnector |
FeatureStore.getOnlineStorageConnector()
Get a previously created JDBC compliant storage connector from the feature store
to connect to the online feature store.
|
FeatureGroup |
FeatureStore.getOrCreateFeatureGroup(String name,
Integer version)
Get feature group metadata object or create a new one if it doesn't exist.
|
FeatureGroup |
FeatureStore.getOrCreateFeatureGroup(String name,
Integer version,
List<String> primaryKeys,
boolean onlineEnabled,
String eventTime)
Get feature group metadata object or create a new one if it doesn't exist.
|
FeatureGroup |
FeatureStore.getOrCreateFeatureGroup(String name,
Integer version,
List<String> primaryKeys,
List<String> partitionKeys,
boolean onlineEnabled,
String eventTime)
Get feature group metadata object or create a new one if it doesn't exist.
|
FeatureGroup |
FeatureStore.getOrCreateFeatureGroup(String name,
Integer version,
String description,
List<String> primaryKeys,
List<String> partitionKeys,
String hudiPrecombineKey,
boolean onlineEnabled,
TimeTravelFormat timeTravelFormat,
StatisticsConfig statisticsConfig,
String eventTime)
Get feature group metadata object or create a new one if it doesn't exist.
|
FeatureView |
FeatureStore.getOrCreateFeatureView(String name,
Query query,
Integer version)
Get feature view metadata object or create a new one if it doesn't exist.
|
FeatureView |
FeatureStore.getOrCreateFeatureView(String name,
Query query,
Integer version,
String description,
List<String> labels)
Get feature view metadata object or create a new one if it doesn't exist.
|
StreamFeatureGroup |
FeatureStore.getOrCreateStreamFeatureGroup(String name,
Integer version)
Get stream feature group metadata object or create a new one if it doesn't exist.
|
StreamFeatureGroup |
FeatureStore.getOrCreateStreamFeatureGroup(String name,
Integer version,
List<String> primaryKeys,
boolean onlineEnabled,
String eventTime)
Get stream feature group metadata object or create a new one if it doesn't exist.
|
StreamFeatureGroup |
FeatureStore.getOrCreateStreamFeatureGroup(String name,
Integer version,
List<String> primaryKeys,
List<String> partitionKeys,
boolean onlineEnabled,
String eventTime)
Get stream feature group metadata object or create a new one if it doesn't exist.
|
StreamFeatureGroup |
FeatureStore.getOrCreateStreamFeatureGroup(String name,
Integer version,
String description,
List<String> primaryKeys,
List<String> partitionKeys,
String hudiPrecombineKey,
boolean onlineEnabled,
StatisticsConfig statisticsConfig,
String eventTime)
Get stream feature group metadata object or create a new one if it doesn't exist.
|
HashSet<String> |
FeatureView.getPrimaryKeys()
Get set of primary key names that is used as keys in input dict object for `getServingVector` method.
|
String |
TrainingDataset.getQuery(Storage storage,
boolean withLabel) |
StorageConnector.RedshiftConnector |
FeatureStore.getRedshiftConnector(String name)
Get a previously created Redshift compliant storage connector from the feature store.
|
StorageConnector.S3Connector |
FeatureStore.getS3Connector(String name)
Get a previously created S3 compliant storage connector from the feature store.
|
List<Object> |
TrainingDataset.getServingVector(Map<String,Object> entry)
Retrieve feature vector from online feature store.
|
StorageConnector.SnowflakeConnector |
FeatureStore.getSnowflakeConnector(String name)
Get a previously created Snowflake compliant storage connector from the feature store.
|
Statistics |
TrainingDataset.getStatistics()
Get the last statistics commit for the training dataset.
|
Statistics |
ExternalFeatureGroup.getStatistics() |
Statistics |
StreamFeatureGroup.getStatistics() |
Statistics |
FeatureGroup.getStatistics()
Get the last statistics commit for the feature group.
|
Statistics |
TrainingDataset.getStatistics(String commitTime)
Get the statistics of a specific commit time for the training dataset.
|
StorageConnector |
FeatureStore.getStorageConnector(String name)
Get a previously created storage connector from the feature store.
|
StreamFeatureGroup |
FeatureStore.getStreamFeatureGroup(String name)
Get a stream feature group object with default version `1` from the feature store.
|
StreamFeatureGroup |
FeatureStore.getStreamFeatureGroup(@NonNull String name,
@NonNull Integer version)
Get a stream feature group object from the feature store.
|
Object |
TrainingDataset.getTag(String name)
Get a single tag value of the training dataset.
|
Object |
FeatureView.getTag(String name)
Get a single tag value of the feature view.
|
Map<String,Object> |
TrainingDataset.getTags()
Get all tags of the training dataset.
|
Map<String,Object> |
FeatureView.getTags()
Get all tags of the feature view.
|
List<org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>> |
FeatureView.getTrainingData(Integer version)
Get training data created by `featureView.createTrainingData` or `featureView.trainingData`.
|
List<org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>> |
FeatureView.getTrainingData(Integer version,
Map<String,String> readOptions)
Get training data created by `featureView.createTrainingData` or `featureView.trainingData`.
|
TrainingDataset |
FeatureStore.getTrainingDataset(String name)
Deprecated.
|
TrainingDataset |
FeatureStore.getTrainingDataset(@NonNull String name,
@NonNull Integer version)
Deprecated.
|
scala.collection.Seq<TrainingDataset> |
FeatureStore.getTrainingDatasets(@NonNull String name)
Deprecated.
|
Object |
FeatureView.getTrainingDatasetTag(Integer version,
String name)
Get a single tag value of the training dataset.
|
Map<String,Object> |
FeatureView.getTrainingDatasetTags(Integer version)
Get all tags of the training dataset.
|
List<org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>> |
FeatureView.getTrainTestSplit(Integer version)
Get training data created by `featureView.createTrainTestSplit` or `featureView.trainTestSplit`.
|
List<org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>> |
FeatureView.getTrainTestSplit(Integer version,
Map<String,String> readOptions)
Get training data created by `featureView.createTrainTestSplit` or `featureView.trainTestSplit`.
|
List<org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>> |
FeatureView.getTrainValidationTestSplit(Integer version)
Get training data created by `featureView.createTrainValidationTestSplit` or featureView.trainValidationTestSplit`.
|
List<org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>> |
FeatureView.getTrainValidationTestSplit(Integer version,
Map<String,String> readOptions)
Get training data created by `featureView.createTrainValidationTestSplit` or featureView.trainValidationTestSplit`.
|
void |
ExternalFeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData)
Incrementally insert data to the online storage of an external feature group.
|
void |
StreamFeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData)
Incrementally insert data to a stream feature group or overwrite all data contained in the feature group.
|
void |
FeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData)
Incrementally insert data to a feature group or overwrite all data contained in the feature group.
|
void |
ExternalFeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
boolean overwrite) |
void |
StreamFeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
boolean overwrite)
Incrementally insert data to a stream feature group or overwrite all data contained in the feature group.
|
void |
FeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
boolean overwrite)
Incrementally insert data to a feature group or overwrite all data contained in the feature group.
|
void |
ExternalFeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
boolean overwrite,
Map<String,String> writeOptions) |
void |
StreamFeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
boolean overwrite,
Map<String,String> writeOptions)
Incrementally insert data to a stream feature group or overwrite all data contained in the feature group.
|
void |
FeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
boolean overwrite,
Map<String,String> writeOptions)
Incrementally insert data to a feature group or overwrite all data contained in the feature group.
|
void |
ExternalFeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
boolean overwrite,
Map<String,String> writeOptions,
JobConfiguration jobConfiguration) |
void |
StreamFeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
boolean overwrite,
Map<String,String> writeOptions,
JobConfiguration jobConfiguration)
Incrementally insert data to a stream feature group or overwrite all data contained in the feature group.
|
void |
FeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
boolean overwrite,
Map<String,String> writeOptions,
JobConfiguration jobConfiguration) |
void |
ExternalFeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
HudiOperationType operation) |
void |
StreamFeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
HudiOperationType operation) |
void |
FeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
HudiOperationType operation)
Incrementally insert data to a feature group or overwrite all data contained in the feature group.
|
void |
ExternalFeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
JobConfiguration jobConfiguration) |
void |
StreamFeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
JobConfiguration jobConfiguration)
Incrementally insert data to a stream feature group or overwrite all data contained in the feature group.
|
void |
FeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
JobConfiguration jobConfiguration) |
void |
ExternalFeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Map<String,String> writeOptions)
Incrementally insert data to the online storage of an external feature group.
|
void |
StreamFeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Map<String,String> writeOptions)
Incrementally insert data to a stream feature group or overwrite all data contained in the feature group.
|
void |
FeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Map<String,String> writeOptions)
Incrementally insert data to a feature group or overwrite all data contained in the feature group.
|
void |
ExternalFeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Storage storage) |
void |
StreamFeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Storage storage) |
void |
FeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Storage storage)
Incrementally insert data to a feature group or overwrite all data contained in the feature group.
|
void |
ExternalFeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Storage storage,
boolean overwrite) |
void |
StreamFeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Storage storage,
boolean overwrite) |
void |
FeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Storage storage,
boolean overwrite)
Incrementally insert data to a feature group or overwrite all data contained in the feature group.
|
void |
ExternalFeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Storage storage,
boolean overwrite,
HudiOperationType operation,
Map<String,String> writeOptions) |
void |
StreamFeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Storage storage,
boolean overwrite,
HudiOperationType operation,
Map<String,String> writeOptions)
Incrementally insert data to a stream feature group or overwrite all data contained in the feature group.
|
void |
FeatureGroup.insert(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Storage storage,
boolean overwrite,
HudiOperationType operation,
Map<String,String> writeOptions)
Incrementally insert data to a feature group or overwrite all data contained in the feature group.
|
org.apache.spark.sql.streaming.StreamingQuery |
FeatureGroup.insertStream(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData)
Deprecated.
insertStream method is deprecated FeatureGroups. Full capability insertStream is available for StreamFeatureGroups.
|
org.apache.spark.sql.streaming.StreamingQuery |
FeatureGroup.insertStream(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Map<String,String> writeOptions)
Deprecated.
|
org.apache.spark.sql.streaming.StreamingQuery |
FeatureGroup.insertStream(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
String queryName)
Deprecated.
|
org.apache.spark.sql.streaming.StreamingQuery |
FeatureGroup.insertStream(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
String queryName,
Map<String,String> writeOptions) |
org.apache.spark.sql.streaming.StreamingQuery |
FeatureGroup.insertStream(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
String queryName,
String outputMode)
Deprecated.
|
org.apache.spark.sql.streaming.StreamingQuery |
FeatureGroup.insertStream(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
String queryName,
String outputMode,
boolean awaitTermination,
Long timeout)
Deprecated.
insertStream method is deprecated FeatureGroups. Full capability insertStream is available for StreamFeatureGroups.
|
org.apache.spark.sql.streaming.StreamingQuery |
FeatureGroup.insertStream(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
String queryName,
String outputMode,
boolean awaitTermination,
Long timeout,
String checkpointLocation)
Deprecated.
|
org.apache.spark.sql.streaming.StreamingQuery |
FeatureGroup.insertStream(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
String queryName,
String outputMode,
boolean awaitTermination,
Long timeout,
String checkpointLocation,
Map<String,String> writeOptions)
Deprecated.
|
Object |
FeatureGroup.insertStream(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
String queryName,
String outputMode,
boolean awaitTermination,
Long timeout,
String checkpointLocation,
Map<String,String> writeOptions,
JobConfiguration jobConfiguration) |
org.apache.spark.sql.streaming.StreamingQuery |
FeatureGroup.insertStream(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
String queryName,
String outputMode,
boolean awaitTermination,
String checkpointLocation)
Deprecated.
insertStream method is deprecated FeatureGroups. Full capability insertStream is available for StreamFeatureGroups.
|
org.apache.spark.sql.streaming.StreamingQuery |
FeatureGroup.insertStream(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
String queryName,
String outputMode,
String checkpointLocation)
Deprecated.
|
void |
FeatureView.purgeAllTrainingData()
Delete all training datasets in this feature view (data only).
|
void |
FeatureView.purgeTrainingData(Integer version)
Delete a training dataset (data only).
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
TrainingDataset.read()
Read the content of the training dataset.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
ExternalFeatureGroup.read() |
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
StreamFeatureGroup.read()
Reads the feature group from the offline storage as Spark DataFrame.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
FeatureGroup.read()
Reads the feature group from the offline storage as Spark DataFrame.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
ExternalFeatureGroup.read(boolean online) |
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
StreamFeatureGroup.read(boolean online)
Reads the stream feature group from the offline or online storage as Spark DataFrame.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
FeatureGroup.read(boolean online)
Reads the feature group from the offline or online storage as Spark DataFrame.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
ExternalFeatureGroup.read(boolean online,
Map<String,String> readOptions) |
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
StreamFeatureGroup.read(boolean online,
Map<String,String> readOptions)
Reads the stream feature group from the offline or online storage as Spark DataFrame.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
FeatureGroup.read(boolean online,
Map<String,String> readOptions)
Reads the feature group from the offline or online storage as Spark DataFrame.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
TrainingDataset.read(Map<String,String> readOptions)
Read the content of the training dataset.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
ExternalFeatureGroup.read(Map<String,String> readOptions) |
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
StreamFeatureGroup.read(Map<String,String> readOptions)
Reads the stream feature group from the offline storage as Spark DataFrame.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
FeatureGroup.read(Map<String,String> readOptions)
Reads the feature group from the offline storage as Spark DataFrame.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
TrainingDataset.read(String split)
Read all a single split from the training dataset.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
ExternalFeatureGroup.read(String wallclockTime) |
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
StreamFeatureGroup.read(String wallclockTime)
Reads stream Feature group into a dataframe at a specific point in time.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
FeatureGroup.read(String wallclockTime)
Reads Feature group into a dataframe at a specific point in time.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
TrainingDataset.read(String split,
Map<String,String> readOptions)
Read a single split from the training dataset.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
ExternalFeatureGroup.read(String wallclockTime,
Map<String,String> readOptions) |
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
StreamFeatureGroup.read(String wallclockTime,
Map<String,String> readOptions)
Reads stream Feature group into a dataframe at a specific point in time.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
FeatureGroup.read(String wallclockTime,
Map<String,String> readOptions)
Reads Feature group into a dataframe at a specific point in time.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
StreamFeatureGroup.readChanges(String wallclockStartTime,
String wallclockEndTime)
Deprecated.
`readChanges` method is deprecated. Use `asOf(wallclockEndTime, wallclockStartTime).read()` instead.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
FeatureGroup.readChanges(String wallclockStartTime,
String wallclockEndTime)
Deprecated.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
StreamFeatureGroup.readChanges(String wallclockStartTime,
String wallclockEndTime,
Map<String,String> readOptions)
Deprecated.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
FeatureGroup.readChanges(String wallclockStartTime,
String wallclockEndTime,
Map<String,String> readOptions)
Deprecated.
|
void |
FeatureView.recreateTrainingDataset(Integer version,
Map<String,String> writeOptions)
Recreate a training dataset.
|
void |
ExternalFeatureGroup.save() |
void |
FeatureGroup.save(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData)
Deprecated.
|
void |
StreamFeatureGroup.save(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Map<String,String> writeOptions)
Deprecated.
|
void |
FeatureGroup.save(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Map<String,String> writeOptions)
Deprecated.
|
void |
StreamFeatureGroup.save(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Map<String,String> writeOptions,
JobConfiguration jobConfiguration)
Deprecated.
|
void |
TrainingDataset.save(Query query) |
void |
TrainingDataset.save(Query query,
Map<String,String> writeOptions)
Create the training dataset based on the content of the feature store query.
|
void |
TrainingDataset.show(int numRows)
Show numRows from the training dataset (across all splits).
|
void |
ExternalFeatureGroup.show(int numRows) |
void |
StreamFeatureGroup.show(int numRows)
Show the first `n` rows of the feature group.
|
void |
FeatureGroup.show(int numRows)
Show the first `n` rows of the feature group.
|
void |
ExternalFeatureGroup.show(int numRows,
boolean online) |
void |
StreamFeatureGroup.show(int numRows,
boolean online)
Show the first `n` rows of the feature group.
|
void |
FeatureGroup.show(int numRows,
boolean online)
Show the first `n` rows of the feature group.
|
List<org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>> |
FeatureView.trainingData(String startTime,
String endTime,
String description)
Create the metadata for a training dataset and get the corresponding training data from the offline feature store.
|
List<org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>> |
FeatureView.trainingData(String startTime,
String endTime,
String description,
Long seed,
StatisticsConfig statisticsConfig,
Map<String,String> readOptions,
FilterLogic extraFilterLogic,
Filter extraFilter)
Create the metadata for a training dataset and get the corresponding training data from the offline feature store.
|
List<org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>> |
FeatureView.trainTestSplit(Float testSize,
String trainStart,
String trainEnd,
String testStart,
String testEnd,
String description)
Create the metadata for a training dataset and get the corresponding training data from the offline feature store.
|
List<org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>> |
FeatureView.trainTestSplit(Float testSize,
String trainStart,
String trainEnd,
String testStart,
String testEnd,
String description,
Long seed,
StatisticsConfig statisticsConfig,
Map<String,String> readOptions,
FilterLogic extraFilterLogic,
Filter extraFilter)
Create the metadata for a training dataset and get the corresponding training data from the offline feature store.
|
List<org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>> |
FeatureView.trainValidationTestSplit(Float validationSize,
Float testSize,
String trainStart,
String trainEnd,
String validationStart,
String validationEnd,
String testStart,
String testEnd,
String description)
Create the metadata for a training dataset and get the corresponding training data from the offline feature store.
|
List<org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>> |
FeatureView.trainValidationTestSplit(Float validationSize,
Float testSize,
String trainStart,
String trainEnd,
String validationStart,
String validationEnd,
String testStart,
String testEnd,
String description,
Long seed,
StatisticsConfig statisticsConfig,
Map<String,String> readOptions,
FilterLogic extraFilterLogic,
Filter extraFilter)
Create the metadata for a training dataset and get the corresponding training data from the offline feature store.
|
FeatureView |
FeatureView.update(FeatureView other)
Update the description of the feature view.
|
void |
ExternalFeatureGroup.updateFeatures(Feature feature) |
void |
StreamFeatureGroup.updateFeatures(Feature feature)
Update the metadata of feature.
|
void |
FeatureGroup.updateFeatures(Feature feature)
Update the metadata of feature.
|
void |
ExternalFeatureGroup.updateFeatures(List<Feature> features) |
void |
StreamFeatureGroup.updateFeatures(List<Feature> features)
Update the metadata of multiple features.
|
void |
FeatureGroup.updateFeatures(List<Feature> features)
Update the metadata of multiple features.
|
void |
TrainingDataset.updateStatisticsConfig()
Update the statistics configuration of the training dataset.
|
| Constructor and Description |
|---|
HopsworksConnection(String host,
int port,
String project,
software.amazon.awssdk.regions.Region region,
SecretStore secretStore,
boolean hostnameVerification,
String trustStorePath,
String certPath,
String apiKeyFilePath,
String apiKeyValue) |
TrainingDataset(@NonNull String name,
Integer version,
String description,
DataFormat dataFormat,
Boolean coalesce,
StorageConnector storageConnector,
String location,
List<Split> splits,
String trainSplit,
Long seed,
FeatureStore featureStore,
StatisticsConfig statisticsConfig,
List<String> label,
String eventStartTime,
String eventEndTime,
TrainingDatasetType trainingDatasetType,
Float validationSize,
Float testSize,
String trainStart,
String trainEnd,
String validationStart,
String validationEnd,
String testStart,
String testEnd,
Integer timeSplitSize,
FilterLogic extraFilterLogic,
Filter extraFilter) |
| Modifier and Type | Method and Description |
|---|---|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
Query.read() |
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
Query.read(boolean online) |
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
Query.read(boolean online,
Map<String,String> readOptions) |
void |
FsQuery.registerHudiFeatureGroups(Map<String,String> readOptions) |
void |
FsQuery.registerOnDemandFeatureGroups() |
void |
Query.show(boolean online,
int numRows) |
void |
Query.show(int numRows) |
| Modifier and Type | Method and Description |
|---|---|
void |
TrainingDatasetEngine.addTag(TrainingDataset trainingDataset,
String name,
Object value) |
<T extends FeatureGroupBase> |
FeatureGroupEngine.appendFeatures(FeatureGroupBase featureGroup,
List<Feature> features,
Class<T> fgClass) |
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
SparkEngine.castColumnType(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset,
List<TrainingDatasetFeature> features) |
String |
SparkEngine.checkpointDirPath(String queryName,
String onlineTopicName) |
FeatureGroupCommit |
FeatureGroupEngine.commitDelete(FeatureGroupBase featureGroupBase,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> genericDataset,
Map<String,String> writeOptions) |
Map<Long,Map<String,String>> |
FeatureGroupEngine.commitDetails(FeatureGroupBase featureGroupBase,
Integer limit) |
Map<Long,Map<String,String>> |
FeatureGroupEngine.commitDetailsByWallclockTime(FeatureGroupBase featureGroup,
String wallclockTime,
Integer limit) |
Statistics |
StatisticsEngine.computeStatistics(FeatureGroupBase featureGroup,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataFrame,
Long commitId) |
Statistics |
StatisticsEngine.computeStatistics(FeatureView featureView,
TrainingDataset trainingDataset,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataFrame) |
Statistics |
FeatureViewEngine.computeStatistics(FeatureView featureView,
TrainingDataset trainingDataset,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>[] datasets) |
Statistics |
StatisticsEngine.computeStatistics(TrainingDataset trainingDataset,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataFrame) |
String |
SparkEngine.constructCheckpointPath(FeatureGroupBase featureGroup,
String queryName,
String queryPrefix) |
TrainingDatasetBundle |
FeatureViewEngine.createTrainingDataset(FeatureView featureView,
TrainingDataset trainingDataset,
Map<String,String> userWriteOptions) |
void |
TrainingDatasetEngine.delete(TrainingDataset trainingDataset) |
void |
TrainingDatasetEngine.deleteTag(TrainingDataset trainingDataset,
String name) |
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
SparkEngine.encodeComplexFeatures(FeatureGroupBase featureGroupBase,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset)
Encodes all complex type features to binary using their avro type as schema.
|
Statistics |
StatisticsEngine.get(FeatureGroupBase featureGroup,
String commitTime) |
FeatureView |
FeatureViewEngine.get(FeatureStore featureStore,
String name,
Integer version) |
Statistics |
StatisticsEngine.get(TrainingDataset trainingDataset,
String commitTime) |
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
FeatureViewEngine.getBatchData(FeatureView featureView,
Date startTime,
Date endTime,
Map<String,String> readOptions,
Integer trainingDataVersion) |
Query |
FeatureViewEngine.getBatchQuery(FeatureView featureView,
Date startTime,
Date endTime,
Boolean withLabels,
Integer trainingDataVersion) |
String |
FeatureViewEngine.getBatchQueryString(FeatureView featureView,
Date startTime,
Date endTime,
Integer trainingDataVersion) |
ExternalFeatureGroup |
FeatureGroupEngine.getExternalFeatureGroup(FeatureStore featureStore,
String fgName,
Integer fgVersion) |
List<ExternalFeatureGroup> |
FeatureGroupEngine.getExternalFeatureGroups(FeatureStore featureStore,
String fgName) |
FeatureGroup |
FeatureGroupEngine.getFeatureGroup(FeatureStore featureStore,
String fgName,
Integer fgVersion) |
List<FeatureGroup> |
FeatureGroupEngine.getFeatureGroups(FeatureStore featureStore,
String fgName) |
Map<String,String> |
SparkEngine.getKafkaConfig(FeatureGroupBase featureGroup,
Map<String,String> writeOptions) |
Statistics |
StatisticsEngine.getLast(FeatureGroupBase featureGroup) |
Statistics |
StatisticsEngine.getLast(TrainingDataset trainingDataset) |
FeatureGroup |
FeatureGroupEngine.getOrCreateFeatureGroup(FeatureStore featureStore,
String name,
Integer version,
String description,
List<String> primaryKeys,
List<String> partitionKeys,
String hudiPrecombineKey,
boolean onlineEnabled,
TimeTravelFormat timeTravelFormat,
StatisticsConfig statisticsConfig,
String eventTime) |
FeatureView |
FeatureViewEngine.getOrCreateFeatureView(FeatureStore featureStore,
String name,
Integer version,
Query query,
String description,
List<String> labels) |
StreamFeatureGroup |
FeatureGroupEngine.getOrCreateStreamFeatureGroup(FeatureStore featureStore,
String name,
Integer version,
String description,
List<String> primaryKeys,
List<String> partitionKeys,
String hudiPrecombineKey,
boolean onlineEnabled,
StatisticsConfig statisticsConfig,
String eventTime) |
String |
TrainingDatasetEngine.getQuery(TrainingDataset trainingDataset,
Storage storage,
boolean withLabel,
boolean isHiveQuery) |
Statistics |
StatisticsEngine.getSplitStatistics(TrainingDataset trainingDataset) |
StreamFeatureGroup |
FeatureGroupEngine.getStreamFeatureGroup(FeatureStore featureStore,
String fgName,
Integer fgVersion) |
List<StreamFeatureGroup> |
FeatureGroupEngine.getStreamFeatureGroups(FeatureStore featureStore,
String fgName) |
Object |
TrainingDatasetEngine.getTag(TrainingDataset trainingDataset,
String name) |
Map<String,Object> |
TrainingDatasetEngine.getTags(TrainingDataset trainingDataset) |
TrainingDatasetBundle |
FeatureViewEngine.getTrainingDataset(FeatureView featureView,
Integer trainingDatasetVersion,
List<String> requestedSplits,
Map<String,String> userReadOptions) |
TrainingDatasetBundle |
FeatureViewEngine.getTrainingDataset(FeatureView featureView,
TrainingDataset trainingDataset,
List<String> requestedSplits,
Map<String,String> userReadOptions) |
TrainingDatasetBundle |
FeatureViewEngine.getTrainingDataset(FeatureView featureView,
TrainingDataset trainingDataset,
Map<String,String> userReadOptions) |
void |
FeatureGroupEngine.insert(ExternalFeatureGroup externalFeatureGroup,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Map<String,String> writeOptions) |
void |
FeatureGroupEngine.insert(FeatureGroup featureGroup,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
Storage storage,
HudiOperationType operation,
org.apache.spark.sql.SaveMode saveMode,
List<String> partitionKeys,
String hudiPrecombineKey,
Map<String,String> writeOptions) |
void |
FeatureGroupEngine.insert(StreamFeatureGroup streamFeatureGroup,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
org.apache.spark.sql.SaveMode saveMode,
List<String> partitionKeys,
String hudiPrecombineKey,
Map<String,String> writeOptions,
JobConfiguration jobConfiguration) |
org.apache.spark.sql.streaming.StreamingQuery |
FeatureGroupEngine.insertStream(FeatureGroup featureGroup,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
String queryName,
String outputMode,
boolean awaitTermination,
Long timeout,
String checkpointLocation,
List<String> partitionKeys,
String hudiPrecombineKey,
Map<String,String> writeOptions)
Deprecated.
|
List<Feature> |
SparkEngine.parseFeatureGroupSchema(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset,
TimeTravelFormat timeTravelFormat) |
List<TrainingDatasetFeature> |
TrainingDatasetUtils.parseTrainingDatasetSchema(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset) |
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
SparkEngine.read(StorageConnector storageConnector,
String dataFormat,
Map<String,String> readOptions,
String location) |
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
TrainingDatasetEngine.read(TrainingDataset trainingDataset,
String split,
Map<String,String> providedOptions) |
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
SparkEngine.readStream(StorageConnector storageConnector,
String dataFormat,
String messageFormat,
String schema,
Map<String,String> options,
boolean includeMetadata) |
void |
FeatureViewEngine.recreateTrainingDataset(FeatureView featureView,
Integer version,
Map<String,String> userWriteOptions) |
void |
SparkEngine.registerHudiTemporaryTable(FeatureGroupAlias featureGroupAlias,
Map<String,String> readOptions) |
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
SparkEngine.registerOnDemandTemporaryTable(ExternalFeatureGroup onDemandFeatureGroup,
String alias) |
Statistics |
StatisticsEngine.registerSplitStatistics(FeatureView featureView,
TrainingDataset trainingDataset,
Map<String,org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>> splitDatasets) |
Statistics |
StatisticsEngine.registerSplitStatistics(TrainingDataset trainingDataset) |
FeatureGroup |
FeatureGroupEngine.save(FeatureGroup featureGroup,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset,
List<String> partitionKeys,
String hudiPrecombineKey,
Map<String,String> writeOptions)
Create the metadata and write the data to the online/offline feature store.
|
StreamFeatureGroup |
FeatureGroupEngine.save(StreamFeatureGroup featureGroup,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset,
List<String> partitionKeys,
String hudiPrecombineKey,
Map<String,String> writeOptions,
JobConfiguration sparkJobConfiguration)
Create the metadata and write the data to the online/offline feature store.
|
TrainingDataset |
TrainingDatasetEngine.save(TrainingDataset trainingDataset,
Query query,
Map<String,String> userWriteOptions,
List<String> labels)
Make a REST call to Hopsworks to create the metadata and write the data on the File System.
|
void |
FeatureGroupEngine.saveDataframe(FeatureGroup featureGroup,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset,
Storage storage,
HudiOperationType operation,
Map<String,String> offlineWriteOptions,
Map<String,String> onlineWriteOptions,
Integer validationId) |
ExternalFeatureGroup |
FeatureGroupEngine.saveExternalFeatureGroup(ExternalFeatureGroup externalFeatureGroup) |
FeatureGroup |
FeatureGroupEngine.saveFeatureGroupMetaData(FeatureGroup featureGroup,
List<String> partitionKeys,
String hudiPrecombineKey,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
boolean saveEmpty) |
StreamFeatureGroup |
FeatureGroupEngine.saveFeatureGroupMetaData(StreamFeatureGroup featureGroup,
List<String> partitionKeys,
String hudiPrecombineKey,
Map<String,String> writeOptions,
JobConfiguration sparkJobConfiguration,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData) |
static void |
TrainingDatasetUtils.setLabelFeature(List<TrainingDatasetFeature> features,
List<String> labels) |
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>[] |
SparkEngine.splitDataset(TrainingDataset trainingDataset,
Query query,
Map<String,String> readOptions) |
void |
TrainingDatasetUtils.trainingDatasetSchemaMatch(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset,
List<TrainingDatasetFeature> features) |
FeatureView |
FeatureViewEngine.update(FeatureView featureView) |
void |
TrainingDatasetEngine.updateStatisticsConfig(TrainingDataset trainingDataset) |
void |
SparkEngine.validateSparkConfiguration() |
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>[] |
SparkEngine.write(TrainingDataset trainingDataset,
Query query,
Map<String,String> queryReadOptions,
Map<String,String> writeOptions,
org.apache.spark.sql.SaveMode saveMode)
Setup Spark to write the data on the File System.
|
void |
SparkEngine.writeEmptyDataframe(FeatureGroupBase featureGroup) |
void |
SparkEngine.writeOfflineDataframe(FeatureGroupBase featureGroup,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset,
HudiOperationType operation,
Map<String,String> writeOptions,
Integer validationId) |
void |
SparkEngine.writeOnlineDataframe(FeatureGroupBase featureGroupBase,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset,
String onlineTopicName,
Map<String,String> writeOptions)
Writes feature group dataframe to kafka for online-fs ingestion.
|
<S> org.apache.spark.sql.streaming.StreamingQuery |
SparkEngine.writeStreamDataframe(FeatureGroupBase featureGroupBase,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset,
String queryName,
String outputMode,
boolean awaitTermination,
Long timeout,
String checkpointLocation,
Map<String,String> writeOptions) |
void |
FeatureViewEngine.writeTrainingDataset(FeatureView featureView,
TrainingDataset trainingDataset,
Map<String,String> userWriteOptions) |
| Modifier and Type | Method and Description |
|---|---|
FeatureGroupCommit |
HudiEngine.deleteRecord(org.apache.spark.sql.SparkSession sparkSession,
FeatureGroupBase featureGroup,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> deleteDF,
Map<String,String> writeOptions) |
void |
HudiEngine.reconcileHudiSchema(org.apache.spark.sql.SparkSession sparkSession,
FeatureGroupAlias featureGroupAlias,
Map<String,String> hudiArgs) |
void |
HudiEngine.saveHudiFeatureGroup(org.apache.spark.sql.SparkSession sparkSession,
FeatureGroupBase featureGroup,
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset,
HudiOperationType operation,
Map<String,String> writeOptions,
Integer validationId) |
| Modifier and Type | Method and Description |
|---|---|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
StorageConnectorUtils.read(StorageConnector.AdlsConnector connector,
String dataFormat,
Map<String,String> options,
String path)
Reads path into a spark dataframe using the AdlsConnector.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
StorageConnectorUtils.read(StorageConnector.BigqueryConnector connector,
String query,
Map<String,String> options,
String path)
Reads a query or a path into a spark dataframe using the sBigqueryConnector.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
StorageConnectorUtils.read(StorageConnector.GcsConnector connector,
String dataFormat,
Map<String,String> options,
String path)
Reads a path into a spark dataframe using the GcsConnector.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
StorageConnectorUtils.read(StorageConnector.HopsFsConnector connector,
String dataFormat,
Map<String,String> options,
String path)
Reads path into a spark dataframe using the HopsFsConnector.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
StorageConnectorUtils.read(StorageConnector.JdbcConnector connector,
String query)
Reads query into a spark dataframe using the JdbcConnector.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
StorageConnectorUtils.read(StorageConnector.RedshiftConnector connector,
String query)
Reads query into a spark dataframe using the RedshiftConnector.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
StorageConnectorUtils.read(StorageConnector.S3Connector connector,
String dataFormat,
Map<String,String> options,
String path)
Reads path into a spark dataframe using the S3Connector.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
StorageConnectorUtils.read(StorageConnector.SnowflakeConnector connector,
String query)
Reads query into a spark dataframe using the SnowflakeConnector.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
StorageConnectorUtils.read(StorageConnector connector,
String query,
String dataFormat,
Map<String,String> options,
String path)
Reads a query or a path into a spark dataframe using the storage connector.
|
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> |
StorageConnectorUtils.readStream(StorageConnector.KafkaConnector connector,
String topic,
boolean topicPattern,
String messageFormat,
String schema,
Map<String,String> options,
boolean includeMetadata)
Reads stream into a spark dataframe using the kafka storage connector.
|
Copyright © 2023. All rights reserved.