com.logicalclocks.hsfs.spark.engine.hudi

Class HudiEngine

java.lang.Object

com.logicalclocks.hsfs.spark.engine.hudi.HudiEngine

public class HudiEngine
extends Object

Field Summary

Fields

Modifier and Type	Field and Description
protected static String	COMMIT_METADATA_KEYPREFIX_OPT_KEY
protected static String	DEFAULT_HIVE_PARTITION_EXTRACTOR_CLASS_OPT_VAL
protected static String	DELTA_SOURCE_ORDERING_FIELD_OPT_KEY
protected static String	DELTA_STREAMER_TRANSFORMER
protected static String	DELTASTREAMER_CHECKPOINT_KEY
protected static String	FEATURE_GROUP_COMPLEX_FEATURES
protected static String	FEATURE_GROUP_ENCODED_SCHEMA
protected static String	FEATURE_GROUP_KAFKA_OFFSET_RESET
protected static String	FEATURE_GROUP_NAME
protected static String	FEATURE_GROUP_SCHEMA
protected static String	FEATURE_GROUP_VERSION
protected static String	FEATURE_STORE_NAME
protected static String	FUNCTION_TYPE
protected static String	HIVE_AUTO_CREATE_DATABASE_OPT_KEY
protected static String	HIVE_AUTO_CREATE_DATABASE_OPT_VAL
protected static String	HIVE_NON_PARTITION_EXTRACTOR_CLASS_OPT_VAL
protected static String	HIVE_PARTITION_EXTRACTOR_CLASS_OPT_KEY
protected static String	HUDI_BASE_PATH
protected static String	HUDI_BEGIN_INSTANTTIME_OPT_KEY
protected static String	HUDI_COMPLEX_KEY_GENERATOR_OPT_VAL
protected static String	HUDI_COPY_ON_WRITE
protected static String	HUDI_END_INSTANTTIME_OPT_KEY
protected static String	HUDI_HIVE_SYNC_DB
protected static String	HUDI_HIVE_SYNC_ENABLE
protected static String	HUDI_HIVE_SYNC_MODE
protected static String	HUDI_HIVE_SYNC_MODE_VAL
protected static String	HUDI_HIVE_SYNC_PARTITION_FIELDS
protected static String	HUDI_HIVE_SYNC_TABLE
protected static String	HUDI_KAFKA_TOPIC
protected static String	HUDI_KEY_GENERATOR_OPT_KEY
protected static String	HUDI_PARTITION_FIELD
protected static String	HUDI_PRECOMBINE_FIELD
protected static String	HUDI_QUERY_TIME_TRAVEL_AS_OF_INSTANT
protected static String	HUDI_QUERY_TYPE_INCREMENTAL_OPT_VAL
protected static String	HUDI_QUERY_TYPE_OPT_KEY
protected static String	HUDI_QUERY_TYPE_SNAPSHOT_OPT_VAL
protected static String	HUDI_RECORD_KEY
static String	HUDI_SPARK_FORMAT
protected static String	HUDI_TABLE_NAME
protected static String	HUDI_TABLE_OPERATION
protected static String	HUDI_TABLE_STORAGE_TYPE
protected static String	HUDI_WRITE_INSERT_DROP_DUPLICATES
protected static String	INITIAL_CHECKPOINT_STRING
protected static String	KAFKA_SOURCE
protected static String	MIN_SYNC_INTERVAL_SECONDS
protected static String	PAYLOAD_CLASS_OPT_KEY
protected static String	PAYLOAD_CLASS_OPT_VAL
protected static String	PROJECT_ID
protected static String	SCHEMA_PROVIDER
protected static String	SPARK_MASTER
protected static String	STREAMING_QUERY

Constructor Summary

Constructors

Constructor and Description
HudiEngine()

Method Summary

Modifier and Type	Method and Description
FeatureGroupCommit	deleteRecord(org.apache.spark.sql.SparkSession sparkSession, FeatureGroupBase featureGroup, org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> deleteDF, Map<String,String> writeOptions)
void	reconcileHudiSchema(org.apache.spark.sql.SparkSession sparkSession, FeatureGroupAlias featureGroupAlias, Map<String,String> hudiArgs)
void	registerTemporaryTable(org.apache.spark.sql.SparkSession sparkSession, FeatureGroupAlias featureGroupAlias, Map<String,String> readOptions)
void	saveHudiFeatureGroup(org.apache.spark.sql.SparkSession sparkSession, FeatureGroupBase featureGroup, org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset, HudiOperationType operation, Map<String,String> writeOptions, Integer validationId)
Map<String,String>	setupHudiReadOpts(Long startTimestamp, Long endTimestamp, Map<String,String> readOptions)
boolean	sparkSchemasMatch(String[] schema1, String[] schema2)
void	streamToHoodieTable(org.apache.spark.sql.SparkSession sparkSession, StreamFeatureGroup streamFeatureGroup, Map<String,String> writeOptions)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

HUDI_SPARK_FORMAT

public static final String HUDI_SPARK_FORMAT

See Also:

Constant Field Values

HUDI_BASE_PATH

protected static final String HUDI_BASE_PATH

See Also:

Constant Field Values

HUDI_TABLE_NAME

protected static final String HUDI_TABLE_NAME

See Also:

Constant Field Values

HUDI_TABLE_STORAGE_TYPE

protected static final String HUDI_TABLE_STORAGE_TYPE

See Also:

Constant Field Values

HUDI_TABLE_OPERATION

protected static final String HUDI_TABLE_OPERATION

See Also:

Constant Field Values

HUDI_KEY_GENERATOR_OPT_KEY

protected static final String HUDI_KEY_GENERATOR_OPT_KEY

See Also:

Constant Field Values

HUDI_COMPLEX_KEY_GENERATOR_OPT_VAL

protected static final String HUDI_COMPLEX_KEY_GENERATOR_OPT_VAL

See Also:

Constant Field Values

HUDI_RECORD_KEY

protected static final String HUDI_RECORD_KEY

See Also:

Constant Field Values

HUDI_PARTITION_FIELD

protected static final String HUDI_PARTITION_FIELD

See Also:

Constant Field Values

HUDI_PRECOMBINE_FIELD

protected static final String HUDI_PRECOMBINE_FIELD

See Also:

Constant Field Values

HUDI_HIVE_SYNC_ENABLE

protected static final String HUDI_HIVE_SYNC_ENABLE

See Also:

Constant Field Values

HUDI_HIVE_SYNC_TABLE

protected static final String HUDI_HIVE_SYNC_TABLE

See Also:

Constant Field Values

HUDI_HIVE_SYNC_DB

protected static final String HUDI_HIVE_SYNC_DB

See Also:

Constant Field Values

HUDI_HIVE_SYNC_MODE

protected static final String HUDI_HIVE_SYNC_MODE

See Also:

Constant Field Values

HUDI_HIVE_SYNC_MODE_VAL

protected static final String HUDI_HIVE_SYNC_MODE_VAL

See Also:

Constant Field Values

HUDI_HIVE_SYNC_PARTITION_FIELDS

protected static final String HUDI_HIVE_SYNC_PARTITION_FIELDS

See Also:

Constant Field Values

HIVE_PARTITION_EXTRACTOR_CLASS_OPT_KEY

protected static final String HIVE_PARTITION_EXTRACTOR_CLASS_OPT_KEY

See Also:

Constant Field Values

DEFAULT_HIVE_PARTITION_EXTRACTOR_CLASS_OPT_VAL

protected static final String DEFAULT_HIVE_PARTITION_EXTRACTOR_CLASS_OPT_VAL

See Also:

Constant Field Values

HIVE_NON_PARTITION_EXTRACTOR_CLASS_OPT_VAL

protected static final String HIVE_NON_PARTITION_EXTRACTOR_CLASS_OPT_VAL

See Also:

Constant Field Values

HIVE_AUTO_CREATE_DATABASE_OPT_KEY

protected static final String HIVE_AUTO_CREATE_DATABASE_OPT_KEY

See Also:

Constant Field Values

HIVE_AUTO_CREATE_DATABASE_OPT_VAL

protected static final String HIVE_AUTO_CREATE_DATABASE_OPT_VAL

See Also:

Constant Field Values

HUDI_COPY_ON_WRITE

protected static final String HUDI_COPY_ON_WRITE

See Also:

Constant Field Values

HUDI_QUERY_TYPE_OPT_KEY

protected static final String HUDI_QUERY_TYPE_OPT_KEY

See Also:

Constant Field Values

HUDI_QUERY_TYPE_INCREMENTAL_OPT_VAL

protected static final String HUDI_QUERY_TYPE_INCREMENTAL_OPT_VAL

See Also:

Constant Field Values

HUDI_QUERY_TYPE_SNAPSHOT_OPT_VAL

protected static final String HUDI_QUERY_TYPE_SNAPSHOT_OPT_VAL

See Also:

Constant Field Values

HUDI_QUERY_TIME_TRAVEL_AS_OF_INSTANT

protected static final String HUDI_QUERY_TIME_TRAVEL_AS_OF_INSTANT

See Also:

Constant Field Values

HUDI_BEGIN_INSTANTTIME_OPT_KEY

protected static final String HUDI_BEGIN_INSTANTTIME_OPT_KEY

See Also:

Constant Field Values

HUDI_END_INSTANTTIME_OPT_KEY

protected static final String HUDI_END_INSTANTTIME_OPT_KEY

See Also:

Constant Field Values

HUDI_WRITE_INSERT_DROP_DUPLICATES

protected static final String HUDI_WRITE_INSERT_DROP_DUPLICATES

See Also:

Constant Field Values

PAYLOAD_CLASS_OPT_KEY

protected static final String PAYLOAD_CLASS_OPT_KEY

See Also:

Constant Field Values

PAYLOAD_CLASS_OPT_VAL

protected static final String PAYLOAD_CLASS_OPT_VAL

See Also:

Constant Field Values

HUDI_KAFKA_TOPIC

protected static final String HUDI_KAFKA_TOPIC

See Also:

Constant Field Values

COMMIT_METADATA_KEYPREFIX_OPT_KEY

protected static final String COMMIT_METADATA_KEYPREFIX_OPT_KEY

See Also:

Constant Field Values

DELTASTREAMER_CHECKPOINT_KEY

protected static final String DELTASTREAMER_CHECKPOINT_KEY

See Also:

Constant Field Values

INITIAL_CHECKPOINT_STRING

protected static final String INITIAL_CHECKPOINT_STRING

See Also:

Constant Field Values

FEATURE_GROUP_SCHEMA

protected static final String FEATURE_GROUP_SCHEMA

See Also:

Constant Field Values

FEATURE_GROUP_ENCODED_SCHEMA

protected static final String FEATURE_GROUP_ENCODED_SCHEMA

See Also:

Constant Field Values

FEATURE_GROUP_COMPLEX_FEATURES

protected static final String FEATURE_GROUP_COMPLEX_FEATURES

See Also:

Constant Field Values

KAFKA_SOURCE

protected static final String KAFKA_SOURCE

See Also:

Constant Field Values

SCHEMA_PROVIDER

protected static final String SCHEMA_PROVIDER

See Also:

Constant Field Values

DELTA_STREAMER_TRANSFORMER

protected static final String DELTA_STREAMER_TRANSFORMER

See Also:

Constant Field Values

DELTA_SOURCE_ORDERING_FIELD_OPT_KEY

protected static final String DELTA_SOURCE_ORDERING_FIELD_OPT_KEY

See Also:

Constant Field Values

MIN_SYNC_INTERVAL_SECONDS

protected static final String MIN_SYNC_INTERVAL_SECONDS

See Also:

Constant Field Values

SPARK_MASTER

protected static final String SPARK_MASTER

See Also:

Constant Field Values

PROJECT_ID

protected static final String PROJECT_ID

See Also:

Constant Field Values

FEATURE_STORE_NAME

protected static final String FEATURE_STORE_NAME

See Also:

Constant Field Values

FEATURE_GROUP_NAME

protected static final String FEATURE_GROUP_NAME

See Also:

Constant Field Values

FEATURE_GROUP_VERSION

protected static final String FEATURE_GROUP_VERSION

See Also:

Constant Field Values

FEATURE_GROUP_KAFKA_OFFSET_RESET

protected static final String FEATURE_GROUP_KAFKA_OFFSET_RESET

See Also:

Constant Field Values

FUNCTION_TYPE

protected static final String FUNCTION_TYPE

See Also:

Constant Field Values

STREAMING_QUERY

protected static final String STREAMING_QUERY

See Also:

Constant Field Values

Constructor Detail

HudiEngine

public HudiEngine()

Method Detail

saveHudiFeatureGroup

public void saveHudiFeatureGroup(org.apache.spark.sql.SparkSession sparkSession,
                                 FeatureGroupBase featureGroup,
                                 org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset,
                                 HudiOperationType operation,
                                 Map<String,String> writeOptions,
                                 Integer validationId)
                          throws IOException,
                                 FeatureStoreException,
                                 ParseException

Throws:

IOException

FeatureStoreException

ParseException

deleteRecord

public FeatureGroupCommit deleteRecord(org.apache.spark.sql.SparkSession sparkSession,
                                       FeatureGroupBase featureGroup,
                                       org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> deleteDF,
                                       Map<String,String> writeOptions)
                                throws IOException,
                                       FeatureStoreException,
                                       ParseException

Throws:

IOException

FeatureStoreException

ParseException

registerTemporaryTable

public void registerTemporaryTable(org.apache.spark.sql.SparkSession sparkSession,
                                   FeatureGroupAlias featureGroupAlias,
                                   Map<String,String> readOptions)

setupHudiReadOpts

public Map<String,String> setupHudiReadOpts(Long startTimestamp,
                                            Long endTimestamp,
                                            Map<String,String> readOptions)

reconcileHudiSchema

public void reconcileHudiSchema(org.apache.spark.sql.SparkSession sparkSession,
                                FeatureGroupAlias featureGroupAlias,
                                Map<String,String> hudiArgs)
                         throws FeatureStoreException

Throws:

FeatureStoreException

sparkSchemasMatch

public boolean sparkSchemasMatch(String[] schema1,
                                 String[] schema2)

streamToHoodieTable

public void streamToHoodieTable(org.apache.spark.sql.SparkSession sparkSession,
                                StreamFeatureGroup streamFeatureGroup,
                                Map<String,String> writeOptions)
                         throws Exception

Throws:

Exception