Package com.logicalclocks.hsfs.spark.engine.hudi

Class HudiEngine

java.lang.Object

com.logicalclocks.hsfs.spark.engine.hudi.HudiEngine

public class HudiEngine
extends Object

Field Summary

Fields

Modifier and Type	Field	Description
protected static final String	COMMIT_METADATA_KEYPREFIX_OPT_KEY
protected static final String	DEFAULT_HIVE_PARTITION_EXTRACTOR_CLASS_OPT_VAL
protected static final String	DELTA_SOURCE_ORDERING_FIELD_OPT_KEY
protected static final String	DELTA_STREAMER_TRANSFORMER
protected static final String	DELTASTREAMER_CHECKPOINT_KEY
protected static final String	FEATURE_GROUP_COMPLEX_FEATURES
protected static final String	FEATURE_GROUP_ENCODED_SCHEMA
protected static final String	FEATURE_GROUP_ID
protected static final String	FEATURE_GROUP_NAME
protected static final String	FEATURE_GROUP_SCHEMA
protected static final String	FEATURE_GROUP_VERSION
protected static final String	FEATURE_STORE_NAME
protected static final String	FUNCTION_TYPE
protected static final String	HIVE_AUTO_CREATE_DATABASE_OPT_KEY
protected static final String	HIVE_AUTO_CREATE_DATABASE_OPT_VAL
protected static final String	HIVE_NON_PARTITION_EXTRACTOR_CLASS_OPT_VAL
protected static final String	HIVE_PARTITION_EXTRACTOR_CLASS_OPT_KEY
protected static final String	HUDI_BASE_PATH
protected static final String	HUDI_BEGIN_INSTANTTIME_OPT_KEY
protected static final String	HUDI_COMPLEX_KEY_GENERATOR_OPT_VAL
protected static final String	HUDI_COPY_ON_WRITE
protected static final String	HUDI_END_INSTANTTIME_OPT_KEY
protected static final String	HUDI_HIVE_SYNC_DB
protected static final String	HUDI_HIVE_SYNC_ENABLE
protected static final String	HUDI_HIVE_SYNC_MODE
protected static final String	HUDI_HIVE_SYNC_MODE_VAL
protected static final String	HUDI_HIVE_SYNC_PARTITION_FIELDS
protected static final String	HUDI_HIVE_SYNC_TABLE
protected static final String	HUDI_KAFKA_TOPIC
protected static final String	HUDI_KEY_GENERATOR_OPT_KEY
protected static final String	HUDI_METADATA_ENABLE
protected static final String	HUDI_PARTITION_FIELD
protected static final String	HUDI_QUERY_TIME_TRAVEL_AS_OF_INSTANT
protected static final String	HUDI_QUERY_TYPE_INCREMENTAL_OPT_VAL
protected static final String	HUDI_QUERY_TYPE_OPT_KEY
protected static final String	HUDI_QUERY_TYPE_SNAPSHOT_OPT_VAL
static final String	HUDI_SPARK_FORMAT
protected static final String	HUDI_TABLE_BASE_FILE_FORMAT
protected static final String	HUDI_TABLE_KEY_GENERATOR_CLASS
protected static final String	HUDI_TABLE_METADATA_PARTITIONS
protected static final String	HUDI_TABLE_NAME
protected static final String	HUDI_TABLE_OPERATION
protected static final String	HUDI_TABLE_PARTITION_KEY_FIELDS
protected static final String	HUDI_TABLE_PRECOMBINE_FIELD
protected static final String	HUDI_TABLE_RECORD_KEY_FIELD
protected static final String	HUDI_TABLE_STORAGE_TYPE
protected static final String	HUDI_TABLE_TYPE
protected static final String	HUDI_TABLE_VERSION
protected static final String	HUDI_WRITE_INSERT_DROP_DUPLICATES
protected static final String	HUDI_WRITE_PRECOMBINE_FIELD
protected static final String	HUDI_WRITE_RECORD_KEY
protected static final String	INITIAL_CHECKPOINT_STRING
protected static HudiEngine	INSTANCE
protected static final String	KAFKA_SOURCE
protected static final String	MIN_SYNC_INTERVAL_SECONDS
protected static final String	PROJECT_ID
protected static final String	SCHEMA_PROVIDER
protected static final String	SPARK_MASTER
protected static final String	STREAMER_CHECKPOINT_KEY_V2
protected static final String	STREAMING_QUERY
protected static final String	SUBJECT_ID

Method Summary

Modifier and Type	Method	Description
FeatureGroupCommit	deleteRecord(org.apache.spark.sql.SparkSession sparkSession, FeatureGroupBase featureGroup, org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> deleteDF, Map<String,String> writeOptions)
Long	getHudiTableSize(org.apache.spark.api.java.JavaSparkContext jsc, String basePath)
static HudiEngine	getInstance()
void	reconcileHudiSchema(org.apache.spark.sql.SparkSession sparkSession, FeatureGroupAlias featureGroupAlias, Map<String,String> hudiArgs)
void	saveHudiFeatureGroup(org.apache.spark.sql.SparkSession sparkSession, FeatureGroupBase featureGroup, org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset, HudiOperationType operation, Map<String,String> writeOptions, Integer validationId)
Map<String,String>	setupHudiReadOpts(Long startTimestamp, Long endTimestamp, Map<String,String> readOptions)
boolean	sparkSchemasMatch(String[] schema1, String[] schema2)
void	streamToHoodieTable(org.apache.spark.sql.SparkSession sparkSession, StreamFeatureGroup streamFeatureGroup, Map<String,String> writeOptions)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

HUDI_SPARK_FORMAT

public static final String HUDI_SPARK_FORMAT

See Also:

• Constant Field Values

HUDI_TABLE_VERSION

protected static final String HUDI_TABLE_VERSION

See Also:

• Constant Field Values

HUDI_BASE_PATH

protected static final String HUDI_BASE_PATH

See Also:

• Constant Field Values

HUDI_TABLE_NAME

protected static final String HUDI_TABLE_NAME

See Also:

• Constant Field Values

HUDI_TABLE_TYPE

protected static final String HUDI_TABLE_TYPE

See Also:

• Constant Field Values

HUDI_TABLE_STORAGE_TYPE

protected static final String HUDI_TABLE_STORAGE_TYPE

See Also:

• Constant Field Values

HUDI_TABLE_OPERATION

protected static final String HUDI_TABLE_OPERATION

See Also:

• Constant Field Values

HUDI_METADATA_ENABLE

protected static final String HUDI_METADATA_ENABLE

See Also:

• Constant Field Values

HUDI_TABLE_RECORD_KEY_FIELD

protected static final String HUDI_TABLE_RECORD_KEY_FIELD

See Also:

• Constant Field Values

HUDI_TABLE_PARTITION_KEY_FIELDS

protected static final String HUDI_TABLE_PARTITION_KEY_FIELDS

See Also:

• Constant Field Values

HUDI_TABLE_KEY_GENERATOR_CLASS

protected static final String HUDI_TABLE_KEY_GENERATOR_CLASS

See Also:

• Constant Field Values

HUDI_TABLE_PRECOMBINE_FIELD

protected static final String HUDI_TABLE_PRECOMBINE_FIELD

See Also:

• Constant Field Values

HUDI_TABLE_BASE_FILE_FORMAT

protected static final String HUDI_TABLE_BASE_FILE_FORMAT

See Also:

• Constant Field Values

HUDI_TABLE_METADATA_PARTITIONS

protected static final String HUDI_TABLE_METADATA_PARTITIONS

See Also:

• Constant Field Values

HUDI_KEY_GENERATOR_OPT_KEY

protected static final String HUDI_KEY_GENERATOR_OPT_KEY

See Also:

• Constant Field Values

HUDI_COMPLEX_KEY_GENERATOR_OPT_VAL

protected static final String HUDI_COMPLEX_KEY_GENERATOR_OPT_VAL

See Also:

• Constant Field Values

HUDI_WRITE_RECORD_KEY

protected static final String HUDI_WRITE_RECORD_KEY

See Also:

• Constant Field Values

HUDI_PARTITION_FIELD

protected static final String HUDI_PARTITION_FIELD

See Also:

• Constant Field Values

HUDI_WRITE_PRECOMBINE_FIELD

protected static final String HUDI_WRITE_PRECOMBINE_FIELD

See Also:

• Constant Field Values

HUDI_HIVE_SYNC_ENABLE

protected static final String HUDI_HIVE_SYNC_ENABLE

See Also:

• Constant Field Values

HUDI_HIVE_SYNC_TABLE

protected static final String HUDI_HIVE_SYNC_TABLE

See Also:

• Constant Field Values

HUDI_HIVE_SYNC_DB

protected static final String HUDI_HIVE_SYNC_DB

See Also:

• Constant Field Values

HUDI_HIVE_SYNC_MODE

protected static final String HUDI_HIVE_SYNC_MODE

See Also:

• Constant Field Values

HUDI_HIVE_SYNC_MODE_VAL

protected static final String HUDI_HIVE_SYNC_MODE_VAL

See Also:

• Constant Field Values

HUDI_HIVE_SYNC_PARTITION_FIELDS

protected static final String HUDI_HIVE_SYNC_PARTITION_FIELDS

See Also:

• Constant Field Values

HIVE_PARTITION_EXTRACTOR_CLASS_OPT_KEY

protected static final String HIVE_PARTITION_EXTRACTOR_CLASS_OPT_KEY

See Also:

• Constant Field Values

DEFAULT_HIVE_PARTITION_EXTRACTOR_CLASS_OPT_VAL

protected static final String DEFAULT_HIVE_PARTITION_EXTRACTOR_CLASS_OPT_VAL

See Also:

• Constant Field Values

HIVE_NON_PARTITION_EXTRACTOR_CLASS_OPT_VAL

protected static final String HIVE_NON_PARTITION_EXTRACTOR_CLASS_OPT_VAL

See Also:

• Constant Field Values

HIVE_AUTO_CREATE_DATABASE_OPT_KEY

protected static final String HIVE_AUTO_CREATE_DATABASE_OPT_KEY

See Also:

• Constant Field Values

HIVE_AUTO_CREATE_DATABASE_OPT_VAL

protected static final String HIVE_AUTO_CREATE_DATABASE_OPT_VAL

See Also:

• Constant Field Values

HUDI_COPY_ON_WRITE

protected static final String HUDI_COPY_ON_WRITE

See Also:

• Constant Field Values

HUDI_QUERY_TYPE_OPT_KEY

protected static final String HUDI_QUERY_TYPE_OPT_KEY

See Also:

• Constant Field Values

HUDI_QUERY_TYPE_INCREMENTAL_OPT_VAL

protected static final String HUDI_QUERY_TYPE_INCREMENTAL_OPT_VAL

See Also:

• Constant Field Values

HUDI_QUERY_TYPE_SNAPSHOT_OPT_VAL

protected static final String HUDI_QUERY_TYPE_SNAPSHOT_OPT_VAL

See Also:

• Constant Field Values

HUDI_QUERY_TIME_TRAVEL_AS_OF_INSTANT

protected static final String HUDI_QUERY_TIME_TRAVEL_AS_OF_INSTANT

See Also:

• Constant Field Values

HUDI_BEGIN_INSTANTTIME_OPT_KEY

protected static final String HUDI_BEGIN_INSTANTTIME_OPT_KEY

See Also:

• Constant Field Values

HUDI_END_INSTANTTIME_OPT_KEY

protected static final String HUDI_END_INSTANTTIME_OPT_KEY

See Also:

• Constant Field Values

HUDI_WRITE_INSERT_DROP_DUPLICATES

protected static final String HUDI_WRITE_INSERT_DROP_DUPLICATES

See Also:

• Constant Field Values

HUDI_KAFKA_TOPIC

protected static final String HUDI_KAFKA_TOPIC

See Also:

• Constant Field Values

COMMIT_METADATA_KEYPREFIX_OPT_KEY

protected static final String COMMIT_METADATA_KEYPREFIX_OPT_KEY

See Also:

• Constant Field Values

STREAMER_CHECKPOINT_KEY_V2

protected static final String STREAMER_CHECKPOINT_KEY_V2

See Also:

• Constant Field Values

DELTASTREAMER_CHECKPOINT_KEY

protected static final String DELTASTREAMER_CHECKPOINT_KEY

See Also:

• Constant Field Values

INITIAL_CHECKPOINT_STRING

protected static final String INITIAL_CHECKPOINT_STRING

See Also:

• Constant Field Values

FEATURE_GROUP_SCHEMA

protected static final String FEATURE_GROUP_SCHEMA

See Also:

• Constant Field Values

FEATURE_GROUP_ENCODED_SCHEMA

protected static final String FEATURE_GROUP_ENCODED_SCHEMA

See Also:

• Constant Field Values

FEATURE_GROUP_COMPLEX_FEATURES

protected static final String FEATURE_GROUP_COMPLEX_FEATURES

See Also:

• Constant Field Values

KAFKA_SOURCE

protected static final String KAFKA_SOURCE

See Also:

• Constant Field Values

SCHEMA_PROVIDER

protected static final String SCHEMA_PROVIDER

See Also:

• Constant Field Values

DELTA_STREAMER_TRANSFORMER

protected static final String DELTA_STREAMER_TRANSFORMER

See Also:

• Constant Field Values

DELTA_SOURCE_ORDERING_FIELD_OPT_KEY

protected static final String DELTA_SOURCE_ORDERING_FIELD_OPT_KEY

See Also:

• Constant Field Values

MIN_SYNC_INTERVAL_SECONDS

protected static final String MIN_SYNC_INTERVAL_SECONDS

See Also:

• Constant Field Values

SPARK_MASTER

protected static final String SPARK_MASTER

See Also:

• Constant Field Values

PROJECT_ID

protected static final String PROJECT_ID

See Also:

• Constant Field Values

FEATURE_STORE_NAME

protected static final String FEATURE_STORE_NAME

See Also:

• Constant Field Values

SUBJECT_ID

protected static final String SUBJECT_ID

See Also:

• Constant Field Values

FEATURE_GROUP_ID

protected static final String FEATURE_GROUP_ID

See Also:

• Constant Field Values

FEATURE_GROUP_NAME

protected static final String FEATURE_GROUP_NAME

See Also:

• Constant Field Values

FEATURE_GROUP_VERSION

protected static final String FEATURE_GROUP_VERSION

See Also:

• Constant Field Values

FUNCTION_TYPE

protected static final String FUNCTION_TYPE

See Also:

• Constant Field Values

STREAMING_QUERY

protected static final String STREAMING_QUERY

See Also:

• Constant Field Values

INSTANCE

protected static HudiEngine INSTANCE

Method Details

getInstance

public static HudiEngine getInstance()

saveHudiFeatureGroup

public void saveHudiFeatureGroup(org.apache.spark.sql.SparkSession sparkSession,
 FeatureGroupBase featureGroup,
 org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset,
 HudiOperationType operation,
 Map<String,String> writeOptions,
 Integer validationId)
                          throws IOException,
FeatureStoreException,
ParseException

Throws:

IOException

FeatureStoreException

ParseException

deleteRecord

public FeatureGroupCommit deleteRecord(org.apache.spark.sql.SparkSession sparkSession,
 FeatureGroupBase featureGroup,
 org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> deleteDF,
 Map<String,String> writeOptions)
                                throws IOException,
FeatureStoreException,
ParseException

Throws:

IOException

FeatureStoreException

ParseException

setupHudiReadOpts

public Map<String,String> setupHudiReadOpts(Long startTimestamp,
 Long endTimestamp,
 Map<String,String> readOptions)

reconcileHudiSchema

public void reconcileHudiSchema(org.apache.spark.sql.SparkSession sparkSession,
 FeatureGroupAlias featureGroupAlias,
 Map<String,String> hudiArgs)
                         throws FeatureStoreException

Throws:

FeatureStoreException

sparkSchemasMatch

public boolean sparkSchemasMatch(String[] schema1,
 String[] schema2)

streamToHoodieTable

public void streamToHoodieTable(org.apache.spark.sql.SparkSession sparkSession,
 StreamFeatureGroup streamFeatureGroup,
 Map<String,String> writeOptions)
                         throws Exception

Throws:

Exception

getHudiTableSize

public Long getHudiTableSize(org.apache.spark.api.java.JavaSparkContext jsc,
 String basePath)
                      throws IOException

Throws:

IOException