apache / fluss

[flink] Support PARTITION_DYNAMIC sink dynamic shuffle based on partition traffics (#1784)

Hongshun Wang

Hongshun Wang committed on Jan 15, 2026, 09:35 AM

Showing 40 changed files +3393 additions -84 deletions

A fluss-flink/fluss-flink-1.18/src/main/java/org/apache/fluss/flink/adapter/RuntimeContextAdapter.java +39

M fluss-flink/fluss-flink-1.18/src/test/java/org/apache/fluss/flink/sink/Flink118TableSinkITCase.java +21 -1

M fluss-flink/fluss-flink-1.19/src/test/java/org/apache/fluss/flink/sink/Flink119TableSinkITCase.java +22 -1

M fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/FlinkConnectorOptions.java +28 -1

M fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/adapter/RuntimeContextAdapter.java +5

M fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/catalog/FlinkTableFactory.java +5 -2

M fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/row/RowWithOp.java +19 -1

M fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/FlinkSink.java +103 -29

M fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/FlinkTableSink.java +20 -8

M fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/FlussSinkBuilder.java +15 -5

M fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/RowDataSerializationSchema.java +93 -1

A fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/shuffle/AggregatedStatisticsTracker.java +148

A fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/shuffle/DataStatistics.java +70

A fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/shuffle/DataStatisticsCoordinator.java +359

A fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/shuffle/DataStatisticsCoordinatorProvider.java +40

A fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/shuffle/DataStatisticsOperator.java +168

A fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/shuffle/DataStatisticsOperatorFactory.java +93

A fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/shuffle/DataStatisticsSerializer.java +144

A fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/shuffle/DistributionMode.java +26

A fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/shuffle/PartitionAssignment.java +34

A fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/shuffle/StatisticsEvent.java +53

A fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/shuffle/StatisticsOrRecord.java +117

A fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/shuffle/StatisticsOrRecordChannelComputer.java +313

A fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/shuffle/StatisticsOrRecordSerializer.java +222

A fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/shuffle/StatisticsOrRecordTypeInformation.java +118

A fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/shuffle/StatisticsUtil.java +56

A fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/shuffle/WeightedBucketIdAssignment.java +79

A fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/shuffle/WeightedRandomAssignment.java +157

M fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/metrics/FlinkMetricsITCase.java +2

M fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/FlinkTableSinkITCase.java +107 -25

M fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/FlussSinkBuilderTest.java +12 -7

M fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/FlussSinkITCase.java +2 -2

A fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/shuffle/DataStatisticOperatorTest.java +114

A fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/shuffle/DataStatisticsCoordinatorTest.java +177

A fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/shuffle/DataStatisticsSerializerTest.java +53

A fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/shuffle/MockRandom.java +42

A fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/shuffle/PartitionAssignmentTest.java +86

A fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/shuffle/StatisticsOrRecordChannelComputerTest.java +197

M fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/flink/FlinkUnionReadLogTableITCase.java +2

M website/docs/engine-flink/options.md +32 -1

Browse files at this commit →