|
|
@ -24,6 +24,7 @@ import com.qniao.iot.rc.event.RootCloudIotDataReceiptedEventDeserializationSchem |
|
|
import org.apache.flink.api.common.eventtime.WatermarkStrategy; |
|
|
import org.apache.flink.api.common.eventtime.WatermarkStrategy; |
|
|
import org.apache.flink.api.common.functions.MapFunction; |
|
|
import org.apache.flink.api.common.functions.MapFunction; |
|
|
import org.apache.flink.api.common.serialization.SimpleStringEncoder; |
|
|
import org.apache.flink.api.common.serialization.SimpleStringEncoder; |
|
|
|
|
|
import org.apache.flink.api.java.utils.ParameterTool; |
|
|
import org.apache.flink.connector.base.DeliveryGuarantee; |
|
|
import org.apache.flink.connector.base.DeliveryGuarantee; |
|
|
import org.apache.flink.connector.kafka.sink.KafkaRecordSerializationSchema; |
|
|
import org.apache.flink.connector.kafka.sink.KafkaRecordSerializationSchema; |
|
|
import org.apache.flink.connector.kafka.sink.KafkaSink; |
|
|
import org.apache.flink.connector.kafka.sink.KafkaSink; |
|
|
@ -52,12 +53,14 @@ import org.apache.flink.streaming.api.functions.sink.filesystem.StreamingFileSin |
|
|
public class RootCloudIotDataFormatterJob { |
|
|
public class RootCloudIotDataFormatterJob { |
|
|
|
|
|
|
|
|
public static void main(String[] args) throws Exception { |
|
|
public static void main(String[] args) throws Exception { |
|
|
|
|
|
final ParameterTool params = ParameterTool.fromArgs(args); |
|
|
|
|
|
|
|
|
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); |
|
|
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); |
|
|
env.enableCheckpointing(60000L, CheckpointingMode.EXACTLY_ONCE); |
|
|
env.enableCheckpointing(60000L, CheckpointingMode.EXACTLY_ONCE); |
|
|
KafkaSource<RootCloudIotDataReceiptedEvent> source = KafkaSource.<RootCloudIotDataReceiptedEvent>builder() |
|
|
KafkaSource<RootCloudIotDataReceiptedEvent> source = KafkaSource.<RootCloudIotDataReceiptedEvent>builder() |
|
|
.setBootstrapServers("120.25.199.30:9092") |
|
|
|
|
|
|
|
|
.setBootstrapServers(params.get("source.bootstrap.servers")) |
|
|
.setTopics("root_cloud_iot_report_data_event") |
|
|
.setTopics("root_cloud_iot_report_data_event") |
|
|
.setGroupId("flink-kafka-demo") |
|
|
|
|
|
|
|
|
.setGroupId("root_cloud_iot_data_etl") |
|
|
.setStartingOffsets(OffsetsInitializer.earliest()) |
|
|
.setStartingOffsets(OffsetsInitializer.earliest()) |
|
|
.setValueOnlyDeserializer(new RootCloudIotDataReceiptedEventDeserializationSchema()) |
|
|
.setValueOnlyDeserializer(new RootCloudIotDataReceiptedEventDeserializationSchema()) |
|
|
.build(); |
|
|
.build(); |
|
|
@ -79,7 +82,7 @@ public class RootCloudIotDataFormatterJob { |
|
|
// 再发送到kafka队列中 |
|
|
// 再发送到kafka队列中 |
|
|
transformDs.sinkTo( |
|
|
transformDs.sinkTo( |
|
|
KafkaSink.<MachineIotDataReceivedEvent>builder() |
|
|
KafkaSink.<MachineIotDataReceivedEvent>builder() |
|
|
.setBootstrapServers("120.25.199.30:9092") |
|
|
|
|
|
|
|
|
.setBootstrapServers(params.get("sink.bootstrap.servers")) |
|
|
.setRecordSerializer( |
|
|
.setRecordSerializer( |
|
|
KafkaRecordSerializationSchema.builder() |
|
|
KafkaRecordSerializationSchema.builder() |
|
|
.setTopic("machine_iot_data_received_event") |
|
|
.setTopic("machine_iot_data_received_event") |
|
|
|