hupenghui@qniao.cn
3 years ago
4 changed files with 112 additions and 62 deletions
Split View
Diff Options
-
117cloud-box-job/src/main/java/com/qniao/iot/CloudBoxEventJob1.java
-
6cloud-box-job/src/main/java/com/qniao/iot/DruidDataSourceUtil.java
-
43cloud-box-job/src/main/java/com/qniao/iot/EsRestClientService.java
-
8cloud-box-job/src/main/java/com/qniao/iot/SinkMysqlFunc.java
@ -1,61 +1,116 @@ |
|||
package com.qniao.iot; |
|||
|
|||
import org.apache.flink.api.java.DataSet; |
|||
import org.apache.flink.api.java.ExecutionEnvironment; |
|||
import org.apache.flink.api.java.tuple.Tuple3; |
|||
import org.apache.flink.util.FileUtils; |
|||
import cn.hutool.core.util.StrUtil; |
|||
import com.alibaba.druid.pool.DruidDataSource; |
|||
|
|||
import java.io.File; |
|||
import java.sql.Connection; |
|||
import java.sql.Date; |
|||
import java.sql.PreparedStatement; |
|||
import java.text.SimpleDateFormat; |
|||
import java.util.List; |
|||
import java.util.Map; |
|||
import java.util.concurrent.*; |
|||
|
|||
|
|||
public class CloudBoxEventJob1 { |
|||
|
|||
private static EsRestClientService esRestClientService = new EsRestClientService(); |
|||
private static final EsRestClientService esRestClientService = new EsRestClientService(); |
|||
|
|||
public static void main(String[] args) throws Exception { |
|||
private static final DruidDataSource DATA_SOURCE = DruidDataSourceUtil.getInstance(); |
|||
|
|||
public static void main(String[] args) throws Exception { |
|||
|
|||
// set up the execution environment |
|||
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); |
|||
|
|||
// 查询数据searchResponse |
|||
String scrollId = null; |
|||
DataSet<Tuple3<String, String, Integer>> dataSet = null; |
|||
List<Tuple3<String, String, Integer>> dataList = null; |
|||
List<Body> dataList = null; |
|||
|
|||
int count = 0; |
|||
|
|||
int nThreads = Runtime.getRuntime().availableProcessors(); |
|||
ExecutorService executorService = Executors.newFixedThreadPool(nThreads); |
|||
Map<String, Object> map = new ConcurrentHashMap<>(10000); |
|||
while (!"none".equals(scrollId)) { |
|||
|
|||
Map<String, Object> map = esRestClientService.queryDeviceListPage(scrollId); |
|||
if (map.get("tupleList") instanceof List) |
|||
dataList = (List<Tuple3<String, String, Integer>>) map.get("tupleList"); |
|||
map.clear(); |
|||
esRestClientService.queryDeviceListPage(scrollId, map, count); |
|||
if (map.get("tupleList") != null) { |
|||
dataList = (List<Body>) map.get("tupleList"); |
|||
} |
|||
scrollId = map.get("scrollId").toString(); |
|||
|
|||
if (dataList == null || dataList.size() < 10000 || count > 3) |
|||
break; |
|||
|
|||
// 导入数据 |
|||
DataSet<Tuple3<String, String, Integer>> dataSetTemp = env.fromCollection(dataList); |
|||
if (dataSet == null) { |
|||
dataSet = dataSetTemp; |
|||
} else { |
|||
dataSet = dataSet.union(dataSetTemp); |
|||
if(count>861) { |
|||
if (dataList == null || dataList.size() < 10000 || count > 1000) { |
|||
executorService.shutdown(); |
|||
while (!executorService.isTerminated()) { |
|||
System.out.println("任务正在执行中，请稍等。。。"); |
|||
Thread.sleep(10000); |
|||
} |
|||
System.out.println("任务执行完成。。。"); |
|||
break; |
|||
} |
|||
// 导入数据到mysql |
|||
List<Body> finalDataList = dataList; |
|||
executorService.execute(() -> { |
|||
try { |
|||
invoke(finalDataList); |
|||
} catch (Exception e) { |
|||
e.printStackTrace(); |
|||
} |
|||
}); |
|||
} |
|||
++count; |
|||
} |
|||
// 分组计算规则 |
|||
dataSet = dataSet.groupBy(0).sum(2); |
|||
|
|||
} |
|||
|
|||
//dataSet.print(); |
|||
public static void invoke(List<Body> values) throws Exception { |
|||
|
|||
if (values.size() > 0) { |
|||
Connection connection = DATA_SOURCE.getConnection(); |
|||
PreparedStatement ps = connection.prepareStatement(getSql()); |
|||
for (Body body : values) { |
|||
if (body != null) { |
|||
ps.setInt(1, body.getData_source()); |
|||
ps.setLong(2, body.getMac()); |
|||
Integer dataType = body.getData_type(); |
|||
ps.setInt(3, dataType == 1 ? 0 : 1); |
|||
ps.setInt(4, dataType == 2 ? 1 : 0); |
|||
Long totalProduction = body.getTotal_production(); |
|||
ps.setLong(5, totalProduction == null ? 0 : totalProduction); |
|||
Long quantity = body.getQuantity(); |
|||
ps.setLong(6, quantity == null ? 0L : quantity); |
|||
ps.setBigDecimal(7, body.getRunningDuration()); |
|||
ps.setBigDecimal(8, body.getWaitingDuration()); |
|||
ps.setBigDecimal(9, null); |
|||
ps.setBigDecimal(10, null); |
|||
String createTimeStr = body.getCreate_time(); |
|||
Date createDate = null; |
|||
if (StrUtil.isNotEmpty(createTimeStr)) { |
|||
long time = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parse(createTimeStr).getTime(); |
|||
createDate = new Date(time); |
|||
} |
|||
ps.setDate(11, createDate); |
|||
Long id = body.getId(); |
|||
ps.setString(12, id == null ? "0" : StrUtil.toString(id)); |
|||
ps.setString(13, body.getDocId()); |
|||
ps.addBatch(); |
|||
} |
|||
} |
|||
ps.executeBatch(); |
|||
if (connection != null) { |
|||
connection.close(); |
|||
} |
|||
if (ps != null) { |
|||
ps.close(); |
|||
} |
|||
} |
|||
} |
|||
|
|||
String output = "C:\\Users\\10499\\Downloads\\1223.txt"; |
|||
FileUtils.deleteFileOrDirectory(new File(output)); |
|||
dataSet.writeAsText(output); |
|||
private static String getSql() { |
|||
|
|||
env.execute("read es"); |
|||
return "insert into qn_cloud_box_event(data_source, machine_iot_mac, machine_pwr_stat, machine_working_stat, acc_job_count,\n" + |
|||
" curr_job_count, curr_job_duration, curr_waiting_duration, curr_stoping_duration, ig_stat,\n" + |
|||
" report_time,event_id, doc_id)\n" + |
|||
"values (?, ?, ?, ?, ?,?, ?, ?, ?, ?, ?, ?, ?)"; |
|||
} |
|||
} |
|||
Write
Preview
Loading…
Cancel
Save