hupenghui@qniao.cn
3 years ago
4 changed files with 112 additions and 62 deletions
Unified View
Diff Options
-
117cloud-box-job/src/main/java/com/qniao/iot/CloudBoxEventJob1.java
-
6cloud-box-job/src/main/java/com/qniao/iot/DruidDataSourceUtil.java
-
43cloud-box-job/src/main/java/com/qniao/iot/EsRestClientService.java
-
8cloud-box-job/src/main/java/com/qniao/iot/SinkMysqlFunc.java
@ -1,61 +1,116 @@ |
|||||
package com.qniao.iot; |
package com.qniao.iot; |
||||
|
|
||||
import org.apache.flink.api.java.DataSet; |
|
||||
import org.apache.flink.api.java.ExecutionEnvironment; |
|
||||
import org.apache.flink.api.java.tuple.Tuple3; |
|
||||
import org.apache.flink.util.FileUtils; |
|
||||
|
import cn.hutool.core.util.StrUtil; |
||||
|
import com.alibaba.druid.pool.DruidDataSource; |
||||
|
|
||||
import java.io.File; |
|
||||
|
import java.sql.Connection; |
||||
|
import java.sql.Date; |
||||
|
import java.sql.PreparedStatement; |
||||
|
import java.text.SimpleDateFormat; |
||||
import java.util.List; |
import java.util.List; |
||||
import java.util.Map; |
import java.util.Map; |
||||
|
import java.util.concurrent.*; |
||||
|
|
||||
|
|
||||
public class CloudBoxEventJob1 { |
public class CloudBoxEventJob1 { |
||||
|
|
||||
private static EsRestClientService esRestClientService = new EsRestClientService(); |
|
||||
|
private static final EsRestClientService esRestClientService = new EsRestClientService(); |
||||
|
|
||||
public static void main(String[] args) throws Exception { |
|
||||
|
private static final DruidDataSource DATA_SOURCE = DruidDataSourceUtil.getInstance(); |
||||
|
|
||||
|
public static void main(String[] args) throws Exception { |
||||
|
|
||||
// set up the execution environment |
|
||||
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); |
|
||||
|
|
||||
// 查询数据searchResponse |
// 查询数据searchResponse |
||||
String scrollId = null; |
String scrollId = null; |
||||
DataSet<Tuple3<String, String, Integer>> dataSet = null; |
|
||||
List<Tuple3<String, String, Integer>> dataList = null; |
|
||||
|
List<Body> dataList = null; |
||||
|
|
||||
int count = 0; |
int count = 0; |
||||
|
|
||||
|
int nThreads = Runtime.getRuntime().availableProcessors(); |
||||
|
ExecutorService executorService = Executors.newFixedThreadPool(nThreads); |
||||
|
Map<String, Object> map = new ConcurrentHashMap<>(10000); |
||||
while (!"none".equals(scrollId)) { |
while (!"none".equals(scrollId)) { |
||||
|
|
||||
Map<String, Object> map = esRestClientService.queryDeviceListPage(scrollId); |
|
||||
if (map.get("tupleList") instanceof List) |
|
||||
dataList = (List<Tuple3<String, String, Integer>>) map.get("tupleList"); |
|
||||
|
map.clear(); |
||||
|
esRestClientService.queryDeviceListPage(scrollId, map, count); |
||||
|
if (map.get("tupleList") != null) { |
||||
|
dataList = (List<Body>) map.get("tupleList"); |
||||
|
} |
||||
scrollId = map.get("scrollId").toString(); |
scrollId = map.get("scrollId").toString(); |
||||
|
|
||||
if (dataList == null || dataList.size() < 10000 || count > 3) |
|
||||
break; |
|
||||
|
|
||||
// 导入数据 |
|
||||
DataSet<Tuple3<String, String, Integer>> dataSetTemp = env.fromCollection(dataList); |
|
||||
if (dataSet == null) { |
|
||||
dataSet = dataSetTemp; |
|
||||
} else { |
|
||||
dataSet = dataSet.union(dataSetTemp); |
|
||||
|
if(count>861) { |
||||
|
if (dataList == null || dataList.size() < 10000 || count > 1000) { |
||||
|
executorService.shutdown(); |
||||
|
while (!executorService.isTerminated()) { |
||||
|
System.out.println("任务正在执行中，请稍等。。。"); |
||||
|
Thread.sleep(10000); |
||||
|
} |
||||
|
System.out.println("任务执行完成。。。"); |
||||
|
break; |
||||
|
} |
||||
|
// 导入数据到mysql |
||||
|
List<Body> finalDataList = dataList; |
||||
|
executorService.execute(() -> { |
||||
|
try { |
||||
|
invoke(finalDataList); |
||||
|
} catch (Exception e) { |
||||
|
e.printStackTrace(); |
||||
|
} |
||||
|
}); |
||||
} |
} |
||||
++count; |
++count; |
||||
} |
} |
||||
// 分组计算规则 |
|
||||
dataSet = dataSet.groupBy(0).sum(2); |
|
||||
|
|
||||
|
} |
||||
|
|
||||
//dataSet.print(); |
|
||||
|
public static void invoke(List<Body> values) throws Exception { |
||||
|
|
||||
|
if (values.size() > 0) { |
||||
|
Connection connection = DATA_SOURCE.getConnection(); |
||||
|
PreparedStatement ps = connection.prepareStatement(getSql()); |
||||
|
for (Body body : values) { |
||||
|
if (body != null) { |
||||
|
ps.setInt(1, body.getData_source()); |
||||
|
ps.setLong(2, body.getMac()); |
||||
|
Integer dataType = body.getData_type(); |
||||
|
ps.setInt(3, dataType == 1 ? 0 : 1); |
||||
|
ps.setInt(4, dataType == 2 ? 1 : 0); |
||||
|
Long totalProduction = body.getTotal_production(); |
||||
|
ps.setLong(5, totalProduction == null ? 0 : totalProduction); |
||||
|
Long quantity = body.getQuantity(); |
||||
|
ps.setLong(6, quantity == null ? 0L : quantity); |
||||
|
ps.setBigDecimal(7, body.getRunningDuration()); |
||||
|
ps.setBigDecimal(8, body.getWaitingDuration()); |
||||
|
ps.setBigDecimal(9, null); |
||||
|
ps.setBigDecimal(10, null); |
||||
|
String createTimeStr = body.getCreate_time(); |
||||
|
Date createDate = null; |
||||
|
if (StrUtil.isNotEmpty(createTimeStr)) { |
||||
|
long time = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parse(createTimeStr).getTime(); |
||||
|
createDate = new Date(time); |
||||
|
} |
||||
|
ps.setDate(11, createDate); |
||||
|
Long id = body.getId(); |
||||
|
ps.setString(12, id == null ? "0" : StrUtil.toString(id)); |
||||
|
ps.setString(13, body.getDocId()); |
||||
|
ps.addBatch(); |
||||
|
} |
||||
|
} |
||||
|
ps.executeBatch(); |
||||
|
if (connection != null) { |
||||
|
connection.close(); |
||||
|
} |
||||
|
if (ps != null) { |
||||
|
ps.close(); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
String output = "C:\\Users\\10499\\Downloads\\1223.txt"; |
|
||||
FileUtils.deleteFileOrDirectory(new File(output)); |
|
||||
dataSet.writeAsText(output); |
|
||||
|
private static String getSql() { |
||||
|
|
||||
env.execute("read es"); |
|
||||
|
return "insert into qn_cloud_box_event(data_source, machine_iot_mac, machine_pwr_stat, machine_working_stat, acc_job_count,\n" + |
||||
|
" curr_job_count, curr_job_duration, curr_waiting_duration, curr_stoping_duration, ig_stat,\n" + |
||||
|
" report_time,event_id, doc_id)\n" + |
||||
|
"values (?, ?, ?, ?, ?,?, ?, ?, ?, ?, ?, ?, ?)"; |
||||
} |
} |
||||
} |
} |
||||
Write
Preview
Loading…
Cancel
Save