You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
61 lines
1.8 KiB
61 lines
1.8 KiB
package com.qniao.iot;
|
|
|
|
import org.apache.flink.api.java.DataSet;
|
|
import org.apache.flink.api.java.ExecutionEnvironment;
|
|
import org.apache.flink.api.java.tuple.Tuple3;
|
|
import org.apache.flink.util.FileUtils;
|
|
|
|
import java.io.File;
|
|
import java.util.List;
|
|
import java.util.Map;
|
|
|
|
|
|
public class CloudBoxEventJob1 {
|
|
|
|
private static EsRestClientService esRestClientService = new EsRestClientService();
|
|
|
|
public static void main(String[] args) throws Exception {
|
|
|
|
|
|
// set up the execution environment
|
|
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
|
|
|
|
// 查询数据searchResponse
|
|
String scrollId = null;
|
|
DataSet<Tuple3<String, String, Integer>> dataSet = null;
|
|
List<Tuple3<String, String, Integer>> dataList = null;
|
|
|
|
int count = 0;
|
|
|
|
while (!"none".equals(scrollId)) {
|
|
|
|
Map<String, Object> map = esRestClientService.queryDeviceListPage(scrollId);
|
|
if (map.get("tupleList") instanceof List)
|
|
dataList = (List<Tuple3<String, String, Integer>>) map.get("tupleList");
|
|
scrollId = map.get("scrollId").toString();
|
|
|
|
if (dataList == null || dataList.size() < 10000 || count > 3)
|
|
break;
|
|
|
|
// 导入数据
|
|
DataSet<Tuple3<String, String, Integer>> dataSetTemp = env.fromCollection(dataList);
|
|
if (dataSet == null) {
|
|
dataSet = dataSetTemp;
|
|
} else {
|
|
dataSet = dataSet.union(dataSetTemp);
|
|
}
|
|
++count;
|
|
}
|
|
// 分组计算规则
|
|
dataSet = dataSet.groupBy(0).sum(2);
|
|
|
|
|
|
//dataSet.print();
|
|
|
|
String output = "C:\\Users\\10499\\Downloads\\1223.txt";
|
|
FileUtils.deleteFileOrDirectory(new File(output));
|
|
dataSet.writeAsText(output);
|
|
|
|
env.execute("read es");
|
|
}
|
|
}
|