增加自动修补数据定时任务

kaiyu

2020-11-20 1c8d100cbf2916b797de1a0fde5c29d132ead593

增加自动修补数据定时任务

 src/main/java/com/moral/strategy/RepairDataContext.java

New file
@@ -0,0 +1,24 @@
package com.moral.strategy;

import lombok.Data;
import org.springframework.util.MultiValueMap;

import java.util.Map;

/**
* @Description: 策略上下文
        * @Param:
        * @return:
        * @Author: 下雨听风
        * @Date: 2020/11/11
        */
@Data
public class RepairDataContext {
    //策略实现类
    private RepairDataStrategy repairDataStrategy;

    //执行策略的方法
    public void executeStrategy(String time,MultiValueMap<String,String> result){
         repairDataStrategy.repairData(time,result);
    }
}

 src/main/java/com/moral/strategy/RepairDataStrategy.java

New file
@@ -0,0 +1,10 @@
package com.moral.strategy;

import org.springframework.util.MultiValueMap;

import java.util.Map;

//策略模式的接口，主要用于修补插入失败的数据，重新插入失败的数据。
public interface RepairDataStrategy {
   void repairData(String time,MultiValueMap<String,String> result);
}

 src/main/java/com/moral/strategy/RepairFiveMinutelyData.java

New file
@@ -0,0 +1,110 @@
package com.moral.strategy;

import com.alibaba.fastjson.JSON;
import com.moral.entity.History;
import com.moral.service.HistoryFiveMinutelyService;
import com.moral.service.SensorService;
import com.moral.util.DateUtil;
import com.xxl.job.core.biz.model.ReturnT;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Service;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import org.springframework.util.ObjectUtils;

import javax.annotation.Resource;
import java.text.SimpleDateFormat;
import java.util.*;

@Service
public class RepairFiveMinutelyData implements RepairDataStrategy {
    @Resource
    SensorService sensorService;
    @Resource
    HistoryFiveMinutelyService historyFiveMinutelyService;
    @Resource
    RedisTemplate redisTemplate;
    @Override
    public void repairData(String time,MultiValueMap<String,String> result) {
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        String repairTime = sdf.format(new Date(Long.parseLong(time) * 1000));
        /*根据时间从分钟表数据钟查询五分钟数据的平均值*/
        Map<String, Object> params = getStartAndEndTime(time);
        List<String> sensorKeys = sensorService.getSensorKeys();
        params.put("sensorKeys", sensorKeys);
        List<Map<String, Object>> fiveMinutesSensorDatas;
        /*将得到的数据进行转换*/
        List<Map<String, Object>> insertDatas;
        try {
            fiveMinutesSensorDatas = historyFiveMinutelyService.getFiveMinutesSensorData(params);
            insertDatas = new ArrayList<>();
            for (Map<String, Object> data : fiveMinutesSensorDatas) {
                String mac = (String) data.get("mac");
                data.remove("mac");
                Map<String, Object> keyAndValueMap = new LinkedHashMap<>();
                Map<String, Object> insertDataMap = new LinkedHashMap<>();
                data.forEach((key, value) -> {
                    key = key.substring(3);
                    List<Object> list = null;
                    if (ObjectUtils.isEmpty(keyAndValueMap.get(key))) {
                        list = new ArrayList<>();
                    } else {
                        list = (List<Object>) keyAndValueMap.get(key);
                    }
                    if (value instanceof Double) {
                        value = String.valueOf(value);
                        value = value.equals("0.0") ? 0 : Double.valueOf((String) value);
                    }
                    list.add(value);
                    keyAndValueMap.put(key, list);
                });
                String keyAndValueJson = JSON.toJSONString(keyAndValueMap);
                insertDataMap.put("mac", mac);
                insertDataMap.put("time", params.get("end"));
                insertDataMap.put("json", keyAndValueJson);
                insertDatas.add(insertDataMap);
            }
            /*将数据插入数据库*/
            historyFiveMinutelyService.insertHistoryFiveMinutely(insertDatas, (String) params.get("yearAndMonth"));
        } catch (Exception e) {
            result.add("500","修补五分钟数据失败-"+repairTime);
            redisTemplate.opsForList().leftPush("unSuccessRepair_data","fiveMinutely_"+repairTime);
            return;
        }
        result.add("200","修补五分钟数据成功-"+repairTime);
    }


    private Map<String, Object> getStartAndEndTime(String paramTime) {
        Map<String, Object> map = new HashMap<>();
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        Date date = new Date(Long.parseLong(paramTime) * 1000);
        String minute = DateUtil.getMinute(date);
        String year = "";
        String month = "";
        String yearAndMonth = "";
        String startTime = "";
        String endTime = "";
        Integer endMinute = Integer.parseInt(String.valueOf(minute.charAt(minute.length() - 1)));
        if (endMinute >= 5) {
            StringBuilder time = new StringBuilder(sdf.format(date));
            startTime = time.replace(15, 19, "0:00").toString();
            endTime = time.replace(15, 19, "5:00").toString();
        } else {
            StringBuilder endTimesb = new StringBuilder(sdf.format(date));
            endTime = endTimesb.replace(15, 19, "0:00").toString();
            date = DateUtil.rollMinute(date, -5);
            StringBuilder startTimesb = new StringBuilder(sdf.format(date));
            startTime = startTimesb.replace(15, 19, "5:00").toString();

        }

        year = DateUtil.getYear(date);
        month = DateUtil.getMonth(date);
        yearAndMonth = year + month;
        map.put("start", startTime);
        map.put("end", endTime);
        map.put("yearAndMonth", yearAndMonth);
        return map;
    }
}

 src/main/java/com/moral/strategy/RepairHourlyData.java

New file
@@ -0,0 +1,103 @@
package com.moral.strategy;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.moral.service.DeviceService;
import com.moral.service.HistoryHourlyService;
import com.moral.service.HistoryMinutelyService;
import com.moral.service.SensorService;
import com.moral.util.DateUtil;
import com.xxl.job.core.biz.model.ReturnT;
import com.xxl.job.core.log.XxlJobLogger;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import org.springframework.util.ObjectUtils;

import javax.annotation.Resource;
import java.math.BigDecimal;
import java.text.SimpleDateFormat;
import java.util.*;

@Service
public class RepairHourlyData implements RepairDataStrategy {
    @Resource
    SensorService sensorService;
    @Resource
    DeviceService deviceService;
    @Resource
    HistoryMinutelyService historyMinutelyService;
    @Resource
    RedisTemplate redisTemplate;
    @Resource
    HistoryHourlyService historyHourlyService;


    @Override
    public void repairData(String time,MultiValueMap<String,String> result) {

        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        Date endTimeDate = new Date(Long.parseLong(time) * 1000);
        Date startTimeDate = DateUtil.rollHour(endTimeDate, -1);
        StringBuilder endTime = new StringBuilder(sdf.format(endTimeDate));
        StringBuilder startTime = new StringBuilder(sdf.format(startTimeDate));
        endTime.replace(14, 19, "00:00");
        startTime.replace(14, 19, "00:00");
        String yearAndMonth = DateUtil.getYear(startTimeDate) + DateUtil.getMonth(startTimeDate);

        List<String> sensorKeys = sensorService.getSensorKeys();
        List<String> macs = deviceService.getMacs();
        Map<String, Object> devices = new HashMap<>();
        devices.put("sensorKeys", sensorKeys);
        devices.put("start", startTime.toString());
        devices.put("end", endTime.toString());
        devices.put("macs", macs);
        devices.put("yearAndMonth", yearAndMonth);

        try {
            List<Map<String, Object>> hourlyData = historyMinutelyService.getMinutelySensorData(devices);
            XxlJobLogger.log("historyHourlyData:" + hourlyData.size());
            List<Map<String, Object>> hourlyDataList = new ArrayList<>();
            for (Map<String, Object> deviceData : hourlyData) {
                if (!ObjectUtils.isEmpty(deviceData)) {
                    Map<String, Object> hourlyDataMap = new LinkedHashMap<>();
                    JSONObject jo = new JSONObject(true);
                    hourlyDataMap.put("mac", deviceData.get("mac"));
                    hourlyDataMap.put("time", endTime.toString());
                    JSONArray jsonArray = new JSONArray();
                    for (String key : deviceData.keySet()) {
                        if (!key.equals("mac") && !key.startsWith("M")) {
                            List<Object> date = new ArrayList<>();
                            date.add(deviceData.get(key));
                            if (deviceData.get("MIN" + key) instanceof String) {
                                date.add(new BigDecimal(deviceData.get("MIN" + key).toString()));
                                date.add(new BigDecimal(deviceData.get("MAX" + key).toString()));
                            } else if (deviceData.get("MIN" + key) instanceof byte[]) {
                                date.add(new BigDecimal(new String((byte[]) (deviceData.get("MIN" + key)))));
                                date.add(new BigDecimal(new String((byte[]) (deviceData.get("MAX" + key)))));
                            }
                            jo.put(key, date);
                        }
                    }
                    jsonArray.add(jo);
                    hourlyDataMap.put("json", jsonArray.get(0).toString());
                    hourlyDataList.add(hourlyDataMap);
                }
            }
            if (!CollectionUtils.isEmpty(hourlyDataList)) {
                historyHourlyService.insertHistoryHourly(hourlyDataList);
                result.add("200", "修补小时数据成功-"+endTime);
                return;
            }
        } catch (Exception e) {
            XxlJobLogger.log("historyHourlyException:" + e.getMessage());
            result.add("500", "修补小时数据失败-"+endTime);
            redisTemplate.opsForList().leftPush("unSuccessRepair_data","hourly_"+endTime);
            return;
        }
        result.add("500", "修补小时数据失败-"+endTime);
        redisTemplate.opsForList().leftPush("unSuccessRepair_data","hourly_"+endTime);
    }
}

 src/main/java/com/moral/strategy/RepairMinutelyData.java

New file
@@ -0,0 +1,97 @@
package com.moral.strategy;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.moral.service.DeviceService;
import com.moral.service.HistoryMinutelyService;
import com.moral.service.SensorService;
import com.moral.util.DateUtil;
import com.xxl.job.core.biz.model.ReturnT;
import com.xxl.job.core.log.XxlJobLogger;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import org.springframework.util.ObjectUtils;

import javax.annotation.Resource;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;

@Service
public class RepairMinutelyData implements RepairDataStrategy {

    @Resource
    SensorService sensorService;
    @Resource
    DeviceService deviceService;
    @Resource
    HistoryMinutelyService historyMinutelyService;
    @Resource
    RedisTemplate redisTemplate;

    @Override
    public void repairData(String time, MultiValueMap<String, String> result) {

        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        Date endTimeDate = new Date(Long.parseLong(time) * 1000);
        Date startTimeDate = DateUtil.rollMinute(endTimeDate, -1);
        StringBuilder endTime = new StringBuilder(sdf.format(endTimeDate));
        StringBuilder startTime = new StringBuilder(sdf.format(startTimeDate));
        endTime.replace(17, 19, "00");
        startTime.replace(17, 19, "00");
        String queryYearAndMonthDay = DateUtil.getYear(startTimeDate) + DateUtil.getMonth(startTimeDate) + DateUtil.getDay(startTimeDate);


        Map<String, Object> devices = new HashMap<>();
        List<String> sensorKeys = sensorService.getSensorKeys();
        devices.put("sensorKeys", sensorKeys);
        devices.put("start", startTime.toString());
        devices.put("end", endTime.toString());
        devices.put("yearAndMonthDay", queryYearAndMonthDay);

        try {
            List<Map<String, Object>> minutelyData = deviceService.getSensorData(devices);
            XxlJobLogger.log("historyMinutelyData:" + minutelyData.size());
            List<Map<String, Object>> minutelyDataList = new ArrayList<>();
            for (Map<String, Object> deviceData : minutelyData) {
                if (!ObjectUtils.isEmpty(deviceData)) {
                    Map<String, Object> minutelyDataMap = new LinkedHashMap<>();
                    JSONObject jo = new JSONObject(true);
                    minutelyDataMap.put("mac", deviceData.get("mac"));
                    minutelyDataMap.put("time", endTime.toString());
                    JSONArray jsonArray = new JSONArray();
                    for (String key : deviceData.keySet()) {
                        if (!key.equals("mac") && !key.startsWith("M")) {
                            List<Object> date = new ArrayList<>();
                            date.add(deviceData.get(key));
                            date.add(deviceData.get("MIN" + key));
                            date.add(deviceData.get("MAX" + key));
                            jo.put(key, date);
                        }
                    }
                    jsonArray.add(jo);
                    minutelyDataMap.put("json", jsonArray.get(0).toString());
                    minutelyDataList.add(minutelyDataMap);
                }
            }
            if (!CollectionUtils.isEmpty(minutelyDataList)) {
                String insertYearAndMonth = DateUtil.getYear(endTimeDate) + DateUtil.getMonth(endTimeDate);
                historyMinutelyService.insertHistoryMinutely(minutelyDataList, insertYearAndMonth);
                result.add("200", "修补分钟数据成功-"+endTime);
                return;
            }
        } catch (Exception e) {
            XxlJobLogger.log("repairHistoryMinutelyException:" + e.getMessage());
            result.add("500", "修补分钟数据失败-"+endTime);
            redisTemplate.opsForList().leftPush("unSuccessRepair_data","minutely_"+endTime);
            return;
        }
        result.add("500", "修补分钟数据失败-"+endTime);
        redisTemplate.opsForList().leftPush("unSuccessRepair_data","minutely_"+endTime);
    }

}


 src/main/java/com/moral/strategy/RepairStrategyFactory.java

New file
@@ -0,0 +1,29 @@
package com.moral.strategy;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;

/**
* @Description: 策略工厂
        * @Param:
        * @return:
        * @Author: 下雨听风
        * @Date: 2020/11/11
        */
@Component
public class RepairStrategyFactory {

    @Resource
    Map<String,RepairDataStrategy> strategys = new ConcurrentHashMap<>();

    public RepairDataStrategy getInstance(String type){
        String[] types = type.split("_");
        return strategys.get(types[0]);
    }

}

 src/main/java/com/moral/task/RepairDataTask.java

New file
@@ -0,0 +1,66 @@
package com.moral.task;

import com.moral.strategy.RepairDataContext;
import com.moral.strategy.RepairDataStrategy;
import com.moral.strategy.RepairStrategyFactory;
import com.xxl.job.core.biz.model.ReturnT;
import com.xxl.job.core.handler.annotation.XxlJob;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Component;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import org.springframework.util.ObjectUtils;

import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import java.text.ParseException;
import java.util.*;

@Component
public class RepairDataTask {
    @Resource
    RedisTemplate redisTemplate;
    @Resource
    RepairStrategyFactory factory;

    private Map<String,Integer> sortMap = new HashMap<>();

    @PostConstruct
    public void init(){
        this.sortMap.put("repairHourlyData",3);
        this.sortMap.put("repairFiveMinutelyData",2);
        this.sortMap.put("repairMinutelyData",1);
    }

    @XxlJob("repairData")
    public ReturnT repairData(String param) throws ParseException {
        //从缓存中读取插入失败的数据
        List<String> records = redisTemplate.opsForList().range("unrepair_data", 0, -1);
        if(ObjectUtils.isEmpty(records))
            return new ReturnT(200,"无数据需要修补");
        //对错误信息进行排序
        sorRecords(records);
        redisTemplate.delete("unrepair_data");
        MultiValueMap<String, String> results = new LinkedMultiValueMap<>();
        RepairDataContext context = new RepairDataContext();
        for (String record : records) {
            RepairDataStrategy instance = factory.getInstance(record);
            context.setRepairDataStrategy(instance);
            context.executeStrategy(record.split("_")[1], results);
        }
        return new ReturnT(200,results.toString());
    }


    public void sorRecords(List<String> records){
        Collections.sort(records, new Comparator<String>() {
            @Override
            public int compare(String o1, String o2) {
                Integer integer = sortMap.get(o1.split("_")[0]);
                Integer integer1 = sortMap.get(o2.split("_")[0]);
                return integer>integer1?1:-1;
            }
        });
    }

}