内存分页的几种方法

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
/**
  * 处理内存分页
  * @param list
  * @param pageNo
  * @param pageSize
  * @return
  */
 private <T> List<T> getPageConfigList(List<T> list, int pageNo, int pageSize){
     List<T> currentPagelist = null;
     if (list != null && !list.isEmpty()) {
         currentPagelist = new ArrayList<T>();
         int iEnd = 0;
         int iStart = 0;

         iStart = pageSize * (pageNo - 1);
         iEnd = iStart + pageSize;
         if (iEnd > list.size()) {
             iEnd = list.size();
         }
         for (int i = iStart; i < iEnd; i++) {
             currentPagelist.add(list.get(i));
         }

     }
     return currentPagelist;
 }

1
2
3
CollectionContentUtils.sortList(sList, new String[] {"rtn_urlpathmethod"}, "desc");
List<List<TipConfig>> partition = Lists.partition(sList, length);
page.setData(partition.get((start+length-1)/length));

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
import com.google.common.base.Function;
import com.google.common.collect.Lists;

import java.util.ArrayList;
import java.util.List;


public class Main {
    public static void main(String[] args) throws Exception {

        List<Long> list = new ArrayList<>();
        list.add(1L);
        list.add(2L);
        list.add(3L);
        list.add(4L);
        list.add(5L);
        list.add(6L);
        list.add(7L);
        list.add(8L);
        list.add(9L);

        List<List<Long>> originalPageList = Lists.partition(list, 3);
        List<String> pageList = Lists.transform(originalPageList, new Function<List<Long>, String>() {
            @Override
            public String apply(List<Long> list) {
                final StringBuffer pageSkuIds = new StringBuffer();
                for(Long info : list) {
                    pageSkuIds.append("AA_").append(info).append(",");
                }
                return pageSkuIds.toString();
            }
        });
        System.out.println(pageList.toString());
    }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
@DataSource(value = "master")
public void recordRanking() {
    logger.error("recordRanking start====================================");
    String distributedLockKey = "";//分布式锁key
    String rankingSizeKey = "";//某个活动的排行榜大小key
    String batchInsertErrorKey = "";//某个活动批次插入报错key
    String promotionAndRuleKey = "";//时光机活动规则key
    try {
        // 获取最近一条需要发放大奖的活动（活动结束即可）
        promotionAndRuleKey = "promotionAndRule_promotionType_" + SnsPromotionType.TIMEMACHINE_GAME.getCode();
        SnsPromotionShareRuleVo spsrVo = (SnsPromotionShareRuleVo) CacheUtil.getObject(promotionAndRuleKey);
        if (spsrVo == null) {
            spsrVo = snsPromotionDAO.selectPrixPromotionAndRuleByType(SnsPromotionType.TIMEMACHINE_GAME.getCode());
            CacheUtil.setObject(promotionAndRuleKey, spsrVo, 30);
            if (spsrVo == null) {
                logger.error("没有已结束需要发放大奖的时光机活动");
                return;
            }
        }
        Long promotionId = spsrVo.getPromotionId();

        distributedLockKey = String.format("distributedLockKey_recordRanking_promotionId_%d", promotionId);
        rankingSizeKey = String.format("rankingSizeKey_recordRanking_promotionId_%d", promotionId);
        batchInsertErrorKey = String.format("batchInsertErrorKey_recordRanking_promotionId_%d", promotionId);

        // 分布式锁（保证下面逻辑不会在同一时间重复执行）
        if(!CacheUtil.set(distributedLockKey, distributedLockKey, 60*15)){
            Object[] args = {distributedLockKey, promotionId};
            logger.error("未获取到分布式锁,key:{},活动id:{}", args);
            return;
        }

        // 先查看排行榜表中是否已经全部写入本次活动的数据
        SnsShareRecordRankingExample example = new SnsShareRecordRankingExample();
        example.createCriteria().andPromotionIdEqualTo(promotionId);
        int count = snsShareRecordRankingDAO.countByExample(example);
        String rankingSizeStr = CacheUtil.get(rankingSizeKey);
        if (StringUtils.isNotBlank(rankingSizeStr) && Integer.valueOf(rankingSizeStr) <= count) {
            logger.error("活动id：" + promotionId + "的排行榜数据已经全部写入临时表,排行榜数量：" + count + ",请勿重复操作");
            return;
        }

        // 处理插入失败的数据
        boolean handleFlag = handleBatchInsertErrorList(batchInsertErrorKey, promotionId);
        if (handleFlag) {
            return;
        }

        // 获取排行榜
        StopWatch sw1 = new StopWatch();
        sw1.start();
        List<SnsShareRecord> ssrList = snsShareRecordDAO.getListOfAccomplishedAndFilteredByPromotionId(promotionId);
        sw1.stop();
        logger.error("snsShareRecordDAO.getListOfAccomplishedAndFilteredByPromotionId耗时：" + sw1.getTotalTimeMillis() + "ms");
        if (CollectionUtils.isEmpty(ssrList)) {
            logger.error("该时光机活动排行榜为空,活动id：" + promotionId);
            return;
        }
        int rankingSize = ssrList.size();
        // 将排行榜大小放入缓存
        CacheUtil.set(rankingSizeKey, String.valueOf(rankingSize));
        logger.error("活动id：" + promotionId + "，排行榜数量：" + rankingSize);

        // 分批次写入
        StopWatch sw2 = new StopWatch();
        sw2.start();
        // n:计数器,m:页数,l:模数,maxSize:每批处理数量
        int n = 0, m = 0, size = rankingSize, maxSize = 500, l = size%maxSize;
        int flag = (l == 0) ? 0 : 1;
        size = size/maxSize;
        List<SnsShareRecordRanking> ssrrListTemp = new ArrayList<>(maxSize);
        SnsShareRecordRanking ssrrTemp = null;
        for (int i = 0; i < rankingSize; i++) {
            SnsShareRecord ssr = ssrList.get(i);
            ssrrTemp = new SnsShareRecordRanking();
            ssrrTemp.setShareRecordId(ssr.getId());
            ssrrTemp.setPromotionId(ssr.getPromotionId());
            ssrrTemp.setUserId(ssr.getUserId());
            ssrrTemp.setPrixResultState(ssr.getPrixResultState());
            ssrrTemp.setTotalRegisterDuration(ssr.getTotalRegisterDuration());
            ssrrTemp.setRanking(i+1);
            ssrrListTemp.add(ssrrTemp);

            n++;
            //预处理数据不能整除最大行数时,最后几行数据的处理,n==l表示最后一部分数据;
            if(size == m && n == l) {
                flag = 0;
            }

            /**
             * case1：大于等于最大行数
             * case2：预处理数据不能整除最大行数时,最后几行数据的处理,flag=0
             * case3：预处理数据量可以整除最大行数,flag=0
             */
            if(n >= maxSize || flag == 0) {
                try {
                    snsShareRecordRankingDAO.batchInsert(ssrrListTemp);
                    Object[] args = {m, ssrrListTemp.size(), rankingSize};
                    logger.error("snsShareRecordRankingDAO.batchInsert 第{}次写入成功,本次写入数据量：{},总数据量：{}", args);
                } catch (Exception e) {
                    Object[] args = {m, ssrrListTemp.size(), rankingSize};
                    logger.error("snsShareRecordRankingDAO.batchInsert 第{}次写入失败,本次写入数据量：{},总数据量：{}", args);
                    logger.error("snsShareRecordRankingDAO.batchInsert has error", e.getMessage());

                    // 记录该批次数据到缓存,后面重新插入
                    String field = String.format("promotionId_%d_page_%d", promotionId, m);
                    Integer rankingMix = ssrrListTemp.get(0).getRanking();
                    Integer rankingMax = ssrrListTemp.get(ssrrListTemp.size()-1).getRanking();
                    String range = String.format("%d_%d", rankingMix, rankingMax);
                    CacheUtil.hset(batchInsertErrorKey, field, range);
                } finally {
                    ssrrListTemp.clear();
                    n = 0;
                    m++;
                }
            }
        }
        sw2.stop();
        logger.error("分批次写入排行榜数据总耗时：" + sw2.getTotalTimeMillis() + "ms");
    } catch (Exception e) {
        logger.error("recordRanking has error", e.getMessage());
    } finally {
        // 释放分布式锁
        CacheUtil.del(distributedLockKey);
        logger.error("recordRanking end====================================");
    }
}

/**
 * 处理批次插入失败的数据
 * @param batchInsertErrorKey
 * @param promotionId
 * @return
 */
private boolean handleBatchInsertErrorList(String batchInsertErrorKey, Long promotionId) {
    logger.error("handleBatchInsertErrorList start==========batchInsertErrorKey：" + batchInsertErrorKey);
    boolean flag = false;

    StopWatch sw1 = new StopWatch();
    sw1.start();

    Map<String, String> batchInsertErrorMap = CacheUtil.hgetAll(batchInsertErrorKey);
    if (MapUtils.isNotEmpty(batchInsertErrorMap)) {
        // 将批量插入失败的数据重新插入一次
        for (Map.Entry<String, String> entry : batchInsertErrorMap.entrySet()) {
            String field = entry.getKey();
            String range = entry.getValue();
            String[] rangeArr = StringUtils.split(range, "_");
            Integer rankingMix = Integer.valueOf(StringUtils.replace(rangeArr[0], """, ""));
            Integer rankingMax = Integer.valueOf(StringUtils.replace(rangeArr[1], """, ""));
            // 获取排名范围内的排行榜数据
            List<SnsShareRecord> ssrList = snsShareRecordDAO.getListOfAccomplishedAndFilteredByPromotionIdAndRange(promotionId, rankingMix-1, rankingMax - rankingMix + 1);
            if (CollectionUtils.isEmpty(ssrList)) {
                Object[] args1 = {field};
                logger.error("handleBatchInsertErrorList batchInsertErrorMap中field:{}没有获取到该范围排行榜数据,下面将该field从缓存删除", args1);
                CacheUtil.hdel(batchInsertErrorKey, field);
                continue;
            }

            int size = ssrList.size();
            Object[] args2 = {field, size};
            logger.error("handleBatchInsertErrorList batchInsertErrorMap中field:{}对应的数据量:{},下面将进行批量插入", args2);

            List<SnsShareRecordRanking> ssrrListTemp = new ArrayList<>(size);
            SnsShareRecordRanking ssrrTemp = null;
            int ranking = rankingMix.intValue();//最小排名
            for (int i = 0; i < size; i++) {
                SnsShareRecord ssr = ssrList.get(i);
                ssrrTemp = new SnsShareRecordRanking();
                ssrrTemp.setShareRecordId(ssr.getId());
                ssrrTemp.setPromotionId(ssr.getPromotionId());
                ssrrTemp.setUserId(ssr.getUserId());
                ssrrTemp.setPrixResultState(ssr.getPrixResultState());
                ssrrTemp.setTotalRegisterDuration(ssr.getTotalRegisterDuration());
                ssrrTemp.setRanking(ranking + i);
                ssrrListTemp.add(ssrrTemp);
            }
            // 重新执行批量插入
            try {
                snsShareRecordRankingDAO.batchInsert(ssrrListTemp);
                // 插入成功之后删除该批次失败标记
                CacheUtil.hdel(batchInsertErrorKey, field);
                Object[] args3 = {size};
                logger.error("handleBatchInsertErrorList.batchInsert 写入成功,本次写入数据量：{}", args3);
            } catch (Exception e) {
                Object[] args4 = {size};
                logger.error("handleBatchInsertErrorList.batchInsert 写入失败,本次写入数据量：{}", args4);
                logger.error("handleBatchInsertErrorList.batchInsert has error", e.getMessage());
            } finally {
                ssrrListTemp.clear();
            }
        }
        flag = true;
    }

    sw1.stop();
    logger.error("handleBatchInsertErrorList end==========batchInsertErrorKey：" + batchInsertErrorKey + ",flag is " + flag + ",耗时：" + sw1.getTotalTimeMillis() + "ms");
    return flag;
}