（Java）記一次通過API遞回分頁“爬取”網頁資料的開發經歷

public boolean getIpPoolOriginLinksInfo(HashMap<String, Object> commonPageMap, HashMap<String, String> headersMap,String charset){
        //介面呼叫頻率限制
        check(commonAPICheckData);
        String linkUrl = "https://xxx.xxx.com/api/v1/xxx/xxx";
        HashMap<String, Object> linkParamsMap = new HashMap<>();
        linkParamsMap.put("page",commonPageMap.get("page"));
        linkParamsMap.put("size",commonPageMap.get("size"));
        String httpLinkResponse = null;
        //封裝好的工具類，可以直接用apache的原生httpClient，也可以用Hutool的工具包
        httpLinkResponse = IpPoolHttpUtils.doGet(linkUrl,linkParamsMap,headersMap,charset);
        JSONObject linkJson = null;
        JSONArray linkArray = null;
        linkJson = JSON.parseObject(httpLinkResponse).getJSONObject("data");
        linkArray = linkJson.getJSONArray("resultList");
        if (linkArray != null){
            //遞回計數
            if (!commonPageMap.get("page").toString().equals("1")){
                commonPageMap.put("page","1");
            }
            // 每10條urlId，逐一遍歷
            for (Object linkObj : linkJson.getJSONArray("resultList")) {
                JSONObject info = JSON.parseObject(linkObj.toString());
                String urlId = info.getString("urlId");
                // 獲取到的每個urlId，根據urlId去獲取ip資訊（即內層的業務邏輯，我這里忽略，本質就是拿這里的引數傳到內層的方法中去）
                boolean flag = getIpPoolOriginRecords(urlId, commonPageMap, headersMap, charset);
                if (!flag){
                    break;
                }
            }
            Integer page = Integer.parseInt(linkParamsMap.get("page").toString());
            if (page <= Math.ceil(Integer.parseInt(linkJson.getString("total"))/10)){
                Integer newPage = Integer.parseInt(linkParamsMap.get("page").toString()) + 1;
                linkParamsMap.put("page", Integer.toString(newPage));
                // 遞回分頁拉取
                getIpPoolOriginLinksInfo(linkParamsMap,headersMap,charset);
            }
            else {
                return true;
            }
        }
        return true;
    }

/**
     * 呼叫頻率校驗，按照分鐘為單位校驗
     * @param commonAPICheckData
     */
    private void check(CommonAPICheckData commonAPICheckData){
        int minuteCount = commonAPICheckData.getMinuteCount();
        try {
            if (minuteCount < 2){
                //接近頻率限制時，休眠2秒
                Thread.sleep(2000);
            }else {
                commonAPICheckData.setMinuteCount(minuteCount-1);
            }
        } catch (InterruptedException e) {
            throw new RuntimeException("-------外部API呼叫頻率錯誤！--------");
        }
    }

@Data
public class CommonAPICheckData {

    /**
     * 每秒呼叫次數計數器，限制頻率：每秒鐘2次
     */
    private int secondCount = 3;

    /**
     * 每分鐘呼叫次數計數器，頻率限制：每分鐘100次
     */
    private int minuteCount = 100;

}

/**
   * 資料持久化
   * @param commonIpPool
   */
    private boolean getIpPoolDetails(CommonIpPool commonIpPool){
        try {
            //list元素去重，ip欄位唯一索引
            commonIpPoolService.insert(commonIpPool);
        } catch (Exception e) {
            if (e instanceof DuplicateKeyException){
                return true;
            }
            throw new RuntimeException(e.getMessage());
        }
        return true;
    }