【Java簡易爬蟲】使用Htmlunit爬蟲學校教務網課程表資訊

使用WebClient和htmlunit實作簡易爬蟲

import com.gargoylesoftware.htmlunit.WebClient;

提供了public

P getPage(final String url)方法獲得HtmlPage，

import com.gargoylesoftware.htmlunit.html.*;

包含了HtmlPage、HtmlForm、HtmlTextInput、HtmlPasswordInput、HtmlElement、DomElement等元素，

構造webclient物件

WebClient webClient= new WebClient();

無參默認是BrowserVersion.BEST_SUPPORTED，有參構造支持5種瀏覽器：

BrowserVersion.CHROME
BrowserVersion.EDGE
BrowserVersion.FIREFOX
BrowserVersion.FIREFOX_78
BrowserVersion.INTERNET_EXPLOER

使用webclient.getPage(String url)獲得頁面：

try {
   page = webClient.getPage(url);
} catch (IOException e) {
   e.printStackTrace();
}

利用webClient.getPage(url);方法，將其封裝成一個getHtmlPage靜態方法

private static class innerWebClient{
   private static final WebClient webClient = new WebClient();
}
public static HtmlPage getHtmlPage(String url){
   //呼叫此方法時加載WebClient
   WebClient webClient = innerWebClient.webClient;
   webClient.getOptions().setCssEnabled(false);
   //配置webClient
   webClient.getOptions().setCssEnabled(false);	//設定CSS是否生效
   webClient.getOptions().setJavaScriptEnabled(true);	//設定JS是否生效
   webClient.setAjaxController(new NicelyResynchronizingAjaxController());	//設定AJAX請求
   webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);	//設定是否拋出例外碼
   webClient.getOptions().setThrowExceptionOnScriptError(false);	//設定是否拋出腳本錯誤
   webClient.waitForBackgroundJavaScript(3*1000);	//設定等待JS毫秒數
   webClient.getCookieManager().setCookiesEnabled(true);	//設定是否支持Cookie
   HtmlPage page = null;
   try {
       page = webClient.getPage(url);
   } catch (IOException e) {
       e.printStackTrace();
   }
   return page;
}

在教務官網學期課表頁，拿到對應標簽的ID

登錄教務官網頁面：

靜態決議課程資訊方法：

//獲取周次集合
public static ArrayList<Integer> getWeekCount(String weekAndSection){
   ArrayList<Integer> weekList = new ArrayList<>();
   int index = weekAndSection.indexOf("(周)");
   if(index == -1){
       return new ArrayList<>();
   }
   String subWeek  = weekAndSection.substring(0, index);     //1-3,5,15,18
   String[] weekArr = new String[10];
   int idx = subWeek.indexOf(",");     //1或3
   int num = 0,n = 0;
   while (subWeek.contains(",")){
       weekArr[num] = subWeek.substring(0,idx);    //第一個逗號前面的內容，給陣列
       subWeek = subWeek.substring(idx+1);   //剩余內容
       n = subWeek.indexOf(",");
       idx = n;
       num++;
   }
   weekArr[num] = subWeek;
   for (String s : weekArr) {
       if(s!=null && !s.equals("")){
           if(s.contains("-")){
               int ix = s.indexOf("-");
               int begin = Integer.parseInt(s.substring(0,ix));
               int end = Integer.parseInt(s.substring(ix+1));
               for (int i = begin; i <= end; i++) {
                   weekList.add(i);
               }
           }else{
               weekList.add(Integer.parseInt(s));
           }
       }
   }
   return weekList;
}

//獲取節次集合
public static ArrayList<Integer> getSectionCount(String weekAndSection){
   int begin = weekAndSection.indexOf("[") + 1;
   int end = weekAndSection.indexOf("節");
   String section = weekAndSection.substring(begin, end);
   int len = section.length();
   String first = section.substring(0,2);
   String last = section.substring(len-2,len);
   ArrayList<Integer> sectionList = new ArrayList<>();
   int firstInt = Integer.parseInt(first);
   int lastInt = Integer.parseInt(last);
   for (int i = firstInt; i <= lastInt; i++) {
       sectionList.add(i);
   }
   return sectionList;
}

開始決議課程資訊

DomElement[][] domElements = new DomElement[7][6];  //7天，6個節次部分
String key = "";
//星期一~星期日：1-2~7-2
for (int i = 0;i < 7;i++){     //星期一到星期日
   for (int j = 0;j <= 5;j++){     //sectionIds[0]到sectionIds[5]
       if(j == 2){     //由于第5節為空，略過
           continue;
       }
       key = sectionIds[j] + "-" + (i+1) + "-2";
       if(page3.getElementById(key) == null){
           throw new NullPointerException("Key過期了！");
       }else{
           domElements[i][j] = page3.getElementById(key);
       }
       String course = domElements[i][j].asText();
       String temp[] = new String[10];
       int num = 0;
       int index;
       for (int g = 0; course.contains("---------------------"); g = g + index) {
           index = course.indexOf("---------------------");
           temp[num] = course.substring(0,index);
           course = course.substring(index+21);
           num++;
       }
       temp[num] = course;
       String[] courseInfo = new String[4];
       for (int k = 0;k < temp.length;k++) {
           if(temp[k] == null || temp[k].equals("") || temp[k].equals(" ")){
               continue;
           }
           if(temp[k].indexOf("\n") == 1){
               temp[k] = temp[k].substring(2);
           }
           ArrayList<Integer> weekList;
           ArrayList<Integer> sectionList;
           if(temp[k].contains("網路課")){
               temp[k] = temp[k].substring(0,temp[k].indexOf("\n"));
               courseInfo[0] = temp[k];
               weekList = null;
               sectionList = null;
           }else{
               int idx,cnum = 0;
               for(int h = 0; temp[k].contains("\n") && cnum <= 3;h = h+idx){
                   idx = temp[k].indexOf("\n");
                   courseInfo[cnum] = temp[k].substring(0,idx);
                   temp[k] = temp[k].substring(idx+1);
                   cnum++;
               }
               weekList = getWeekCount(courseInfo[2]);
               sectionList = getSectionCount(courseInfo[2]);
           }
           System.out.println("課程名===" + courseInfo[0]);
           System.out.println("教師名===" + courseInfo[1]);
           System.out.println("周次===" + weekList);
           System.out.println("節次===" + sectionList);
           System.out.println("地點===" + courseInfo[3]);
           System.out.println("星期" + (i+1));
       }
   }
}

輸出效果：

基于Uni-App實作的課程表小程式：