FastDFS并發問題的排查經歷

	@RequestMapping(value = "https://www.cnblogs.com/batchDownloadForThesisCheck2", method = RequestMethod.POST)
	public synchronized void batchDownloadForThesisCheck2(@RequestBody List<FileInfoDto> fileInfoList)  {
		if(CollectionUtils.isEmpty(fileInfoList)){
			throw new EducationException("下載附件失敗");
		}
		List<FileInfoDto> fileInfoDtos = differentFileName(fileInfoList);
//		List<FileInfoDto> fileInfoDtos = new ArrayList<>();

//		for(int fi=0;fi<300;fi++){
//			FileInfoDto it = new FileInfoDto();
//			it.setFileName("20210115_小魚兒"+fi+"_jjlw.doc");
//			fileInfoDtos.add(it);
//		}
		String zipName = request.getParameter("zipName");
		if(StringUtils.isEmpty(zipName)) zipName = "批量下載";
		ZipOutputStream zipOS = null ;
		InputStream is = null;

		OutputStream os = null;
		// 計算百分值
		int index =1;
		int totalSize =CollectionUtils.isNotEmpty(fileInfoDtos) ? fileInfoDtos.size():1;


		try {
			response.setContentType("application/octet-stream; charset=UTF-8");
			response.setHeader("Access-Control-Expose-Headers", "fileName");
			response.setHeader("fileName", URLEncoder.encode(zipName, "UTF-8"));
			os = response.getOutputStream();
			zipOS = new ZipOutputStream(os);
			for (FileInfoDto info : fileInfoDtos) {
				// 機檢論文換名字，學號_姓名_jjlw命名
				String itemFileName= info.getFileName();
//				itemFileName = "S20020804005_陳明鑫_jjlw .docx";
				int secondShowIndex = Common.findNumber(itemFileName,"_",2);
				int firstShowIndex = Common.findNumber(itemFileName,"_",1);
				if("1".equals(info.getPaperToName())){
					// 檔案格式1：學校代碼_學號_LW.doc 2:學號_姓名_jjlw
					itemFileName = "10356_"+itemFileName.substring(0,firstShowIndex)+"_LW"+itemFileName.substring(itemFileName.lastIndexOf("."),itemFileName.length());
				}else{
					itemFileName = itemFileName.substring(0,secondShowIndex)+"_jjlw"+itemFileName.substring(itemFileName.lastIndexOf("."),itemFileName.length());
				}
				logger.error("已下載學生:{} "  ,itemFileName);
				zipOS.putNextEntry(new ZipEntry(itemFileName));
				try{
					is = fastDFS.downloadFile(info.getFileId());
//					is = fastDFS.downloadFile("group1/M00/00/C0/wKgjdWQYEJ-AbefsAdOyZXrKanw028.doc");
					int len = 0;
					byte[] buffer = new byte[1024*8];
					while ((len = is.read(buffer)) != -1) {
						zipOS.write(buffer, 0, len);
					}
//					is.close();

					// 計算進度,向下取整
					double nowProcess = Math.floor((index*100)/totalSize);
					logger.error("已下載”{}",index);
//					createProcessDownFile(nowProcess,"已下載"+nowProcess+"%",info.getTimeId(),info.getUserId());
					index ++;
				}catch(Exception ignored){

				}
				zipOS.flush();
//				zipOS.closeEntry();
			}
		} catch (IOException e) {
//			createProcessDownFile(100d,"批量下載發生錯誤",fileInfoList.get(0).getTimeId(),fileInfoList.get(0).getUserId());
			logger.error("批量下載發生錯誤: " + e.getMessage(), e);
		} finally {
			try {
				if (zipOS != null) {
					zipOS.closeEntry();
					zipOS.close();
				}
				if (os != null) os.close();
//				createProcessDownFile(100d,"已下載100%",fileInfoList.get(0).getTimeId(),fileInfoList.get(0).getUserId());
				logger.warn("關倍訓檢下載 :"  );
			} catch (IOException e) {
//				createProcessDownFile(100d,"批量下載發生錯誤,關閉檔案流失敗",fileInfoList.get(0).getTimeId(),fileInfoList.get(0).getUserId());
				logger.warn("關閉檔案流失敗, cause by :" + e.getMessage());

			}
//			finally {
//				createProcessDownFile(100d,"已下載100%",fileInfoList.get(0).getTimeId(),fileInfoList.get(0).getUserId());
//				logger.warn("關倍訓檢下載 :"  );
//			}

		}
	}

package fastdfs.config;

import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.InetSocketAddress;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

import javax.annotation.PostConstruct;

import org.apache.commons.lang.StringUtils;
import org.csource.common.NameValuePair;
import org.csource.fastdfs.ClientGlobal;
import org.csource.fastdfs.ProtoCommon;
import org.csource.fastdfs.StorageClient1;
import org.csource.fastdfs.StorageServer;
import org.csource.fastdfs.TrackerClient;
import org.csource.fastdfs.TrackerGroup;
import org.csource.fastdfs.TrackerServer;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.util.CollectionUtils;

import com.xx.commons.exception.EducationException;

import lombok.extern.slf4j.Slf4j;



/**
 * FastDfs檔案系統工具類
 * 連接Fast
 * 上傳圖片 
 * 回傳上傳之后的路徑 用此路徑就能訪問此圖片
 *    group1/M00/00/01/wKjIgFWOYc6APpjAAAD-qk29i78248.jpg
 *
 */
@Slf4j
public class FastDFS {

    @Autowired
    private  FastDFSProperty fastDFSProperty;

    /**
     * 跟蹤器
     */
    private TrackerServer trackerServer;
    
    /**
     * 存盤器
     */
    private StorageServer storageServer;
    
    /**
     * 默認編碼
     */
    private static final String DEFAULT_ENCODING = "UTF-8";

    @PostConstruct
    public void init(){
        try {
            ClientGlobal.setG_charset(DEFAULT_ENCODING);
            ClientGlobal.setG_connect_timeout(fastDFSProperty.getConnect_timeout());
            ClientGlobal.setG_network_timeout(fastDFSProperty.getNetwork_timeout());
            ClientGlobal.setG_secret_key(fastDFSProperty.getSecret_key());
            ClientGlobal.setG_tracker_http_port(fastDFSProperty.getTracker_http_port());

            String tracker_server = fastDFSProperty.getTracker_server();
            InetSocketAddress isadd = new InetSocketAddress(
                    tracker_server.substring(0, tracker_server.indexOf(':')), 
                    Integer.parseInt(tracker_server.substring(tracker_server.indexOf(':') + 1, tracker_server.length())));
            InetSocketAddress[] tracker_servers  = {isadd};
            ClientGlobal.setG_tracker_group(new TrackerGroup(tracker_servers));

            TrackerClient trackerClient = new TrackerClient(ClientGlobal.g_tracker_group);
            trackerServer = trackerClient.getConnection();
            if (trackerServer == null) {
                throw new EducationException("getConnection return null");
            }
            storageServer = trackerClient.getStoreStorage(trackerServer);
            if (storageServer == null) {
                throw new EducationException("getStoreStorage return null");
            }
            ProtoCommon.activeTest(storageServer.getSocket());
        } catch (Exception e) {
            throw new EducationException("初始化 fastdfs 配置失敗", e);
        }
    }

    /**
     * 
     * @param file
     *            檔案
     * @param fileName
     *            檔案名
     * @return 回傳Null則為失敗
     */
    public String uploadFile(File file, String fileName) {
        InputStream fis = null;
        try {
            NameValuePair[] meta_list = null; 
            fis = Files.newInputStream(file.toPath());
            byte[] file_buff = new byte[1024];
            int len = fis.available();
            file_buff = new byte[len];
            while (fis.read(file_buff) > 0) {
                break;
            }
            StorageClient1 storageClient1 = new StorageClient1(trackerServer, storageServer);
            String fileid = storageClient1.upload_file1(file_buff, getFileExt(fileName), meta_list);
            return fileid;
        } catch (Exception ex) {
            throw new EducationException("上傳檔案錯誤",ex);
        } finally{
            if (fis != null) {
                try {
                    fis.close();
                } catch (IOException e) {
                    log.error("Close {} InputStream failed", fis);
                }
            }
        }
    }

    /**
     * 上傳檔案
     * @param bytes
     * @param name
     * @param size
     * @return
     */
    public String uploadFile(byte[] bytes, String name, Long size) {
        try {
            //擴展名
            String ext = name.substring(name.lastIndexOf('.')+1);
            NameValuePair[] meta_list = new NameValuePair[3];
            meta_list[0] = new NameValuePair("filename",name);
            meta_list[1] = new NameValuePair("fileext",ext);
            meta_list[2] = new NameValuePair("filesize",String.valueOf(size));
            StorageClient1 storageClient1 = new StorageClient1(trackerServer, storageServer);
            return storageClient1.upload_file1(bytes, ext, meta_list);
        } catch (Exception ex) {
            throw new EducationException("上傳檔案錯誤",ex);
        }
    }

    /**
     * 根據組名和遠程檔案名來洗掉一個檔案
     * 
     * @param groupName
     *            例如 "group1" 如果不指定該值，默認為group1
     * @param fileName
     *            例如"M00/00/00/wKgxgk5HbLvfP86RAAAAChd9X1Y736.jpg"
     * @return 0為成功，非0為失敗，具體為錯誤代碼
     */
    public int deleteFile(String groupName, String fileName) {
        try {
            StorageClient1 storageClient1 = new StorageClient1(trackerServer, storageServer);
            return storageClient1.delete_file(StringUtils.isBlank(groupName)? "group1" : groupName, fileName);
        } catch (Exception ex) {
            throw new EducationException("return null",ex);
        }
    }

    /**
     * 根據fileId來洗掉一個檔案（我們現在用的就是這樣的方式，上傳檔案時直接將fileId保存在了資料庫中）
     * 
     * @param fileId
     *            file_id原始碼中的解釋file_id the file id(including group name and filename);例如 group1/M00/00/00/ooYBAFM6MpmAHM91AAAEgdpiRC0012.xml
     * @return 0為成功，非0為失敗，具體為錯誤代碼
     */
    public int deleteFile(String fileId) {
        try {
            StorageClient1 storageClient1 = new StorageClient1(trackerServer, storageServer);
            return storageClient1.delete_file1(fileId);
        } catch (Exception ex) {
            throw new EducationException("洗掉檔案錯誤",ex);
        }
    }

    /**
     * 修改一個已經存在的檔案
     * 
     * @param oldFileId
     *            原來舊檔案的fileId, file_id原始碼中的解釋file_id the file id(including group name and filename);例如 group1/M00/00/00/ooYBAFM6MpmAHM91AAAEgdpiRC0012.xml
     * @param file
     *            新檔案
     * @param filePath
     *            新檔案路徑
     * @return 回傳空則為失敗
     */
    public String modifyFile(String oldFileId, File file, String filePath) {
        String fileid = null;
        try {
            // 先上傳
            fileid = uploadFile(file, filePath);
            if (fileid == null) {
                return null;
            }
            // 再洗掉
            int delResult = deleteFile(oldFileId);
            if (delResult != 0) {
                return null;
            }
        } catch (Exception ex) {
            throw new EducationException("修改一個已經存在的檔案錯誤",ex);
        }
        return fileid;
    }

    /**
     * 檔案下載
     * 
     * @param fileId
     * @return 回傳一個流
     */
    public InputStream downloadFile(String fileId) {
        try {
            StorageClient1 storageClient1 = new StorageClient1(trackerServer, storageServer);
            byte[] bytes = storageClient1.download_file1(fileId);
            return new ByteArrayInputStream(bytes);
        } catch (Exception ex) {
            throw new EducationException("檔案下載錯誤",ex);
        }
    }

    /**
     * 
     * @param fileId
     * @return 回傳一個位元組陣列
     */
    public byte[] downloadFileToByte(String fileId) {
        try {
            StorageClient1 storageClient1 = new StorageClient1(trackerServer, storageServer);
            return storageClient1.download_file1(fileId);
        } catch (Exception ex) {
            throw new EducationException("檔案下載錯誤",ex);
        }
    }

    /**  
     * 批量檔案下載，在map中給出檔案名：fileName， 檔案對應路徑：filePath；   
     * 方法回傳  由所有檔案生成的壓縮包ZIP   
     * @param fileList 在map中給出檔案名：fileName， 檔案對應路徑：filePath；  
     * @return 回傳一個流  
     */  
    public InputStream downloadFile(List<Map<String, String>> fileList){  
        if(CollectionUtils.isEmpty(fileList)){  
            throw new EducationException("檔案下載錯誤，fileList為空！");  
        }  
        Date date = new Date();  
        long timeStr = date.getTime();  
        
        StorageClient1 storageClient1 = new StorageClient1(trackerServer, storageServer);
        
        try(ZipOutputStream zos = new ZipOutputStream(Files.newOutputStream(Paths.get(".", timeStr+".zip")));) {  
              
            ZipEntry entry;  
            int count, bufferLen = 1024;    
            byte data[] = new byte[bufferLen];  
            for(Map<String, String> map : fileList){  
                entry = new ZipEntry(map.get("fileName"));  
                  
                zos.putNextEntry(entry);  
                byte[] bytes = storageClient1.download_file1(map.get("filePath"));  
                try(BufferedInputStream bis = new BufferedInputStream(new ByteArrayInputStream(bytes));){  
                    while ((count = bis.read(data, 0, bufferLen)) != -1) {    
                        zos.write(data, 0, count);    
                    }    
                    zos.closeEntry();  
                }  
            }  
            return Files.newInputStream(Paths.get(".", timeStr+".zip"));  
        } catch (Exception ex) {  
            throw new EducationException("檔案下載錯誤", ex);  
        }  
    } 

    /**
     * 獲取檔案后綴名（不帶點）.
     * 
     * @return 如："jpg" or "".
     */
    private  String getFileExt(String fileName) {
        if (StringUtils.isBlank(fileName) || !fileName.contains(".")) {
            return "";
        } else {
            return fileName.substring(fileName.lastIndexOf('.') + 1); // 不帶最后的點
        }
    }
}