Java多線程完成疾速切分文件的法式

Java多線程完成疾速切分文件的法式。本站提示廣大學習愛好者：（Java多線程完成疾速切分文件的法式）文章只能為提供參考，不一定能成為您想要的結果。以下是Java多線程完成疾速切分文件的法式正文

前段時光須要停止年夜批量數據導入，DBA給供給的是CVS文件，然則每一個CVS文件都好幾個GB年夜小，直接停止load，數據庫很慢還會發生內存缺乏的成績，為了完成這個功效，寫了個疾速切分文件的法式。

import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
 
import java.io.*;
import java.util.*;
import java.util.concurrent.*;
 
public class FileSplitUtil {
 
  private final static Logger log = LogManager.getLogger(FileSplitUtil.class);
  private static final long originFileSize = 1024 * 1024 * 100;// 100M
  private static final int blockFileSize = 1024 * 1024 * 64;// 避免中文亂碼，必需取2的N次方
  /**
   * CVS文件分隔符
   */
  private static final char cvsSeparator = '^';
  public static void main(String args[]){
    long start = System.currentTimeMillis();
    try {
      String fileName = "D:\\csvtest\\aa.csv";
      File sourceFile = new File(fileName);
      if (sourceFile.length() >= originFileSize) {
        String cvsFileName = fileName.replaceAll("\\\\", "/");
        FileSplitUtil fileSplitUtil = new FileSplitUtil();
        List<String> parts=fileSplitUtil.splitBySize(cvsFileName, blockFileSize);
        for(String part:parts){
          System.out.println("partName is:"+part);
        }
      }
      System.out.println("總文件長度"+sourceFile.length()+",拆分文件耗時:" + (System.currentTimeMillis() - start) + "ms.");
    }catch (Exception e){
      log.info(e.getStackTrace());
    }
 
  }
 
 
 
  /**
   * 拆分文件
   *
   * @param fileName 待拆分的完全文件名
   * @param byteSize 按若干字節年夜小拆分
   * @return 拆分後的文件名列表
   */
  public List<String> splitBySize(String fileName, int byteSize)
      throws IOException, InterruptedException {
    List<String> parts = new ArrayList<String>();
    File file = new File(fileName);
    int count = (int) Math.ceil(file.length() / (double) byteSize);
    int countLen = (count + "").length();
    RandomAccessFile raf = new RandomAccessFile(fileName, "r");
    long totalLen = raf.length();
    CountDownLatch latch = new CountDownLatch(count);
 
    for (int i = 0; i < count; i++) {
      String partFileName = file.getPath() + "."
          + leftPad((i + 1) + "", countLen, '0') + ".cvs";
      int readSize=byteSize;
      long startPos=(long)i * byteSize;
      long nextPos=(long)(i+1) * byteSize;
      if(nextPos>totalLen){
        readSize= (int) (totalLen-startPos);
      }
      new SplitRunnable(readSize, startPos, partFileName, file, latch).run();
      parts.add(partFileName);
    }
    latch.await();//期待一切文件寫完
    //因為切割時能夠會招致行被割斷，加工一切的的朋分文件,歸並行
    mergeRow(parts);
    return parts;
  }
 
  /**
   * 朋分處置Runnable
   *
   * @author supeidong
   */
  private class SplitRunnable implements Runnable {
    int byteSize;
    String partFileName;
    File originFile;
    long startPos;
    CountDownLatch latch;
    public SplitRunnable(int byteSize, long startPos, String partFileName,
               File originFile, CountDownLatch latch) {
      this.startPos = startPos;
      this.byteSize = byteSize;
      this.partFileName = partFileName;
      this.originFile = originFile;
      this.latch = latch;
    }
 
    public void run() {
      RandomAccessFile rFile;
      OutputStream os;
      try {
        rFile = new RandomAccessFile(originFile, "r");
        byte[] b = new byte[byteSize];
        rFile.seek(startPos);// 挪動指針到每“段”開首
        int s = rFile.read(b);
        os = new FileOutputStream(partFileName);
        os.write(b, 0, s);
        os.flush();
        os.close();
        latch.countDown();
      } catch (IOException e) {
        log.error(e.getMessage());
        latch.countDown();
      }
    }
  }
 
  /**
   * 歸並被割斷的行
   *
   * @param parts
   */
  private void mergeRow(List<String> parts) {
    List<PartFile> partFiles = new ArrayList<PartFile>();
    try {
      //組裝被切分表對象
      for (int i=0;i<parts.size();i++) {
        String partFileName=parts.get(i);
        File splitFileTemp = new File(partFileName);
        if (splitFileTemp.exists()) {
          PartFile partFile = new PartFile();
          BufferedReader reader=new BufferedReader(new InputStreamReader(new FileInputStream(splitFileTemp),"gbk"));
          String firstRow = reader.readLine();
          String secondRow = reader.readLine();
          String endRow = readLastLine(partFileName);
          partFile.setPartFileName(partFileName);
          partFile.setFirstRow(firstRow);
          partFile.setEndRow(endRow);
          if(i>=1){
            String prePartFile=parts.get(i - 1);
            String preEndRow = readLastLine(prePartFile);
            partFile.setFirstIsFull(getCharCount(firstRow+preEndRow)>getCharCount(secondRow));
          }
 
          partFiles.add(partFile);
          reader.close();
        }
      }
      //停止須要歸並的行的寫入
      for (int i = 0; i < partFiles.size() - 1; i++) {
        PartFile partFile = partFiles.get(i);
        PartFile partFileNext = partFiles.get(i + 1);
        StringBuilder sb = new StringBuilder();
        if (partFileNext.getFirstIsFull()) {
          sb.append("\r\n");
          sb.append(partFileNext.getFirstRow());
        } else {
          sb.append(partFileNext.getFirstRow());
        }
        writeLastLine(partFile.getPartFileName(),sb.toString());
      }
    } catch (Exception e) {
      log.error(e.getMessage());
    }
  }
 
  /**
   * 獲得某個字符湧現的次數
   * @param s
   * @return
   */
  private int getCharCount(String s) {
    int count = 0;
    for (int i = 0; i < s.length(); i++) {
      if (s.charAt(i) == cvsSeparator) {
        count++;
      }
    }
    return count;
  }
 
  /**
   * 采取BufferedInputStream方法讀取文件行數
   *
   * @param filename
   * @return
   */
  public int getFileRow(String filename) throws IOException {
    InputStream is = new BufferedInputStream(new FileInputStream(filename));
    byte[] c = new byte[1024];
    int count = 0;
    int readChars = 0;
    while ((readChars = is.read(c)) != -1) {
      for (int i = 0; i < readChars; ++i) {
        if (c[i] == '\n')
          ++count;
      }
    }
    is.close();
    return count;
  }
 
  /**
   * 讀取最初一行數據
   * @param filename
   * @return
   * @throws IOException
   */
  private String readLastLine(String filename) throws IOException {
    // 應用RandomAccessFile , 從後找最初一行數據
    RandomAccessFile raf = new RandomAccessFile(filename, "r");
    long len = raf.length();
    String lastLine = "";
    if(len!=0L) {
      long pos = len - 1;
      while (pos > 0) {
        pos--;
        raf.seek(pos);
        if (raf.readByte() == '\n') {
          lastLine = raf.readLine();
          lastLine=new String(lastLine.getBytes("8859_1"), "gbk");
          break;
        }
      }
    }
    raf.close();
    return lastLine;
  }
  /**
   * 修正最初一行數據
   * @param fileName
   * @param lastString
   * @return
   * @throws IOException
   */
  private void writeLastLine(String fileName,String lastString){
    try {
      // 翻開一個隨機拜訪文件流，按讀寫方法
      RandomAccessFile randomFile = new RandomAccessFile(fileName, "rw");
      // 文件長度，字節數
      long fileLength = randomFile.length();
      //將寫文件指針移到文件尾。
      randomFile.seek(fileLength);
      //此處必需加gbk，不然會湧現寫入亂碼
      randomFile.write(lastString.getBytes("gbk"));
      randomFile.close();
    } catch (IOException e) {
      log.error(e.getMessage());
    }
  }
  /**
   * 左填充
   *
   * @param str
   * @param length
   * @param ch
   * @return
   */
  public static String leftPad(String str, int length, char ch) {
    if (str.length() >= length) {
      return str;
    }
    char[] chs = new char[length];
    Arrays.fill(chs, ch);
    char[] src = str.toCharArray();
    System.arraycopy(src, 0, chs, length - src.length, src.length);
    return new String(chs);
  }
 
  /**
   * 歸並文件行外部類
   */
  class PartFile {
    private String partFileName;
    private String firstRow;
    private String endRow;
    private boolean firstIsFull;
 
    public String getPartFileName() {
      return partFileName;
    }
 
    public void setPartFileName(String partFileName) {
      this.partFileName = partFileName;
    }
 
    public String getFirstRow() {
      return firstRow;
    }
 
    public void setFirstRow(String firstRow) {
      this.firstRow = firstRow;
    }
 
    public String getEndRow() {
      return endRow;
    }
 
    public void setEndRow(String endRow) {
      this.endRow = endRow;
    }
 
    public boolean getFirstIsFull() {
      return firstIsFull;
    }
 
    public void setFirstIsFull(boolean firstIsFull) {
      this.firstIsFull = firstIsFull;
    }
  }
 
}

以上就是本文的全體內容，願望對年夜家進修java法式設計有所贊助。