Java读带有BOM的UTF-8文件乱码

推荐原创

359392120 2013-12-09 02:28:31 博主文章分类：Java ©著作权

文章标签 Java DOM UIF-8文件乱码 文章分类 Java 后端开发

©著作权归作者所有：来自51CTO博客作者359392120的原创作品，请联系作者获取转载授权，否则将追究法律责任

通过java写的UTF-8文件，使用Java可以正确的读，但是如果用记事本将相同的内容使用UTF-8格式保存，则在使用程序读取是会从文件中多读出一个不可见字符，多一个?

实例：

新建一个文本：内容为“测试BOM”的txt文本，另存为UTF-8.

处理带BOM的类UnicodeReader

package com.java.io;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PushbackInputStream;
import java.io.Reader;
/**
version: 1.1 / 2007-01-25
- changed BOM recognition ordering (longer boms first)
网络地址：http://koti.mbnet.fi/akini/java/unicodereader/UnicodeReader.java.txt
Original pseudocode   : Thomas Weidenfeller
Implementation tweaked: Aki Nieminen
http://www.unicode.org/unicode/faq/utf_bom.html
BOMs:
  00 00 FE FF    = UTF-32, big-endian
  FF FE 00 00    = UTF-32, little-endian
  EF BB BF       = UTF-8,
  FE FF          = UTF-16, big-endian
  FF FE          = UTF-16, little-endian
Win2k Notepad:
  Unicode format = UTF-16LE
***/
/**
* Generic unicode textreader, which will use BOM mark
* to identify the encoding to be used. If BOM is not found
* then use a given default or system encoding.
*/
public class UnicodeReader extends Reader {
  PushbackInputStream internalIn;
  InputStreamReader   internalIn2 = null;
  String              defaultEnc;
  private static final int BOM_SIZE = 4;
  /**
   *
   * @param in  inputstream to be read
   * @param defaultEnc default encoding if stream does not have
   *                   BOM marker. Give NULL to use system-level default.
   */
  UnicodeReader(InputStream in, String defaultEnc) {
     internalIn = new PushbackInputStream(in, BOM_SIZE);
     this.defaultEnc = defaultEnc;
  }
  public String getDefaultEncoding() {
     return defaultEnc;
  }
  /**
   * Get stream encoding or NULL if stream is uninitialized.
   * Call init() or read() method to initialize it.
   */
  public String getEncoding() {
     if (internalIn2 == null) return null;
     return internalIn2.getEncoding();
  }
  /**
   * Read-ahead four bytes and check for BOM marks. Extra bytes are
   * unread back to the stream, only BOM bytes are skipped.
   */
  protected void init() throws IOException {
     if (internalIn2 != null) return;
     String encoding;
     byte bom[] = new byte[BOM_SIZE];
     int n, unread;
     n = internalIn.read(bom, 0, bom.length);
     if ( (bom[0] == (byte)0x00) && (bom[1] == (byte)0x00) &&
                 (bom[2] == (byte)0xFE) && (bom[3] == (byte)0xFF) ) {
        encoding = "UTF-32BE";
        unread = n - 4;
     } else if ( (bom[0] == (byte)0xFF) && (bom[1] == (byte)0xFE) &&
                 (bom[2] == (byte)0x00) && (bom[3] == (byte)0x00) ) {
        encoding = "UTF-32LE";
        unread = n - 4;
     } else if (  (bom[0] == (byte)0xEF) && (bom[1] == (byte)0xBB) &&
           (bom[2] == (byte)0xBF) ) {
        encoding = "UTF-8";
        unread = n - 3;
     } else if ( (bom[0] == (byte)0xFE) && (bom[1] == (byte)0xFF) ) {
        encoding = "UTF-16BE";
        unread = n - 2;
     } else if ( (bom[0] == (byte)0xFF) && (bom[1] == (byte)0xFE) ) {
        encoding = "UTF-16LE";
        unread = n - 2;
     } else {
        // Unicode BOM mark not found, unread all bytes
        encoding = defaultEnc;
        unread = n;
     }  
     //System.out.println("read=" + n + ", unread=" + unread);
     if (unread > 0) internalIn.unread(bom, (n - unread), unread);
     // Use given encoding
     if (encoding == null) {
        internalIn2 = new InputStreamReader(internalIn);
     } else {
        internalIn2 = new InputStreamReader(internalIn, encoding);
     }
  }
  public void close() throws IOException {
     init();
     internalIn2.close();
  }
  public int read(char[] cbuf, int off, int len) throws IOException {
     init();
     return internalIn2.read(cbuf, off, len);
  }
}

测试类

package com.java.io;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
public class BomRead {
    /**
     * 读带有BOM的UTF-8文件乱码
     * @param args
     */
    public static void main(String[] args)throws Exception {
         File file  = new File("E:\\JS_Exercise\\JavaExercise\\BOM.txt");
         FileInputStream in = new FileInputStream(file);
         BufferedReader br = new BufferedReader(new InputStreamReader(in, "UTF-8"));
         String line = null;
         System.out.println("处理前：");
         while((line = br.readLine()) != null){
             System.out.println(line);
         }
                                                                      
          File file2  = new File("E:\\JS_Exercise\\JavaExercise\\BOM.txt");
          FileInputStream in2 = new FileInputStream(file2);
          BufferedReader br2 = new BufferedReader(new UnicodeReader(in2, "UTF-8"));
          String line2 = null;
          System.out.println("处理后：");
          while((line2 = br2.readLine()) != null){
              System.out.println(line2);
          }
    }
}

输出结果

处理前：

?测试BOM

处理后：

测试BOM

另一种解决方式

从目前来看1.6只是解决了读取带有BOM文件失败的问题，还是不能区别处理有BOM和无BOM的UTF-8编码的文件，从Bug ID:4508058里的描述可以看出，这个问题将作为一个不会修改的问题关闭，对于BOM编码的识别将由应用程序自己来处理，原因可从另处一个bug处查看到，因为Unicode对于BOM的编码的规定可能发生变化。也就是说对于一个UTF-8的文件，应用程序需要知道这个文件有没有写BOM，然后自己决定处理BOM的方式。