(Java port)
* Aaron Madlon-Kay (emoji fix)
*
* Alternatively, the contents of this file may be used under the terms of
* either the GNU General Public License Version 2 or later (the "GPL"), or
* the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
* in which case the provisions of the GPL or the LGPL are applicable instead
* of those above. If you wish to allow use of your version of this file only
* under the terms of either the GPL or the LGPL, and not to allow others to
* use your version of this file under the terms of the MPL, indicate your
* decision by deleting the provisions above and replace them with the notice
* and other provisions required by the GPL or the LGPL. If you do not delete
* the provisions above, a recipient may use your version of this file under
* the terms of any one of the MPL, the GPL or the LGPL.
*
* ***** END LICENSE BLOCK ***** */
package org.mozilla.universalchardet.prober.statemachine;
import static org.mozilla.universalchardet.prober.statemachine.PkgInt.INDEX_SHIFT_4BITS;
import static org.mozilla.universalchardet.prober.statemachine.PkgInt.SHIFT_MASK_4BITS;
import static org.mozilla.universalchardet.prober.statemachine.PkgInt.BIT_SHIFT_4BITS;
import static org.mozilla.universalchardet.prober.statemachine.PkgInt.UNIT_MASK_4BITS;
import org.mozilla.universalchardet.Constants;
public class UTF8SMModel extends SMModel {
////////////////////////////////////////////////////////////////
// constants
////////////////////////////////////////////////////////////////
public static final int UTF8_CLASS_FACTOR = 16;
////////////////////////////////////////////////////////////////
// methods
////////////////////////////////////////////////////////////////
public UTF8SMModel() {
super(
new PkgInt(INDEX_SHIFT_4BITS, SHIFT_MASK_4BITS, BIT_SHIFT_4BITS, UNIT_MASK_4BITS, utf8ClassTable),
UTF8_CLASS_FACTOR,
new PkgInt(INDEX_SHIFT_4BITS, SHIFT_MASK_4BITS, BIT_SHIFT_4BITS, UNIT_MASK_4BITS, utf8StateTable),
utf8CharLenTable,
Constants.CHARSET_UTF_8
);
}
////////////////////////////////////////////////////////////////
// constants continued
////////////////////////////////////////////////////////////////
private static int[] utf8ClassTable = new int[] {
// PkgInt.pack4bits(0,1,1,1,1,1,1,1), // 00 - 07
PkgInt.pack4bits(1,1,1,1,1,1,1,1), // 00 - 07 //allow 0x00 as a legal value
PkgInt.pack4bits(1,1,1,1,1,1,0,0), // 08 - 0f
PkgInt.pack4bits(1,1,1,1,1,1,1,1), // 10 - 17
PkgInt.pack4bits(1,1,1,0,1,1,1,1), // 18 - 1f
PkgInt.pack4bits(1,1,1,1,1,1,1,1), // 20 - 27
PkgInt.pack4bits(1,1,1,1,1,1,1,1), // 28 - 2f
PkgInt.pack4bits(1,1,1,1,1,1,1,1), // 30 - 37
PkgInt.pack4bits(1,1,1,1,1,1,1,1), // 38 - 3f
PkgInt.pack4bits(1,1,1,1,1,1,1,1), // 40 - 47
PkgInt.pack4bits(1,1,1,1,1,1,1,1), // 48 - 4f
PkgInt.pack4bits(1,1,1,1,1,1,1,1), // 50 - 57
PkgInt.pack4bits(1,1,1,1,1,1,1,1), // 58 - 5f
PkgInt.pack4bits(1,1,1,1,1,1,1,1), // 60 - 67
PkgInt.pack4bits(1,1,1,1,1,1,1,1), // 68 - 6f
PkgInt.pack4bits(1,1,1,1,1,1,1,1), // 70 - 77
PkgInt.pack4bits(1,1,1,1,1,1,1,1), // 78 - 7f
PkgInt.pack4bits(2,2,2,2,3,3,3,3), // 80 - 87
PkgInt.pack4bits(4,4,4,4,4,4,4,4), // 88 - 8f
PkgInt.pack4bits(4,4,4,4,4,4,4,4), // 90 - 97
PkgInt.pack4bits(4,4,4,4,4,4,4,4), // 98 - 9f
PkgInt.pack4bits(5,5,5,5,5,5,5,5), // a0 - a7
PkgInt.pack4bits(5,5,5,5,5,5,5,5), // a8 - af
PkgInt.pack4bits(5,5,5,5,5,5,5,5), // b0 - b7
PkgInt.pack4bits(5,5,5,5,5,5,5,5), // b8 - bf
PkgInt.pack4bits(0,0,6,6,6,6,6,6), // c0 - c7
PkgInt.pack4bits(6,6,6,6,6,6,6,6), // c8 - cf
PkgInt.pack4bits(6,6,6,6,6,6,6,6), // d0 - d7
PkgInt.pack4bits(6,6,6,6,6,6,6,6), // d8 - df
PkgInt.pack4bits(7,8,8,8,8,8,8,8), // e0 - e7
PkgInt.pack4bits(8,8,8,8,8,9,8,8), // e8 - ef
PkgInt.pack4bits(10,11,11,11,11,11,11,11), // f0 - f7
PkgInt.pack4bits(12,13,13,13,14,15,0,0) // f8 - ff
};
private static int[] utf8StateTable = new int[] {
PkgInt.pack4bits(ERROR,START,ERROR,ERROR,ERROR,ERROR, 12, 10),//00-07
PkgInt.pack4bits( 9, 11, 8, 7, 6, 5, 4, 3),//08-0f
PkgInt.pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//10-17
PkgInt.pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//18-1f
PkgInt.pack4bits(ITSME,ITSME,ITSME,ITSME,ITSME,ITSME,ITSME,ITSME),//20-27
PkgInt.pack4bits(ITSME,ITSME,ITSME,ITSME,ITSME,ITSME,ITSME,ITSME),//28-2f
PkgInt.pack4bits(ERROR,ERROR, 5, 5, 5, 5,ERROR,ERROR),//30-37
PkgInt.pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//38-3f
PkgInt.pack4bits(ERROR,ERROR,ERROR, 5, 5, 5,ERROR,ERROR),//40-47
PkgInt.pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//48-4f
PkgInt.pack4bits(ERROR,ERROR, 7, 7, 7, 7,ERROR,ERROR),//50-57
PkgInt.pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//58-5f
PkgInt.pack4bits(ERROR,ERROR,ERROR,ERROR, 7, 7,ERROR,ERROR),//60-67
PkgInt.pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//68-6f
PkgInt.pack4bits(ERROR,ERROR, 9, 9, 9, 9,ERROR,ERROR),//70-77
PkgInt.pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//78-7f
PkgInt.pack4bits(ERROR,ERROR,ERROR,ERROR, 9, 9,ERROR,ERROR),//80-87 // Allow emoji
PkgInt.pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//88-8f
PkgInt.pack4bits(ERROR,ERROR, 12, 12, 12, 12,ERROR,ERROR),//90-97
PkgInt.pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//98-9f
PkgInt.pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR, 12,ERROR,ERROR),//a0-a7
PkgInt.pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//a8-af
PkgInt.pack4bits(ERROR,ERROR, 12, 12, 12,ERROR,ERROR,ERROR),//b0-b7
PkgInt.pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//b8-bf
PkgInt.pack4bits(ERROR,ERROR,START,START,START,START,ERROR,ERROR),//c0-c7
PkgInt.pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR) //c8-cf
};
private static int[] utf8CharLenTable = new int[] {
0, 1, 0, 0, 0, 0, 2, 3,
3, 3, 4, 4, 5, 5, 6, 6
};
}
juniversalchardet-2.5.0/src/test/ 0000775 0000000 0000000 00000000000 14625121707 0017006 5 ustar 00root root 0000000 0000000 juniversalchardet-2.5.0/src/test/java/ 0000775 0000000 0000000 00000000000 14625121707 0017727 5 ustar 00root root 0000000 0000000 juniversalchardet-2.5.0/src/test/java/org/ 0000775 0000000 0000000 00000000000 14625121707 0020516 5 ustar 00root root 0000000 0000000 juniversalchardet-2.5.0/src/test/java/org/mozilla/ 0000775 0000000 0000000 00000000000 14625121707 0022165 5 ustar 00root root 0000000 0000000 juniversalchardet-2.5.0/src/test/java/org/mozilla/universalchardet/ 0000775 0000000 0000000 00000000000 14625121707 0025530 5 ustar 00root root 0000000 0000000 juniversalchardet-2.5.0/src/test/java/org/mozilla/universalchardet/BOMTest.java 0000664 0000000 0000000 00000002172 14625121707 0027652 0 ustar 00root root 0000000 0000000 package org.mozilla.universalchardet;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import org.junit.Assert;
import org.junit.Test;
public class BOMTest {
private static final String TEST_STRING =
"========================================================================";
@Test
public void testUTF8 () throws IOException {
Assert.assertEquals(TEST_STRING, getFirstLine("utf8.txt"));
}
@Test
public void testUTF8N () throws IOException {
Assert.assertEquals(TEST_STRING, getFirstLine("utf8n.txt"));
}
@Test
public void testUTF16LE () throws IOException {
Assert.assertEquals(TEST_STRING, getFirstLine("utf16le.txt"));
}
private String getFirstLine(String testFileName) throws IOException{
String fileName = "src/test/resources/" + testFileName;
File file = new File(fileName);
BufferedReader reader = null;
try {
reader = ReaderFactory.createBufferedReader(file);
// return first line
return reader.readLine();
}
finally {
if (reader != null) {
reader.close();
}
}
}
}
BasicFileEncodingDetectionTest.java 0000664 0000000 0000000 00000003676 14625121707 0034317 0 ustar 00root root 0000000 0000000 juniversalchardet-2.5.0/src/test/java/org/mozilla/universalchardet package org.mozilla.universalchardet;
import java.io.File;
import java.io.IOException;
import org.junit.Assert;
import org.junit.Test;
public class BasicFileEncodingDetectionTest {
public BasicFileEncodingDetectionTest() throws IOException {
super();
}
@Test
public void testASCII() throws IOException {
Assert.assertEquals("US-ASCII", getFileEncoding("ascii.txt"));
}
@Test
public void testUTF8 () throws IOException {
Assert.assertEquals("UTF-8", getFileEncoding("utf8.txt"));
}
@Test
public void testUTF8N () throws IOException {
Assert.assertEquals("UTF-8", getFileEncoding("utf8n.txt"));
}
@Test
public void testUTF16LE () throws IOException {
Assert.assertEquals("UTF-16LE", getFileEncoding("utf16le.txt"));
}
@Test
public void testShifJis () throws IOException {
Assert.assertEquals("SHIFT_JIS", getFileEncoding("shiftjis.txt"));
}
@Test
public void testEUC () throws IOException {
Assert.assertEquals("EUC-JP", getFileEncoding("euc.txt"));
}
@Test
public void testISO2022JP () throws IOException {
Assert.assertEquals("ISO-2022-JP", getFileEncoding("iso2022jp.txt"));
}
@Test
public void testBIG5 () throws IOException {
Assert.assertEquals("BIG5", getFileEncoding("big5.txt"));
}
@Test
public void testEUCTW () throws IOException {
Assert.assertEquals("EUC-TW", getFileEncoding("euctw.txt"));
}
@Test
public void testEUCKR() throws IOException {
Assert.assertEquals("EUC-KR", getFileEncoding("euckr.txt"));
}
@Test
public void testWindows1255 () throws IOException {
Assert.assertEquals("WINDOWS-1255", getFileEncoding("windows1255.txt"));
}
@Test
public void testUTF8Emoji () throws IOException {
Assert.assertEquals("UTF-8", getFileEncoding("utf8n-emoji.txt"));
}
private String getFileEncoding(String testFileName) throws IOException{
String fileName = "src/test/resources/" + testFileName;
return UniversalDetector.detectCharset(new File(fileName));
}
}
BasicStreamEncodingDetectionTest.java 0000664 0000000 0000000 00000005126 14625121707 0034663 0 ustar 00root root 0000000 0000000 juniversalchardet-2.5.0/src/test/java/org/mozilla/universalchardet package org.mozilla.universalchardet;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import org.junit.Assert;
import org.junit.Test;
public class BasicStreamEncodingDetectionTest {
@Test
public void testUTF8 () throws IOException {
Assert.assertEquals("UTF-8", getFileEncoding("utf8.txt"));
}
@Test
public void testUTF8N () throws IOException {
Assert.assertEquals("UTF-8", getFileEncoding("utf8n.txt"));
}
@Test
public void testUTF16LE () throws IOException {
Assert.assertEquals("UTF-16LE", getFileEncoding("utf16le.txt"));
}
@Test
public void testShifJis () throws IOException {
Assert.assertEquals("SHIFT_JIS", getFileEncoding("shiftjis.txt"));
}
@Test
public void testEUC () throws IOException {
Assert.assertEquals("EUC-JP", getFileEncoding("euc.txt"));
}
@Test
public void testISO2022JP () throws IOException {
Assert.assertEquals("ISO-2022-JP", getFileEncoding("iso2022jp.txt"));
}
@Test
public void testBIG5 () throws IOException {
Assert.assertEquals("BIG5", getFileEncoding("big5.txt"));
}
@Test
public void testEUCTW () throws IOException {
Assert.assertEquals("EUC-TW", getFileEncoding("euctw.txt"));
}
@Test
public void testEUCKR() throws IOException {
Assert.assertEquals("EUC-KR", getFileEncoding("euckr.txt"));
}
@Test
public void testWindows1255 () throws IOException {
Assert.assertEquals("WINDOWS-1255", getFileEncoding("windows1255.txt"));
}
@Test
public void testUTF8Emoji () throws IOException {
Assert.assertEquals("UTF-8", getFileEncoding("utf8n-emoji.txt"));
}
private String getFileEncoding(String testFileName) throws IOException{
File file = new File("src/test/resources/" + testFileName);
EncodingDetectorInputStream edis = null;
EncodingDetectorOutputStream edos = null;
try {
edis = new EncodingDetectorInputStream(new BufferedInputStream(Files.newInputStream(file.toPath())));
edos = new EncodingDetectorOutputStream(NullOutputStream.NULL_OUTPUT_STREAM);
byte[] buffer = new byte[1024];
int read = 0;
while (( read = edis.read(buffer)) > 0){
edos.write(buffer, 0, read);
}
}
finally {
edos.close();
edis.close();
}
String encodingRead = edis.getDetectedCharset();
String encodingWrite = edos.getDetectedCharset();
Assert.assertNotNull(encodingRead);
Assert.assertNotNull(encodingWrite);
Assert.assertTrue(encodingRead.equals(encodingWrite));
return encodingRead;
}
}
Bug20LatinDetectedAsMaccyrillicTest.java 0000664 0000000 0000000 00000002626 14625121707 0035133 0 ustar 00root root 0000000 0000000 juniversalchardet-2.5.0/src/test/java/org/mozilla/universalchardet package org.mozilla.universalchardet;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
public class Bug20LatinDetectedAsMaccyrillicTest {
private static final String TEST_STRING = "ÄÜÖßäöü,Name1ÄÜÖßäöü,Name2ÄÜÖßäöü,Name3ÄÜÖßäöü,StreetÄÜÖßäöü,MÄÜÖßäöü,DE,80080,München,ContactÄÜÖßäöü,+49(0)ÄÜÖßäöü,ÄÜÖßäöü@gls-itservices.com,CommentÄÜÖßäöü,+49,(0)98,765,432,BlÄÜÖßäöü";
@Test
@Ignore("Bug not fixed yet")
public void testFile () throws IOException {
File testFile = new File("src/test/resources/bug20-example-latin.txt");
String originalEncoding = UniversalDetector.detectCharset(testFile);
Assert.assertEquals("WINDOWS-1252", originalEncoding);
}
@Test
@Ignore("Bug not fixed yet")
public void testLatin() throws IOException {
UniversalDetector detector = new UniversalDetector();
detector.handleData(TEST_STRING.getBytes(Charset.forName("WINDOWS-1252")));
detector.dataEnd();
Assert.assertEquals("WINDOWS-1252", detector.getDetectedCharset());
}
@Test
public void testUTF8() {
UniversalDetector detector = new UniversalDetector();
detector.handleData(TEST_STRING.getBytes(Charset.forName("UTF-8")));
detector.dataEnd();
Assert.assertEquals("UTF-8", detector.getDetectedCharset());
}
}
juniversalchardet-2.5.0/src/test/java/org/mozilla/universalchardet/Bug33USASCIIToGenerous.java 0000664 0000000 0000000 00000001454 14625121707 0032316 0 ustar 00root root 0000000 0000000 package org.mozilla.universalchardet;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
public class Bug33USASCIIToGenerous {
public Bug33USASCIIToGenerous() {
super();
}
@Test
@Ignore("Not sure")
public void testUTF16 () throws IOException {
Assert.assertEquals("UTF-16BE", detect("ab".getBytes("UTF-16BE")));
Assert.assertEquals("UTF-16LE", detect("ab".getBytes("UTF-16LE")));
}
@Test
public void testZipHeader() throws IOException {
byte[] zipHeader = new byte[]{0x50, 0x4b, 0x03, 0x04, 0x14, 0x00, 0x02, 0x00};
Assert.assertNull(detect(zipHeader));
}
private String detect(byte[] data) throws IOException {
return UniversalDetector.detectCharset(new ByteArrayInputStream(data));
}
}
juniversalchardet-2.5.0/src/test/java/org/mozilla/universalchardet/Bug8VariousFailedCharsets.java 0000664 0000000 0000000 00000003512 14625121707 0033354 0 ustar 00root root 0000000 0000000 package org.mozilla.universalchardet;
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
/*
* This charset detection fails because too little data to analize
*/
public class Bug8VariousFailedCharsets {
@Test
@Ignore
public void test1() throws Exception {
/*
=?KR?B?wPzDvCCwy7v2?=
전체 검색
*/
byte[] data = decodeBase64("wPzDvCCwy7v2");
String charset = detect(data);
Assert.assertNotNull(charset);
Assert.assertEquals("전체 검색", new String(data, charset));
}
@Test
public void test2() throws Exception {
/*
=?wws-874?B?W1Nhbm9va10gt7TKzbq70a3L0qHSw83o0rnA0snS5LfCzdWhpMPR6acg8fLz?=
[Sanook] ทดสอบปัญหาการอ่านภาษาไทยอีกครั้ง ๑๒๓
*/
byte[] data = decodeBase64("W1Nhbm9va10gt7TKzbq70a3L0qHSw83o0rnA0snS5LfCzdWhpMPR6acg8fLz");
String charset = detect(data);
Assert.assertNotNull(charset);
Assert.assertEquals("[Sanook] ทดสอบปัญหาการอ่านภาษาไทยอีกครั้ง ๑๒๓", new String(data, charset));
}
@Test
@Ignore
public void test3() throws Exception {
/*
=?gb2312?B?zfjS19PKz+TX1Lavu9i4tDo=?= Re: �˲ű�
网易邮箱自动回复: Re: 人才表
*/
byte[] data = decodeBase64("zfjS19PKz+TX1Lavu9i4tDo=");
String charset = detect(data);
Assert.assertNotNull(charset);
Assert.assertEquals("网易邮箱自动回复:", new String(data, charset));
}
private String detect(byte[] data) {
UniversalDetector detector = new UniversalDetector();
detector.handleData(data);
detector.dataEnd();
String detected = detector.getDetectedCharset();
detector.reset();
return detected;
}
private byte[] decodeBase64(String string) {
return org.apache.commons.codec.binary.Base64.decodeBase64(string);
}
}
juniversalchardet-2.5.0/src/test/java/org/mozilla/universalchardet/ByteArrayReaderTest.java 0000664 0000000 0000000 00000001460 14625121707 0032261 0 ustar 00root root 0000000 0000000 package org.mozilla.universalchardet;
import org.junit.Test;
import java.io.BufferedReader;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import static org.junit.Assert.assertEquals;
public class ByteArrayReaderTest {
private String TEST_STRING = "\u00b0";
@Test
public void utf8() throws IOException {
byte[] data = TEST_STRING.getBytes(StandardCharsets.UTF_8);
try(BufferedReader bufferedReader = ReaderFactory.createBufferedReader(data)) {
assertEquals(TEST_STRING, bufferedReader.readLine());
}
}
@Test
public void ISO_8859_1() throws IOException {
byte[] data = TEST_STRING.getBytes(StandardCharsets.ISO_8859_1);
try(BufferedReader bufferedReader = ReaderFactory.createBufferedReader(data)) {
assertEquals(TEST_STRING, bufferedReader.readLine());
}
}
}
juniversalchardet-2.5.0/src/test/java/org/mozilla/universalchardet/GB18030SMFalsePositiveTest.java 0000664 0000000 0000000 00000002622 14625121707 0033057 0 ustar 00root root 0000000 0000000 package org.mozilla.universalchardet;
import static org.junit.Assert.assertEquals;
import java.io.UnsupportedEncodingException;
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
/**
* @author ian
* @since Jul 13, 2011
*
*/
public class GB18030SMFalsePositiveTest {
public GB18030SMFalsePositiveTest() {
super();
}
@Test
@Ignore
public void testFalsePositiveBug11() throws UnsupportedEncodingException {
String testString = "[°4°0°T°C°C°0°C°T";
byte[] testBuf = new byte[]{91, -80, 52, -80, 48, -80, 84, -80, 67, -80, 67, -80, 48, -80, 67, -80, 84};
byte[] buf = testString.getBytes("WINDOWS-1252");
Assert.assertArrayEquals(testBuf, buf);
UniversalDetector detector = new UniversalDetector();
detector.handleData(buf, 0, buf.length);
detector.dataEnd();
String encoding = detector.getDetectedCharset();
detector.reset();
assertEquals("WINDOWS-1252", encoding);
}
@Test
@Ignore
public void testFalsePositiveBug9() throws UnsupportedEncodingException {
String testString = "Wykamol,£588.95,0.18,0.12,testingSpecialised Products for DIY and Professionals£12";
byte[] buf = testString.getBytes("WINDOWS-1252");
UniversalDetector detector = new UniversalDetector();
detector.handleData(buf);
detector.dataEnd();
String encoding = detector.getDetectedCharset();
detector.reset();
assertEquals("WINDOWS-1252", encoding);
}
} juniversalchardet-2.5.0/src/test/java/org/mozilla/universalchardet/GB18030SamplesTest.java 0000664 0000000 0000000 00000001273 14625121707 0031447 0 ustar 00root root 0000000 0000000 package org.mozilla.universalchardet;
import java.io.File;
import java.io.IOException;
import org.junit.Assert;
import org.junit.Test;
public class GB18030SamplesTest {
public GB18030SamplesTest() {
super();
}
@Test
public void testGB18030Sample() throws IOException {
Assert.assertEquals("GB18030", getFileEncoding("gb2312-sample.txt"));
}
@Test
public void testGBKSample() throws IOException {
Assert.assertEquals("GB18030", getFileEncoding("gbk-sample.txt"));
}
private String getFileEncoding(String testFileName) throws IOException{
String fileName = "src/test/resources/" + testFileName;
return UniversalDetector.detectCharset(new File(fileName));
}
}
juniversalchardet-2.5.0/src/test/java/org/mozilla/universalchardet/NullOutputStream.java 0000664 0000000 0000000 00000002335 14625121707 0031705 0 ustar 00root root 0000000 0000000 package org.mozilla.universalchardet;
import java.io.IOException;
import java.io.OutputStream;
/**
* This OutputStream writes all data to the famous /dev/null.
*
* This output stream has no destination (file/socket etc.) and all
* bytes written to it are ignored and lost.
*
* @author Jeremias Maerki
* @version $Id: NullOutputStream.java 610010 2008-01-08 14:50:59Z niallp $
*/
public class NullOutputStream extends OutputStream {
/**
* A singleton.
*/
public static final NullOutputStream NULL_OUTPUT_STREAM = new NullOutputStream();
/**
* Does nothing - output to /dev/null.
* @param b The bytes to write
* @param off The start offset
* @param len The number of bytes to write
*/
public void write(byte[] b, int off, int len) {
//to /dev/null
}
/**
* Does nothing - output to /dev/null.
* @param b The byte to write
*/
public void write(int b) {
//to /dev/null
}
/**
* Does nothing - output to /dev/null.
* @param b The bytes to write
* @throws IOException never
*/
public void write(byte[] b) throws IOException {
//to /dev/null
}
}
juniversalchardet-2.5.0/src/test/java/org/mozilla/universalchardet/ReaderTest.java 0000664 0000000 0000000 00000003113 14625121707 0030433 0 ustar 00root root 0000000 0000000 package org.mozilla.universalchardet;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import org.junit.Assert;
import org.junit.Test;
public class ReaderTest {
private static final String TEST_STRING =
" コンソール アプリケーション : universalchardet プロジェクトの概要";
@Test
public void testUTF8 () throws IOException {
Assert.assertEquals(TEST_STRING, getSecondLine("utf8.txt"));
}
@Test
public void testUTF8N () throws IOException {
Assert.assertEquals(TEST_STRING, getSecondLine("utf8n.txt"));
}
@Test
public void testUTF16LE () throws IOException {
Assert.assertEquals(TEST_STRING, getSecondLine("utf16le.txt"));
}
@Test
public void testShifJis () throws IOException {
Assert.assertEquals(TEST_STRING, getSecondLine("shiftjis.txt"));
}
@Test
public void testEUC () throws IOException {
Assert.assertEquals(TEST_STRING, getSecondLine("euc.txt"));
}
@Test
public void testISO2022JP () throws IOException {
Assert.assertEquals(TEST_STRING, getSecondLine("iso2022jp.txt"));
}
private String getSecondLine(String testFileName) throws IOException{
String fileName = "src/test/resources/" + testFileName;
File file = new File(fileName);
BufferedReader reader = null;
try {
reader = ReaderFactory.createBufferedReader(file);
// Skip first line
reader.readLine();
// return second line
return reader.readLine();
}
finally {
if (reader != null) {
reader.close();
}
}
}
}
juniversalchardet-2.5.0/src/test/java/org/mozilla/universalchardet/ShortStringTests.java 0000664 0000000 0000000 00000004030 14625121707 0031701 0 ustar 00root root 0000000 0000000 package org.mozilla.universalchardet;
import java.io.UnsupportedEncodingException;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
public class ShortStringTests {
public ShortStringTests() {
super();
}
// Tets case for https://github.com/albfernandez/juniversalchardet/issues/22
@Test
@Ignore("data too short")
public void testDecodeBytes() {
final String string = "aeaCàêäÇ";
Charset s;
byte[] bytes;
bytes = string.getBytes(StandardCharsets.UTF_8);
s = this.guessCharset(bytes);
Assert.assertEquals(string, new String(string.getBytes(s), s)); // SUCCESS
bytes = string.getBytes(StandardCharsets.ISO_8859_1);
s = this.guessCharset(bytes); // detected charset = TIS-620, Thai charset ???!!!
Assert.assertEquals(string, new String(string.getBytes(s), s)); // FAILS of course !
}
// Tets case for https://github.com/albfernandez/juniversalchardet/issues/22
// With less accute characters, it's improved detection
@Test
public void testDecodeBytesBetterStats() {
final String string = "Château";
Charset s;
byte[] bytes;
bytes = string.getBytes(StandardCharsets.UTF_8);
s = this.guessCharset(bytes);
Assert.assertEquals(string, new String(string.getBytes(s), s)); // SUCCESS
bytes = string.getBytes(StandardCharsets.ISO_8859_1);
s = this.guessCharset(bytes);
Assert.assertEquals(string, new String(string.getBytes(s), s)); // SUCCESS
}
@Test
public void testShortString() throws UnsupportedEncodingException {
Assert.assertEquals("US-ASCII", guessCharsetName("abcd".getBytes()));
// Assert.assertNull(guessCharsetName("Ábcd".getBytes("ISO-8859-15")));
}
private Charset guessCharset(final byte[] bytes) {
return Charset.forName(guessCharsetName(bytes));
}
private String guessCharsetName(final byte[] bytes) {
final UniversalDetector detector = new UniversalDetector();
detector.handleData(bytes, 0, bytes.length);
detector.dataEnd();
return detector.getDetectedCharset();
}
}
juniversalchardet-2.5.0/src/test/java/org/mozilla/universalchardet/TIS620BasicTest.java 0000664 0000000 0000000 00000000773 14625121707 0031073 0 ustar 00root root 0000000 0000000 package org.mozilla.universalchardet;
import java.io.File;
import java.io.IOException;
import org.junit.Assert;
import org.junit.Test;
public class TIS620BasicTest {
@Test
public void testTIS620() throws IOException {
Assert.assertEquals("TIS620", getFileEncoding("tis620.txt"));
}
private String getFileEncoding(String testFileName) throws IOException{
String fileName = "src/test/resources/" + testFileName;
return UniversalDetector.detectCharset(new File(fileName));
}
}
juniversalchardet-2.5.0/src/test/java/org/mozilla/universalchardet/example/ 0000775 0000000 0000000 00000000000 14625121707 0027163 5 ustar 00root root 0000000 0000000 TestCreateReaderFromFile.java 0000664 0000000 0000000 00000001134 14625121707 0034560 0 ustar 00root root 0000000 0000000 juniversalchardet-2.5.0/src/test/java/org/mozilla/universalchardet/example package org.mozilla.universalchardet.example;
import org.mozilla.universalchardet.ReaderFactory;
public class TestCreateReaderFromFile {
public static void main (String[] args) throws java.io.IOException {
if (args.length != 1) {
System.err.println("Usage: java TestCreateReaderFromFile FILENAME");
System.exit(1);
}
java.io.Reader reader = null;
try {
java.io.File file = new java.io.File(args[0]);
reader = ReaderFactory.createBufferedReader(file);
// Do whatever you want with the reader
}
finally {
if (reader != null) {
reader.close();
}
}
}
}
juniversalchardet-2.5.0/src/test/java/org/mozilla/universalchardet/example/TestDetector.java 0000664 0000000 0000000 00000005310 14625121707 0032436 0 ustar 00root root 0000000 0000000 /* ***** BEGIN LICENSE BLOCK *****
* Version: MPL 1.1/GPL 2.0/LGPL 2.1
*
* The contents of this file are subject to the Mozilla Public License Version
* 1.1 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* http://www.mozilla.org/MPL/
*
* Software distributed under the License is distributed on an "AS IS" basis,
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
* for the specific language governing rights and limitations under the
* License.
*
* The Original Code is mozilla.org code.
*
* The Initial Developer of the Original Code is
* Netscape Communications Corporation.
* Portions created by the Initial Developer are Copyright (C) 1998
* the Initial Developer. All Rights Reserved.
*
* Contributor(s):
* Kohei TAKETA
* lnezda
*
* Alternatively, the contents of this file may be used under the terms of
* either the GNU General Public License Version 2 or later (the "GPL"), or
* the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
* in which case the provisions of the GPL or the LGPL are applicable instead
* of those above. If you wish to allow use of your version of this file only
* under the terms of either the GPL or the LGPL, and not to allow others to
* use your version of this file under the terms of the MPL, indicate your
* decision by deleting the provisions above and replace them with the notice
* and other provisions required by the GPL or the LGPL. If you do not delete
* the provisions above, a recipient may use your version of this file under
* the terms of any one of the MPL, the GPL or the LGPL.
*
* ***** END LICENSE BLOCK ***** */
package org.mozilla.universalchardet.example;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Paths;
import org.mozilla.universalchardet.UniversalDetector;
public class TestDetector {
public static void main(String[] args) throws java.io.IOException {
if (args.length != 1) {
System.err.println("Usage: java TestDetector FILENAME");
System.exit(1);
}
byte[] buf = new byte[4096];
String fileName = args[0];
try (InputStream fis = Files.newInputStream(Paths.get(fileName))) {
// (1)
UniversalDetector detector = new UniversalDetector();
// (2)
int nread;
while ((nread = fis.read(buf)) > 0 && !detector.isDone()) {
detector.handleData(buf, 0, nread);
}
// (3)
detector.dataEnd();
// (4)
String encoding = detector.getDetectedCharset();
if (encoding != null) {
System.out.println("Detected encoding = " + encoding);
} else {
System.out.println("No encoding detected.");
}
// (5)
detector.reset();
}
}
}
juniversalchardet-2.5.0/src/test/java/org/mozilla/universalchardet/example/TestDetectorFile.java 0000664 0000000 0000000 00000001103 14625121707 0033232 0 ustar 00root root 0000000 0000000 package org.mozilla.universalchardet.example;
import org.mozilla.universalchardet.UniversalDetector;
public class TestDetectorFile {
public static void main (String[] args) throws java.io.IOException {
if (args.length != 1) {
System.err.println("Usage: java TestDetectorFile FILENAME");
System.exit(1);
}
java.io.File file = new java.io.File(args[0]);
String encoding = UniversalDetector.detectCharset(file);
if (encoding != null) {
System.out.println("Detected encoding = " + encoding);
} else {
System.out.println("No encoding detected.");
}
}
}
juniversalchardet-2.5.0/src/test/java/org/mozilla/universalchardet/prober/ 0000775 0000000 0000000 00000000000 14625121707 0027021 5 ustar 00root root 0000000 0000000 juniversalchardet-2.5.0/src/test/java/org/mozilla/universalchardet/prober/statemachine/ 0000775 0000000 0000000 00000000000 14625121707 0031466 5 ustar 00root root 0000000 0000000 PkgIntTest.java 0000664 0000000 0000000 00000001425 14625121707 0034310 0 ustar 00root root 0000000 0000000 juniversalchardet-2.5.0/src/test/java/org/mozilla/universalchardet/prober/statemachine package org.mozilla.universalchardet.prober.statemachine;
import static org.junit.Assert.*;
import org.junit.Test;
public class PkgIntTest
{
@Test
public void testUnpack()
{
int[] data = new int[] {
PkgInt.pack4bits(0, 1, 2, 3, 4, 5, 6, 7),
PkgInt.pack4bits(8, 9, 10, 11, 12, 13, 14, 15)
};
PkgInt pkg = new PkgInt(
PkgInt.INDEX_SHIFT_4BITS,
PkgInt.SHIFT_MASK_4BITS,
PkgInt.BIT_SHIFT_4BITS,
PkgInt.UNIT_MASK_4BITS,
data);
for (int i=0; i<16; ++i) {
int n = pkg.unpack(i);
if (n != i) {
fail("Invalid packed value.");
}
}
}
}
juniversalchardet-2.5.0/src/test/resources/ 0000775 0000000 0000000 00000000000 14625121707 0021020 5 ustar 00root root 0000000 0000000 juniversalchardet-2.5.0/src/test/resources/ascii.txt 0000664 0000000 0000000 00000005412 14625121707 0022653 0 ustar 00root root 0000000 0000000
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean massa ipsum, accumsan at tincidunt at, viverra non nisi. Praesent interdum justo at quam vehicula, vel semper ligula porta. Nam elementum est et velit gravida dignissim. Praesent bibendum bibendum leo id interdum. Vestibulum vel mollis risus, eget consequat odio. Fusce magna magna, rhoncus quis tincidunt eu, laoreet quis lacus. Nullam at ligula id lacus cursus convallis. Mauris ut vulputate mi, quis sodales quam. In diam diam, lacinia ac cursus eget, bibendum sodales risus. Sed est nibh, vestibulum ac euismod in, dictum ullamcorper odio.
Etiam at tristique est. Mauris quis erat at risus dignissim euismod. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas iaculis malesuada pharetra. Quisque semper elit vel ex pellentesque fermentum vitae accumsan tellus. Maecenas ut blandit eros. Vestibulum orci lectus, volutpat in sapien vel, lobortis blandit sapien. Etiam tristique est et eros placerat iaculis.
Phasellus ac mollis est. Suspendisse at nulla ullamcorper, tincidunt nisi quis, aliquam augue. Donec id neque purus. Duis risus neque, pharetra et laoreet bibendum, dictum ut tortor. In malesuada viverra condimentum. Donec cursus lorem risus, a consequat tellus hendrerit non. Pellentesque a feugiat mauris. Ut vel dui sed tortor pharetra tristique ac feugiat ligula. Integer viverra consectetur augue, nec blandit dolor pellentesque tincidunt. Duis ut augue sed dolor efficitur vehicula a iaculis turpis. Suspendisse ultrices, nunc in posuere sagittis, erat elit feugiat lacus, id hendrerit elit neque ut velit. Mauris eu varius massa. Duis gravida, mauris a consequat blandit, neque diam interdum justo, non tincidunt justo elit id mi. Curabitur hendrerit nisi purus, id luctus mi maximus nec.
Pellentesque in volutpat diam, sagittis varius quam. In metus mauris, aliquam gravida feugiat eu, condimentum sed augue. Cras cursus, sem ut venenatis lacinia, quam ipsum pharetra risus, eu scelerisque odio orci non turpis. In commodo, felis accumsan pellentesque tempor, ligula nisl ultrices quam, eu facilisis tellus erat sed dolor. Suspendisse et nibh id nisi aliquet tempus non sed nisi. Donec cursus purus quam, nec dictum felis tempor ac. Nunc eu est orci. Sed fringilla diam id tortor pulvinar, vel faucibus turpis malesuada. Morbi pharetra sit amet dolor eget faucibus. Nam in tristique sem. Ut et eros ac lorem bibendum mattis. Aliquam erat volutpat. Nunc congue mattis neque, eu maximus nisi pellentesque at.
Quisque laoreet faucibus justo a auctor. Nunc condimentum erat tempor, placerat nulla vel, convallis urna. Praesent rhoncus nulla orci, vitae laoreet eros elementum quis. Quisque sed pellentesque tortor, id aliquet libero. Donec ultrices rhoncus felis nec congue. Donec eu ullamcorper massa. Aliquam at auctor elit. juniversalchardet-2.5.0/src/test/resources/big5.txt 0000664 0000000 0000000 00000001044 14625121707 0022406 0 ustar 00root root 0000000 0000000 jX
ʬAۥѪʬ
Big5AS٬jXΤjXAOϥc餤]餤^s̱`Ϊq~rrŶзǡA@13,060Ӻ~r[1]C
XXΥ洫XABig5ݤ夺XAW洫XCCCIIBCNS11643C
Big5ΩxWBPDc餤qϡAHӨëDaaзǡAӥuO~ɼзǡCʤѤtΡBWindowsc餤媩DntΪrŶOHBig5ǡAtӤSUۼW[PyrPyrϡAlͦhؤPC
2003~ABig5QCNS11643зǥ洫XAoFaCoӳ̷sQ٬Big5-2003C
juniversalchardet-2.5.0/src/test/resources/bug20-example-latin.txt 0000664 0000000 0000000 00000000271 14625121707 0025236 0 ustar 00root root 0000000 0000000 umlouds,Name1,Name2,Name3,Street,M,DE,80080,Mnchen,Contact,+49(0),@gls-itservices.com,Comment,+49,(0)98,765,432,Bl juniversalchardet-2.5.0/src/test/resources/bug8-various-failed-charsets.dat 0000664 0000000 0000000 00000001147 14625121707 0027104 0 ustar 00root root 0000000 0000000 =?KR?B?wPzDvCCwy7v2?=
전체 검색
=?wws-874?B?W1Nhbm9va10gt7TKzbq70a3L0qHSw83o0rnA0snS5LfCzdWhpMPR6acg8fLz?=
[Sanook] ทดสอบปัญหาการอ่านภาษาไทยอีกครั้ง ๑๒๓
=?wws-874?B?W1Nhbm9va10gt7TKzbq70a3L0qHSw83o0rnA0snS5LfCzdWhpMPR6acg8fLz?=
[Sanook] ทดสอบปัญหาการอ่านภาษาไทยอีกครั้ง ๑๒๓
=?gb2312?B?zfjS19PKz+TX1Lavu9i4tDo=?= Re: ˲ű
网易邮箱自动回复: Re: 人才表
dracioppa1216@rogers.com
dracioppa1216@rogers.com
German / Deutsch Sd Gr Gott French / Franais
juniversalchardet-2.5.0/src/test/resources/euc.txt 0000664 0000000 0000000 00000002606 14625121707 0022341 0 ustar 00root root 0000000 0000000 ========================================================================
ץꥱ : universalchardet ץȤγ
========================================================================
universalchardet ץꥱϡAppWizard ˤäƺޤ
Υեˤϡuniversalchardet ץꥱƥե
ƤγάҤƤޤ
universalchardet.vcproj
ϡץꥱ ɤ VC++ ץȤΥᥤ
ץ եǤ
ե Visual C++ ΥСȡץꥱ
ɤץåȥեࡢӥץȤεǽ˴ؤ
ҤƤޤ
universalchardet.cpp
ϡᥤΥץꥱ եǤ
/////////////////////////////////////////////////////////////////////////////
¾ɸե :
StdAfx.h, StdAfx.cpp
ΥեϡѥѤߥإå (PCH) ե
universalchardet.pch ȥץꥳѥѤ߷ե StdAfx.obj
ӥɤ뤿˻Ѥޤ
/////////////////////////////////////////////////////////////////////////////
¾Υ :
AppWizard Ǥ "TODO:" ȤѤơ桼ɲäޤϥޥ
ʬޤ
/////////////////////////////////////////////////////////////////////////////
juniversalchardet-2.5.0/src/test/resources/euckr.txt 0000664 0000000 0000000 00000001072 14625121707 0022672 0 ustar 00root root 0000000 0000000 EUC
Ű, 츮 .
EUC(Extended Unix Code, Ȯ н ڵ) ѱ, ߱, Ϻ ȭ ַ Ǵ 8Ʈ ڵ ̴.
EUC ISO 2022 ǥؿ ϰ ִ.
ѹα ͳ ȯ濡 ϰ ̰ ־ ģ EUC-KR ڵ Ͽ ѱ ѱ Ǵ ڸ ǥ ̴. ٸ ȭιΰȭ EUC-CN, ȭα(븸) EUC-TW, Ϻ EUC-JP ڵ Ѵ.
ּ "https://ko.wikipedia.org/w/index.php?title=EUC&oldid=10340616"
з:
ڵ
juniversalchardet-2.5.0/src/test/resources/euctw.txt 0000664 0000000 0000000 00000001044 14625121707 0022707 0 ustar 00root root 0000000 0000000
Big5Ĵҳơơϣڦȭ13,060Զ[1]
šҳůǨ̥ˡBig5ϡǨȴCCCIICNS11643
Big5ůƺٴեƷԭŲ˷ӡWindows˷Big5ҳٸĴٴȩۡ
2003ȡBig5ݷȭ̯CNS11643Ǩ㡢̽ڵħȢԶݷҳBig5-2003
juniversalchardet-2.5.0/src/test/resources/gb18030.txt 0000664 0000000 0000000 00000001752 14625121707 0022552 0 ustar 00root root 0000000 0000000 GB 18030
άٿƣɵİٿȫ
GB 18030ȫƣұGB 18030-2005ϢıַлʱµּGB 18030-2000ϢϢúֱַ䡷档GB 2312-1980ȫݣGBKݣ֧GB 13000Unicodeȫͳһ֣¼70244
GB 18030Ҫص㣺
UTF-8 ͬöֽڱ룬ÿֿ124ֽɡ
ռӴɶ161Ԫ
֧й֣Ҫ
¼Χ庺Լպ֡
ijлϢҵӹҵоݣɹල2000317շа汾ΪලֺܾйұίԱ2005118շ200651ʵʩ
˱ڵĵֽڱ벿֡˫ֽڱ벿ֽ֣ڱ벿¼պͳһչA֣ΪǿԱڹģԱлƷҪ֧ͬʱֽڡ˫ֽںֽڱĹ
ӴҳΪ 54936
juniversalchardet-2.5.0/src/test/resources/gb2312-sample.txt 0000664 0000000 0000000 00000011553 14625121707 0023745 0 ustar 00root root 0000000 0000000 GB18030GBKGBKGB2312
GBKȫơչ淶GBKꡱչƴĵһĸӢƣChinese Internal Code Specification лȫϢίԱ1995121ƶҼලֱ˾ӹҵƼල˾19951215Լ꺯1995 229ļʽȷΪ淶ָļһGBK淶Ϊ1.0档
GBK GB 2312 ݣ֧ ISO 10646.1ʱǰ߹ɹеһµIJISO 10646 ǹʱ֯ ISO һ Universal Multilpe-Octet Coded Character SetUCS½Ϊͨöλַ̨ΪöλԪԪ Unicode ֯ Unicode ȫݡISO 10646.1 Ǹñĵһ֡ϵṹƽ桷ҹ 1993 GB 13000.1 ұʽϿɣ GB 13000.1 ͬ ISO 10646.1
GBK룬GB2312-80ϵչ淶ʹ˫ֽڱ뷽뷶Χ8140FEFExx7F23940λ¼21003֣ȫGB2312-80ֹ֧ʱISO/IEC10646-1ұGB13000-1еȫպ֣BIG5ек֡GBK뷽199510ƶ 199512ʽİWIN95WIN98WINDOWS NTԼWINDOWS 2000WINDOWS XPWIN 7ȶ֧GBK뷽
Ǿʹøֱĺ֣뵽ʲôأν룬Թ̶˳ַԴΪ¼ݡͳһڲַ˳Ϊ롱ֿйصıУGB롢GBK롢̨BIG-5ȣͬĺֿⶼ뺺ֵӦйϵ
ܶʹùУֲãΪʹõҪGBֿ⣬˱ֻ¼6763ú֣GBֻֿ֣ͨŮ洲ƴֳֻ֡ܲĺҪСṹЭͳһֹͼķʽ֣Ź֮ۡûϢϵͳҪѯšʱʵֵġGBKֿ⣬ػȱ
GBֿ䵽GBKֿ⣬14ǧ֡1996ͶʼڡΡ¡GBKֿ⣬19984³ΪһͨȨ֯GBKֿרҵ̡ѽȫתGBKֿ⣬461821003ӵGBKֿij̡
ISO 10646 һϸԵʽԼӷŵıϵеĺֲֳΪCJK ͳһ֡C ָйJ ָձK ָʣей֣Դй½ GB 2312GB 12345ֱִͨȷĺֺͷţԼԴ̨ CNS 11643 е 12 棨ͬ BIG-5 룩 14 ĺֺͷš
뷽ʽ
ʵʲԺͲĵGBKDzõ˫ֽڱ䳤룬Ӣʹõֽڱ룬ȫASCIIַ룬IJֲ˫ֽڱ롣
ֻ
GBK 淶¼ ISO 10646.1 еȫ CJK ֺͷţ䡣
1. GB 2312 еȫ֡Ǻַš
2. GB 13000.1 е CJK ֡Ϻϼ 20902 GB ֡
3. ܱδ GB 13000.1 52 ֡
4. ֵ䡷Ǻδ GB 13000.1 28 Ҫ
5. 13 ֽṹ
6. BIG-5 δ GB 2312 롢 GB 13000.1 е 139 ͼηš
7. GB 12345 6 ƴš
8. ֡𡱡
9. GB 12345 19 űţGB 12345 GB 2312 ű 29 10 δ GB 13000.1 룬 GBK գ
10. GB 13000.1 CJK ѡ 21 ֡
11. GB 13000.1 31 IBM OS/2 ר÷š
12.δ¼롶»ֵ䡷ϵһЩ֣硰|ļ塣
估˳
GBK ˫ֽڱʾ뷶ΧΪ 8140-FEFEֽ 81-FE ֮䣬βֽ 40-FE ֮䣬 xx7F һߡܼ 23940 λ 21886 ֺͼηţк֣21003 ͼη 883
ȫΪ֣
1.
a. GB 2312 GBK/2: B0A1-F7FE¼ GB 2312 6763 ԭ˳С
b. GB 13000.1 人
(1) GBK/3: 8140-A0FE¼ GB 13000.1 е CJK 6080
(2) GBK/4: AA40-FEA0¼ CJK ֺĺ 8160 CJK ǰ UCS СУĺ֣ںֵ䡷ҳ/λС
2. ͼη
a. GB 2312 Ǻַ GBK/1: A1A1-A9FEг GB 2312 ķ⣬ 10 Сдֺ GB 12345 ķšƷ 717
b. GB 13000.1 Ǻ GBK/5: A840-A9A0BIG-5 Ǻַšṹ͡ڴƷ 166
3. ûԶΪ(1)(2)(3)С
(1) AAA1-AFFEλ 564
(2) F8A1-FEFEλ 658
(3) A140-A7A0λ 672
(3)ܶûţʹãΪųδڴַĿԡ
GBK µĹ涨
1. ԭ GB 13000.1 GУԴй½ĺ֣µ/αһ¡
2. CJK ͬܿڣе GBK 뺺ʵʩΡGB ڲǰ£йΡ
3. ڳ CJK ͬġͬδȷ涨ĺ֣ GBK λݰžΡ GBK ͬһֵ¾Ρ
4. ǺַŵΣ GB 2312 Ѿģ GB 2312 һ£ GB 2312 IJ֣ GB 13000.1 һ¡
5. ƴĸȡʽ juniversalchardet-2.5.0/src/test/resources/gbk-sample.txt 0000664 0000000 0000000 00000001273 14625121707 0023606 0 ustar 00root root 0000000 0000000 hփȴaUչҎQGBKȫ顶hփȴaUչҎ(GBK)1.0棬A͇ȫϢg˜ʻgίT1995121ӆҼgO֘˜ʻ˾ӹIƼc|O˾19951215ԡg˺[1995]229̖ļʽ GBK21886hֺ͈Dη̖Нh֣͘21003Dη̖883
GBKK顸UչĝhZƴkuzhnһĸӢȫQChinese Internal Code Extension Specification
GBK ֻ顸gҎָļ춇Ҙ˜ʡ|gO2000317ƳGB 18030-2000˜ʣȡGBKGB 18030-2000ȫGBKah֣ڵڶֹʹùٶMДUչ˴sһقhֹּagnjGBKӼȫՈGB 18030
juniversalchardet-2.5.0/src/test/resources/iso2022jp.txt 0000664 0000000 0000000 00000003100 14625121707 0023205 0 ustar 00root root 0000000 0000000 ========================================================================
$B%3%s%=!<%k(J $B%"%W%j%1!<%7%g%s(J : universalchardet $B%W%m%8%'%/%H$N35MW(J
========================================================================
$B$3$N(J universalchardet $B%"%W%j%1!<%7%g%s$O!"(JAppWizard $B$K$h$C$F:n@.$5$l$^$7$?!#(J
$B$3$N%U%!%$%k$K$O!"(Juniversalchardet $B%"%W%j%1!<%7%g%s$r9=@.$9$k3F%U%!%$%k$N(J
$BFbMF$N35N,$,5-=R$5$l$F$$$^$9!#(J
universalchardet.vcproj
$B$3$l$O!"%"%W%j%1!<%7%g%s(J $B%&%#%6!<%I$G@8@.$5$l$k(J VC++ $B%W%m%8%'%/%H$N%a%$%s$N(J
$B%W%m%8%'%/%H(J $B%U%!%$%k$G$9!#(J
$B%U%!%$%k$r@8@.$7$?(J Visual C++ $B$N%P!<%8%g%s>pJs$H!"%"%W%j%1!<%7%g%s(J
$B%&%#%6!<%I$GA*Br$7$?%W%i%C%H%U%)!<%`!"9=@.!"$*$h$S%W%m%8%'%/%H$N5!G=$K4X$9$k(J
$B>pJs$,5-=R$5$l$F$$$^$9!#(J
universalchardet.cpp
$B$3$l$O!"%a%$%s$N%"%W%j%1!<%7%g%s(J $B%=!<%9(J $B%U%!%$%k$G$9!#(J
/////////////////////////////////////////////////////////////////////////////
$B$=$NB>$NI8=`%U%!%$%k(J :
StdAfx.h, StdAfx.cpp
$B$3$l$i$N%U%!%$%k$O!"%3%s%Q%$%k:Q$_%X%C%@!<(J (PCH) $B%U%!%$%k(J
universalchardet.pch $B$H%W%j%3%s%Q%$%k:Q$_7?%U%!%$%k(J StdAfx.obj $B$r(J
$B%S%k%I$9$k$?$a$K;HMQ$7$^$9!#(J
/////////////////////////////////////////////////////////////////////////////
$B$=$NB>$N%a%b(J :
AppWizard $B$G$O(J "TODO:" $B%3%a%s%H$r;HMQ$7$F!"%f!<%6!<$,DI2C$^$?$O%+%9%?%^%$%:$9$k(J
$B%=!<%9ItJ,$r<($7$^$9!#(J
/////////////////////////////////////////////////////////////////////////////
juniversalchardet-2.5.0/src/test/resources/iso88598.txt 0000664 0000000 0000000 00000000730 14625121707 0023001 0 ustar 00root root 0000000 0000000
() . ( ) , . 19 20 . .
juniversalchardet-2.5.0/src/test/resources/shiftjis.txt 0000664 0000000 0000000 00000002606 14625121707 0023410 0 ustar 00root root 0000000 0000000 ========================================================================
R\[ AvP[V : universalchardet vWFNg̊Tv
========================================================================
universalchardet AvP[V́AAppWizard ɂč쐬܂B
̃t@Cɂ́Auniversalchardet AvP[V\et@C
e̊TLqĂ܂B
universalchardet.vcproj
́AAvP[V EBU[hŐ VC++ vWFNg̃C
vWFNg t@CłB
t@C Visual C++ ̃o[WƁAAvP[V
EBU[hőIvbgtH[A\AуvWFNg̋@\Ɋւ
LqĂ܂B
universalchardet.cpp
́AC̃AvP[V \[X t@CłB
/////////////////////////////////////////////////////////////////////////////
̑̕Wt@C :
StdAfx.h, StdAfx.cpp
̃t@ĆARpCς݃wb_[ (PCH) t@C
universalchardet.pch ƃvRpCς^t@C StdAfx.obj
rh邽߂Ɏgp܂B
/////////////////////////////////////////////////////////////////////////////
̑̃ :
AppWizard ł "TODO:" RggpāA[U[lj܂̓JX^}CY
\[X܂B
/////////////////////////////////////////////////////////////////////////////
juniversalchardet-2.5.0/src/test/resources/tis620.txt 0000664 0000000 0000000 00000000466 14625121707 0022616 0 ustar 00root root 0000000 0000000 شȤس
Һôҽ٧ѵѨ
ѹҾѲԪҡ
ҧҭ蹦Һձ
ø觫ѴִѴ
ѴѪ
ԺѵԻоĵԡ˹
ٴ ҿѧ
ھط
ѹطԾ¹
juniversalchardet-2.5.0/src/test/resources/utf16le.txt 0000664 0000000 0000000 00000004120 14625121707 0023044 0 ustar 00root root 0000000 0000000 = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
00000 00000000 : u n i v e r s a l c h a r d e t 000000n0i
= = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
S0n0 u n i v e r s a l c h a r d e t 00000000o00A p p W i z a r d k00c0f0\ObU00~0W0_00
S0n00000k0o00u n i v e r s a l c h a r d e t 000000000ibY00T0000n0
Q[n0ieuL0U00f0D0~0Y00
u n i v e r s a l c h a r d e t . v c p r o j
S00o0000000000 00000g0 ubU000 V C + + 000000n0000n0
000000 0000g0Y00
00000 ubW0_0 V i s u a l C + + n000000`1Xh0000000000
00000g0xbW0_0000000000ib0J00s0000000n0_jk0Y00
`1XL0U00f0D0~0Y00
u n i v e r s a l c h a r d e t . c p p
S00o00000n000000000 000 0000g0Y00
/ / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / /
]0n0Nn0jn0000 :
S t d A f x . h , S t d A f x . c p p
S000n00000o0000000n00000 ( P C H ) 0000
u n i v e r s a l c h a r d e t . p c h h00000000n0W0000 S t d A f x . o b j 0
000Y00_00k0O(uW0~0Y00
/ / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / /
]0n0Nn000 :
A p p W i z a r d g0o0 " T O D O : " 00000O(uW0f000000L0R~0_0o0000000Y00
000R0:yW0~0Y00
/ / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / /
juniversalchardet-2.5.0/src/test/resources/utf8.txt 0000664 0000000 0000000 00000003350 14625121707 0022450 0 ustar 00root root 0000000 0000000 ========================================================================
コンソール アプリケーション : universalchardet プロジェクトの概要
========================================================================
この universalchardet アプリケーションは、AppWizard によって作成されました。
このファイルには、universalchardet アプリケーションを構成する各ファイルの
内容の概略が記述されています。
universalchardet.vcproj
これは、アプリケーション ウィザードで生成される VC++ プロジェクトのメインの
プロジェクト ファイルです。
ファイルを生成した Visual C++ のバージョン情報と、アプリケーション
ウィザードで選択したプラットフォーム、構成、およびプロジェクトの機能に関する
情報が記述されています。
universalchardet.cpp
これは、メインのアプリケーション ソース ファイルです。
/////////////////////////////////////////////////////////////////////////////
その他の標準ファイル :
StdAfx.h, StdAfx.cpp
これらのファイルは、コンパイル済みヘッダー (PCH) ファイル
universalchardet.pch とプリコンパイル済み型ファイル StdAfx.obj を
ビルドするために使用します。
/////////////////////////////////////////////////////////////////////////////
その他のメモ :
AppWizard では "TODO:" コメントを使用して、ユーザーが追加またはカスタマイズする
ソース部分を示します。
/////////////////////////////////////////////////////////////////////////////
juniversalchardet-2.5.0/src/test/resources/utf8n-emoji.txt 0000664 0000000 0000000 00000003355 14625121707 0023734 0 ustar 00root root 0000000 0000000 ========================================================================
🍣コンソール アプリケーション : 🌏universalchardet プロジェクトの概要
========================================================================
この universalchardet アプリケーションは、AppWizard によって作成されました。
このファイルには、universalchardet アプリケーションを構成する各ファイルの
内容の概略が記述されています。
universalchardet.vcproj
これは、アプリケーション ウィザードで生成される VC++ プロジェクトのメインの
プロジェクト ファイルです。
ファイルを生成した Visual C++ のバージョン情報と、アプリケーション
ウィザードで選択したプラットフォーム、構成、およびプロジェクトの機能に関する
情報が記述されています。
universalchardet.cpp
これは、メインのアプリケーション ソース ファイルです。
/////////////////////////////////////////////////////////////////////////////
その他の標準ファイル :
StdAfx.h, StdAfx.cpp
これらのファイルは、コンパイル済みヘッダー (PCH) ファイル
universalchardet.pch とプリコンパイル済み型ファイル StdAfx.obj を
ビルドするために使用します。
/////////////////////////////////////////////////////////////////////////////
その他のメモ :
AppWizard では "TODO:" コメントを使用して、ユーザーが追加またはカスタマイズする
ソース部分を示します。
/////////////////////////////////////////////////////////////////////////////
juniversalchardet-2.5.0/src/test/resources/utf8n.txt 0000664 0000000 0000000 00000003345 14625121707 0022632 0 ustar 00root root 0000000 0000000 ========================================================================
コンソール アプリケーション : universalchardet プロジェクトの概要
========================================================================
この universalchardet アプリケーションは、AppWizard によって作成されました。
このファイルには、universalchardet アプリケーションを構成する各ファイルの
内容の概略が記述されています。
universalchardet.vcproj
これは、アプリケーション ウィザードで生成される VC++ プロジェクトのメインの
プロジェクト ファイルです。
ファイルを生成した Visual C++ のバージョン情報と、アプリケーション
ウィザードで選択したプラットフォーム、構成、およびプロジェクトの機能に関する
情報が記述されています。
universalchardet.cpp
これは、メインのアプリケーション ソース ファイルです。
/////////////////////////////////////////////////////////////////////////////
その他の標準ファイル :
StdAfx.h, StdAfx.cpp
これらのファイルは、コンパイル済みヘッダー (PCH) ファイル
universalchardet.pch とプリコンパイル済み型ファイル StdAfx.obj を
ビルドするために使用します。
/////////////////////////////////////////////////////////////////////////////
その他のメモ :
AppWizard では "TODO:" コメントを使用して、ユーザーが追加またはカスタマイズする
ソース部分を示します。
/////////////////////////////////////////////////////////////////////////////
juniversalchardet-2.5.0/src/test/resources/windows1255.txt 0000664 0000000 0000000 00000000730 14625121707 0023570 0 ustar 00root root 0000000 0000000
() . ( ) , . 19 20 . .