Comparable: the "better" one comes before the worse.
* First order by probability descending (1 to 0).
* Then order by language ascending (a to z).
*
* This class is immutable.
*
* @author Nakatani Shuyo
* @author Fabian Kessler
*/
public class DetectedLanguage implements Comparable {
@NotNull
private final LdLocale locale;
private final double probability;
/**
* @param locale
* @param probability 0-1
*/
public DetectedLanguage(@NotNull LdLocale locale, double probability) {
if (probability<0d) throw new IllegalArgumentException("Probability must be >= 0 but was "+probability);
if (probability>1d) throw new IllegalArgumentException("Probability must be <= 1 but was "+probability);
this.locale = locale;
this.probability = probability;
}
@NotNull
public LdLocale getLocale() {
return locale;
}
/**
* @return 0-1, the higher the better.
*/
public double getProbability() {
return probability;
}
public String toString() {
return "DetectedLanguage["+ locale + ":" + probability+"]";
}
/**
* See class header.
*/
@Override
public int compareTo(DetectedLanguage o) {
int compare = Double.compare(o.probability, this.probability);
if (compare!=0) return compare;
return this.locale.toString().compareTo(o.locale.toString());
}
}
language-detector-language-detector-0.6/src/main/java/com/optimaize/langdetect/LanguageDetector.java0000664 0000000 0000000 00000003741 12775654761 0033777 0 ustar 00root root 0000000 0000000 package com.optimaize.langdetect;
import com.google.common.base.Optional;
import com.optimaize.langdetect.i18n.LdLocale;
import java.util.List;
/**
* Guesses the language of an input string or text.
*
* See website for details.
*
* This detector cannot handle well:
* Short input text, can work or give wrong results.
* Text written in multiple languages. It likely returns the language for the most prominent text. It's not made for that.
* Text written in languages for which the detector has no profile loaded. It may just return other similar languages.
*
*
* @author Fabian Kessler
*/
public interface LanguageDetector {
/**
* Returns the best detected language if the algorithm is very confident.
*
* Note: you may want to use getProbabilities() instead. This here is very strict, and sometimes returns
* absent even though the first choice in getProbabilities() is correct.
*
* @param text You probably want a {@link com.optimaize.langdetect.text.TextObject}.
* @return The language if confident, absent if unknown or not confident enough.
*/
Optional detect(CharSequence text);
/**
* Returns all languages with at least some likeliness.
*
* There is a configurable cutoff applied for languages with very low probability.
*
* The way the algorithm currently works, it can be that, for example, this method returns a 0.99 for
* Danish and less than 0.01 for Norwegian, and still they have almost the same chance. It would be nice if
* this could be improved in future versions.
*
* @param text You probably want a {@link com.optimaize.langdetect.text.TextObject}.
* @return Sorted from better to worse. May be empty.
* It's empty if the program failed to detect any language, or if the input text did not
* contain any usable text (just noise).
*/
List getProbabilities(CharSequence text);
}
LanguageDetectorBuilder.java 0000664 0000000 0000000 00000015067 12775654761 0035233 0 ustar 00root root 0000000 0000000 language-detector-language-detector-0.6/src/main/java/com/optimaize/langdetect package com.optimaize.langdetect;
import com.google.common.base.Optional;
import com.optimaize.langdetect.i18n.LdLocale;
import com.optimaize.langdetect.ngram.NgramExtractor;
import com.optimaize.langdetect.profiles.LanguageProfile;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
/**
* Builder for {@link LanguageDetector}.
*
* This class does no internal synchronization.
*
* @author Fabian Kessler
*/
public class LanguageDetectorBuilder {
private static final double ALPHA_DEFAULT = 0.5;
@NotNull
private final NgramExtractor ngramExtractor;
private double alpha = ALPHA_DEFAULT;
private Optional seed = Optional.absent();
private int shortTextAlgorithm = 50;
private double prefixFactor = 1.0d;
private double suffixFactor = 1.0d;
private double probabilityThreshold = 0.1;
private double minimalConfidence = 0.9999d;
@Nullable
private Map langWeightingMap;
@NotNull
private final Set languageProfiles = new HashSet<>();
@NotNull
private final Set langsAdded = new HashSet<>();
public static LanguageDetectorBuilder create(@NotNull NgramExtractor ngramExtractor) {
return new LanguageDetectorBuilder(ngramExtractor);
}
private LanguageDetectorBuilder(@NotNull NgramExtractor ngramExtractor) {
this.ngramExtractor = ngramExtractor;
}
public LanguageDetectorBuilder alpha(double alpha) {
if (alpha<0 || alpha>1) throw new IllegalArgumentException("alpha must be between 0 and 1, but was: "+alpha);
this.alpha = alpha;
return this;
}
public LanguageDetectorBuilder seed(long seed) {
return seed(Optional.of(seed));
}
public LanguageDetectorBuilder seed(@NotNull Optional seed) {
this.seed = seed;
return this;
}
/**
* Defaults to 0, which means don't use this feature. That's the old behavior.
*/
public LanguageDetectorBuilder shortTextAlgorithm(int shortTextAlgorithm) {
this.shortTextAlgorithm = shortTextAlgorithm;
return this;
}
/**
* Sets prefixFactor() and suffixFactor() both to the given value.
* @see #prefixFactor(double)
*/
public LanguageDetectorBuilder affixFactor(double affixFactor) {
prefixFactor(affixFactor);
suffixFactor(affixFactor);
return this;
}
/**
* To weight n-grams that are on the left border of a word differently from n-grams
* in the middle of words, assign a value here.
*
* Affixes (prefixes and suffixes) often distinguish the specific features of languages.
* Giving a value greater than 1.0 weights these n-grams higher. A 2.0 weights them double.
*
* Defaults to 1.0, which means don't use this feature.
* @param prefixFactor 0.0 to 10.0, a suggested value is 1.5
*/
public LanguageDetectorBuilder prefixFactor(double prefixFactor) {
this.prefixFactor = prefixFactor;
return this;
}
/**
* Defaults to 1.0, which means don't use this feature.
* @param suffixFactor 0.0 to 10.0, a suggested value is 2.0
* @see #prefixFactor(double)
*/
public LanguageDetectorBuilder suffixFactor(double suffixFactor) {
this.suffixFactor = suffixFactor;
return this;
}
/**
* {@link LanguageDetector#getProbabilities} does not return languages with less probability than this.
* The default currently is 0.1 (the old hardcoded value), but don't rely on it, if you need to be sure
* then set one.
*/
public LanguageDetectorBuilder probabilityThreshold(double probabilityThreshold) {
this.probabilityThreshold = probabilityThreshold;
return this;
}
/**
* {@link LanguageDetector#detect} returns a language if the best detected language has at least this probability.
* The default currently is 0.9999d, but don't rely on it, if you need to be sure then set one.
*/
public LanguageDetectorBuilder minimalConfidence(double minimalConfidence) {
this.minimalConfidence = minimalConfidence;
return this;
}
/**
* TODO document exactly. Also explain how it influences the results.
* Maybe check for unsupported languages at some point, or not, but document whether it does throw or ignore.
* String key = language, Double value = priority (probably 0-1).
*/
public LanguageDetectorBuilder languagePriorities(@Nullable Map langWeightingMap) {
this.langWeightingMap = langWeightingMap;
return this;
}
/**
* @throws IllegalStateException if a profile for the same language was added already (must be a userland bug).
*/
public LanguageDetectorBuilder withProfile(LanguageProfile languageProfile) throws IllegalStateException {
if (langsAdded.contains(languageProfile.getLocale())) {
throw new IllegalStateException("A language profile for language "+languageProfile.getLocale()+" was added already!");
}
for (Integer gramLength : ngramExtractor.getGramLengths()) {
if (!languageProfile.getGramLengths().contains(gramLength)) {
throw new IllegalArgumentException("The NgramExtractor is set to handle "+gramLength+"-grams but the given language profile for "+languageProfile.getLocale()+" does not support this!");
}
}
langsAdded.add(languageProfile.getLocale());
languageProfiles.add(languageProfile);
return this;
}
/**
* @throws IllegalStateException if a profile for the same language was added already (must be a userland bug).
*/
public LanguageDetectorBuilder withProfiles(Iterable languageProfiles) throws IllegalStateException {
for (LanguageProfile languageProfile : languageProfiles) {
withProfile(languageProfile);
}
return this;
}
/**
* @throws IllegalStateException if no LanguageProfile was {@link #withProfile added}.
*/
public LanguageDetector build() throws IllegalStateException {
if (languageProfiles.isEmpty()) throw new IllegalStateException();
return new LanguageDetectorImpl(
NgramFrequencyData.create(languageProfiles, ngramExtractor.getGramLengths()),
alpha, seed, shortTextAlgorithm,
prefixFactor, suffixFactor,
probabilityThreshold, minimalConfidence,
langWeightingMap,
ngramExtractor
);
}
}
LanguageDetectorImpl.java 0000664 0000000 0000000 00000024405 12775654761 0034542 0 ustar 00root root 0000000 0000000 language-detector-language-detector-0.6/src/main/java/com/optimaize/langdetect package com.optimaize.langdetect;
import com.optimaize.langdetect.cybozu.util.Util;
import com.google.common.base.Optional;
import com.optimaize.langdetect.i18n.LdLocale;
import com.optimaize.langdetect.ngram.NgramExtractor;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
/**
*
*
* This class is immutable and thus thread-safe.
*
* @author Nakatani Shuyo
* @author Fabian Kessler
* @author Elmer Garduno
*/
public final class LanguageDetectorImpl implements LanguageDetector {
private static final Logger logger = LoggerFactory.getLogger(LanguageDetectorImpl.class);
/**
* TODO document what this is for, and why that value is chosen.
*/
private static final double ALPHA_WIDTH = 0.05;
/**
* TODO document what this is for, and why that value is chosen.
*/
private static final int ITERATION_LIMIT = 1000;
/**
* TODO document what this is for, and why that value is chosen.
*/
private static final double CONV_THRESHOLD = 0.99999;
/**
* TODO document what this is for, and why that value is chosen.
*/
private static final int BASE_FREQ = 10000;
/**
* TODO document what this is for, and why that value is chosen.
*/
private static final int N_TRIAL = 7;
/**
* This is used when no custom seed was passed in.
* By using the same seed for different calls, the results are consistent also.
*
* Changing this number means that users of the library might suddenly see other results after updating.
* So don't change it hastily. I chose a prime number *clueless*.
* See https://github.com/optimaize/language-detector/issues/14
*/
private static final long DEFAULT_SEED = 41L;
private static final Comparator PROBABILITY_SORTING_COMPARATOR = new Comparator() {
public int compare(DetectedLanguage a, DetectedLanguage b) {
return Double.compare(b.getProbability(), a.getProbability());
}
};
@NotNull
private final NgramFrequencyData ngramFrequencyData;
/**
* User-defined language priorities, in the same order as {@code langlist}.
*/
@Nullable
private final double[] priorMap;
private final double alpha;
private final Optional seed;
private final int shortTextAlgorithm;
private final double prefixFactor;
private final double suffixFactor;
private final double probabilityThreshold;
private final double minimalConfidence;
private final NgramExtractor ngramExtractor;
/**
* Use the {@link LanguageDetectorBuilder}.
*/
LanguageDetectorImpl(@NotNull NgramFrequencyData ngramFrequencyData,
double alpha, Optional seed, int shortTextAlgorithm,
double prefixFactor, double suffixFactor,
double probabilityThreshold,
double minimalConfidence,
@Nullable Map langWeightingMap,
@NotNull NgramExtractor ngramExtractor) {
if (alpha<0d || alpha >1d) throw new IllegalArgumentException("alpha must be between 0 and 1, but was: "+alpha);
if (prefixFactor <0d || prefixFactor >10d) throw new IllegalArgumentException("prefixFactor must be between 0 and 10, but was: "+prefixFactor);
if (suffixFactor <0d || suffixFactor >10d) throw new IllegalArgumentException("suffixFactor must be between 0 and 10, but was: "+suffixFactor);
if (probabilityThreshold<0d || probabilityThreshold>1d) throw new IllegalArgumentException("probabilityThreshold must be between 0 and 1, but was: "+probabilityThreshold);
if (minimalConfidence<0d || minimalConfidence>1d) throw new IllegalArgumentException("minimalConfidence must be between 0 and 1, but was: "+minimalConfidence);
if (langWeightingMap!=null && langWeightingMap.isEmpty()) langWeightingMap = null;
this.ngramFrequencyData = ngramFrequencyData;
this.alpha = alpha;
this.seed = seed;
this.shortTextAlgorithm = shortTextAlgorithm;
this.prefixFactor = prefixFactor;
this.suffixFactor = suffixFactor;
this.probabilityThreshold = probabilityThreshold;
this.minimalConfidence = minimalConfidence;
this.priorMap = (langWeightingMap==null) ? null : Util.makeInternalPrioMap(langWeightingMap, ngramFrequencyData.getLanguageList());
this.ngramExtractor = ngramExtractor;
}
@Override
public Optional detect(CharSequence text) {
List probabilities = getProbabilities(text);
if (probabilities.isEmpty()) {
return Optional.absent();
} else {
DetectedLanguage best = probabilities.get(0);
if (best.getProbability() >= minimalConfidence) {
return Optional.of(best.getLocale());
} else {
return Optional.absent();
}
}
}
@Override
public List getProbabilities(CharSequence text) {
double[] langprob = detectBlock(text);
if (langprob==null) {
return Collections.emptyList();
} else {
return sortProbability(langprob);
}
}
/**
* @return null if there are no "features" in the text (just noise).
*/
@Nullable
private double[] detectBlock(CharSequence text) {
if (text.length() <= shortTextAlgorithm) {
Map ngrams = ngramExtractor.extractCountedGrams(text);
if (ngrams.isEmpty()) return null;
return detectBlockShortText(ngrams);
} else {
List strings = ngramExtractor.extractGrams(text);
if (strings.isEmpty()) return null;
return detectBlockLongText(strings);
}
}
/**
*/
private double[] detectBlockShortText(Map ngrams) {
double[] prob = initProbability();
double alpha = this.alpha; //TODO I don't understand what this does.
for (Map.Entry gramWithCount : ngrams.entrySet()) {
updateLangProb(prob, gramWithCount.getKey(), gramWithCount.getValue(), alpha);
if (Util.normalizeProb(prob) > CONV_THRESHOLD) break; //this break ensures that we quit the loop before all probabilities reach 0
}
Util.normalizeProb(prob);
if (logger.isDebugEnabled()) logger.debug("==> " + sortProbability(prob));
return prob;
}
/**
* This is the original algorithm used for all text length.
* It is inappropriate for short text.
*/
private double[] detectBlockLongText(List ngrams) {
assert !ngrams.isEmpty();
double[] langprob = new double[ngramFrequencyData.getLanguageList().size()];
Random rand = new Random(seed.or(DEFAULT_SEED));
for (int t = 0; t < N_TRIAL; ++t) {
double[] prob = initProbability();
double alpha = this.alpha + (rand.nextGaussian() * ALPHA_WIDTH);
for (int i=0; i CONV_THRESHOLD) break; //this break ensures that we quit the loop before all probabilities reach 0
if (logger.isTraceEnabled()) logger.trace("> " + sortProbability(prob));
}
}
for(int j=0;j " + sortProbability(prob));
}
return langprob;
}
/**
* Initialize the map of language probabilities.
* If there is the specified prior map, use it as initial map.
* @return initialized map of language probabilities
*/
private double[] initProbability() {
double[] prob = new double[ngramFrequencyData.getLanguageList().size()];
if (priorMap != null) {
//TODO analyze and optimize this code, looks like double copy.
System.arraycopy(priorMap, 0, prob, 0, prob.length);
for(int i=0;i1) {
if (prefixFactor !=1.0 && ngram.charAt(0)==' ') {
weight *= prefixFactor;
} else if (suffixFactor!=1.0 && ngram.charAt(ngram.length()-1)==' ') {
weight *= suffixFactor;
}
}
for (int i=0; i sortProbability(double[] prob) {
List list = new ArrayList<>();
//step 1: add all that have reached a minimal probability:
for (int j=0;j= probabilityThreshold) {
list.add(new DetectedLanguage(ngramFrequencyData.getLanguage(j), p));
}
}
//step 2: sort in descending order
if (list.size() >= 2) {
Collections.sort(list, PROBABILITY_SORTING_COMPARATOR);
}
return list;
}
}
NgramFrequencyData.java 0000664 0000000 0000000 00000010372 12775654761 0034221 0 ustar 00root root 0000000 0000000 language-detector-language-detector-0.6/src/main/java/com/optimaize/langdetect package com.optimaize.langdetect;
import com.optimaize.langdetect.i18n.LdLocale;
import com.optimaize.langdetect.profiles.LanguageProfile;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.util.*;
/**
* Contains frequency information for n-grams coming from multiple {@link LanguageProfile}s.
*
* For each n-gram string it knows the locales (languages) in which it occurs, and how frequent it
* occurs in those languages in relation to other n-grams of the same length in those same languages.
*
* Immutable by definition (can't make Arrays unmodifiable).
*
* @author Fabian Kessler
*/
public final class NgramFrequencyData {
/**
* Key = ngram
* Value = array with probabilities per loaded language, in the same order as {@code langlist}.
*/
@NotNull
private final Map wordLangProbMap;
/**
* All the loaded languages, in exactly the same order as the data is in the double[] in wordLangProbMap.
* Example: if wordLangProbMap has an entry for the n-gram "foo" then for each locale in this langlist here
* it has a value there. Languages that don't know the n-gram have the value 0d.
*/
@NotNull
private final List langlist;
/**
* @param gramLengths for example [1,2,3]
* @throws java.lang.IllegalArgumentException if languageProfiles or gramLengths is empty, or if one of the
* languageProfiles does not have the grams of the required sizes.
*/
@NotNull
public static NgramFrequencyData create(@NotNull Collection languageProfiles, @NotNull Collection gramLengths) throws IllegalArgumentException {
if (languageProfiles.isEmpty()) throw new IllegalArgumentException("No languageProfiles provided!");
if (gramLengths.isEmpty()) throw new IllegalArgumentException("No gramLengths provided!");
Map wordLangProbMap = new HashMap<>();
List langlist = new ArrayList<>();
int langsize = languageProfiles.size();
int index = -1;
for (LanguageProfile profile : languageProfiles) {
index++;
langlist.add( profile.getLocale() );
for (Integer gramLength : gramLengths) {
if (!profile.getGramLengths().contains(gramLength)) {
throw new IllegalArgumentException("The language profile for "+profile.getLocale()+" does not contain "+gramLength+"-grams!");
}
for (Map.Entry ngramEntry : profile.iterateGrams(gramLength)) {
String ngram = ngramEntry.getKey();
Integer frequency = ngramEntry.getValue();
if (!wordLangProbMap.containsKey(ngram)) {
wordLangProbMap.put(ngram, new double[langsize]);
}
double prob = frequency.doubleValue() / profile.getNumGramOccurrences(ngram.length());
wordLangProbMap.get(ngram)[index] = prob;
}
}
}
return new NgramFrequencyData(wordLangProbMap, langlist);
}
private NgramFrequencyData(@NotNull Map wordLangProbMap,
@NotNull List langlist) {
//not making immutable copies because I create them here (optimization).
this.wordLangProbMap = Collections.unmodifiableMap(wordLangProbMap);
this.langlist = Collections.unmodifiableList(langlist);
}
@NotNull
public List getLanguageList() {
return langlist;
}
@NotNull
public LdLocale getLanguage(int pos) {
return langlist.get(pos);
}
/**
* Don't modify this data structure! (Can't make array immutable...)
* @return null if no language profile knows that ngram.
* entries are 0 for languages that don't know that ngram at all.
* The array is in the order of the {@link #getLanguageList()} language list, and has exactly that size.
* impl note: this way the caller can handle it more efficient than returning an empty array.
*/
@Nullable
public double[] getProbabilities(String ngram) {
return wordLangProbMap.get(ngram);
}
}
language-detector-language-detector-0.6/src/main/java/com/optimaize/langdetect/cybozu/ 0000775 0000000 0000000 00000000000 12775654761 0031225 5 ustar 00root root 0000000 0000000 CommandLineInterface.java 0000664 0000000 0000000 00000025243 12775654761 0036026 0 ustar 00root root 0000000 0000000 language-detector-language-detector-0.6/src/main/java/com/optimaize/langdetect/cybozu /*
* Copyright 2011 Nakatani Shuyo
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.optimaize.langdetect.cybozu;
import com.optimaize.langdetect.frma.LangProfileWriter;
import com.optimaize.langdetect.cybozu.util.LangProfile;
import com.google.common.base.Optional;
import com.optimaize.langdetect.DetectedLanguage;
import com.optimaize.langdetect.LanguageDetector;
import com.optimaize.langdetect.LanguageDetectorBuilder;
import com.optimaize.langdetect.i18n.LdLocale;
import com.optimaize.langdetect.ngram.NgramExtractors;
import com.optimaize.langdetect.profiles.LanguageProfile;
import com.optimaize.langdetect.profiles.LanguageProfileReader;
import com.optimaize.langdetect.text.CommonTextObjectFactories;
import com.optimaize.langdetect.text.TextObject;
import com.optimaize.langdetect.text.TextObjectFactory;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.io.*;
import java.util.*;
/**
* LangDetect Command Line Interface.
*
* This is a command line interface of Language Detection Library "LangDetect".
*
* Renamed: this class was previously known as "Command".
*
* TODO after my recent changes switching to the new Detector this code is untested. -Fabian
*
* @author Nakatani Shuyo
* @author Francois ROLAND
* @author Fabian Kessler
*/
public class CommandLineInterface {
/** smoothing default parameter (ELE) */
private static final double DEFAULT_ALPHA = 0.5;
/** for Command line easy parser */
private final Map opt_with_value = new HashMap<>();
private final Map values = new HashMap<>();
private final Set opt_without_value = new HashSet<>();
private final List arglist = new ArrayList<>();
/**
* Command Line Interface
* @param args command line arguments
*/
public static void main(String[] args) throws IOException {
CommandLineInterface cli = new CommandLineInterface();
cli.addOpt("-d", "directory", "./");
cli.addOpt("-a", "alpha", "" + DEFAULT_ALPHA);
cli.addOpt("-s", "seed", null);
cli.parse(args);
if (cli.hasParam("--genprofile")) {
cli.generateProfile();
} else if (cli.hasParam("--detectlang")) {
cli.detectLang();
} else if (cli.hasParam("--batchtest")) {
cli.batchTest();
}
}
/**
* Command line easy parser
* @param args command line arguments
*/
private void parse(String[] args) {
for (int i=0; i>>"+value+"<<<", e);
}
}
/**
*/
@Nullable
private Long getParamLongOrNull(String key) {
String value = values.get(key);
if (value == null || value.isEmpty()) {
return null;
}
try {
return Long.valueOf(value);
} catch (NumberFormatException e) {
throw new RuntimeException("Invalid long value: >>>"+value+"<<<", e);
}
}
private boolean hasParam(String opt) {
return opt_without_value.contains(opt);
}
/**
* File search (easy glob)
* @param directory directory path
* @param pattern searching file pattern with regular representation
* @return matched file
*/
private File searchFile(File directory, String pattern) {
if (!directory.isDirectory()) {
throw new IllegalArgumentException("Not a directly: "+directory);
}
File[] files = directory.listFiles();
assert files != null; //checked for directly above.
for (File file : files) {
if (file.getName().matches(pattern)) return file;
}
return null;
}
/**
* Generate Language Profile from a text file.
*
*
* usage: --genprofile [text file] [language name]
*
*
*/
public void generateProfile() {
File directory = new File(arglist.get(0));
String lang = arglist.get(1);
File file = searchFile(directory, lang + "wiki-.*-abstract\\.xml.*");
if (file == null) {
System.err.println("Not Found text file : lang = " + lang);
return;
}
try(FileOutputStream outputStream = new FileOutputStream(new File(lang))) {
LangProfile profile = GenProfile.load(lang, file);
profile.omitLessFreq();
new LangProfileWriter().write(profile, outputStream);
} catch (IOException e) {
e.printStackTrace();
}
}
/**
* Language detection test for each file (--detectlang option)
*
*
* usage: --detectlang -d [profile directory] -a [alpha] -s [seed] [test file(s)]
*
*
*/
public void detectLang() throws IOException {
LanguageDetector languageDetector = makeDetector();
TextObjectFactory textObjectFactory = CommonTextObjectFactories.forDetectingOnLargeText();
for (String filename: arglist) {
try (BufferedReader is = new BufferedReader(new InputStreamReader(new FileInputStream(filename), "utf-8"))) {
TextObject textObject = textObjectFactory.create().append(is);
List probabilities = languageDetector.getProbabilities(textObject);
System.out.println(filename + ":" + probabilities);
}
}
}
/**
* Batch Test of Language Detection (--batchtest option)
*
*
* usage: --batchtest -d [profile directory] -a [alpha] -s [seed] [test data(s)]
*
*
* The format of test data(s):
*
* [correct language name]\t[text body for test]\n
*
*
*/
public void batchTest() throws IOException {
LanguageDetector languageDetector = makeDetector();
TextObjectFactory textObjectFactory = CommonTextObjectFactories.forDetectingOnLargeText();
Map> result = new HashMap<>();
for (String filename : arglist) {
try (BufferedReader is = new BufferedReader(new InputStreamReader(new FileInputStream(filename), "utf-8"))) {
while (is.ready()) {
String line = is.readLine();
int idx = line.indexOf('\t');
if (idx <= 0) continue;
String correctLang = line.substring(0, idx);
String text = line.substring(idx + 1);
TextObject textObject = textObjectFactory.forText(text);
Optional lang = languageDetector.detect(textObject);
if (!result.containsKey(correctLang)) result.put(correctLang, new ArrayList());
if (lang.isPresent()) {
result.get(correctLang).add(lang.toString());
} else {
result.get(correctLang).add("unknown");
}
if (hasParam("--debug")) System.out.println(correctLang + "," + lang + "," + (text.length() > 100 ? text.substring(0, 100) : text));
}
}
List langList = new ArrayList<>(result.keySet());
Collections.sort(langList);
int totalCount = 0, totalCorrect = 0;
for (String lang : langList) {
Map resultCount = new HashMap<>();
int count = 0;
List list = result.get(lang);
for (String detectedLang: list) {
++count;
if (resultCount.containsKey(detectedLang)) {
resultCount.put(detectedLang, resultCount.get(detectedLang) + 1);
} else {
resultCount.put(detectedLang, 1);
}
}
int correct = resultCount.containsKey(lang)?resultCount.get(lang):0;
double rate = correct / (double)count;
System.out.println(String.format("%s (%d/%d=%.2f): %s", lang, correct, count, rate, resultCount));
totalCorrect += correct;
totalCount += count;
}
System.out.println(String.format("total: %d/%d = %.3f", totalCorrect, totalCount, totalCorrect / (double) totalCount));
}
}
/**
* Using all language profiles from the given directory.
*/
private LanguageDetector makeDetector() throws IOException {
double alpha = getParamDouble("alpha", DEFAULT_ALPHA);
String profileDirectory = requireParamString("directory") + "/";
Optional seed = Optional.fromNullable(getParamLongOrNull("seed"));
List languageProfiles = new LanguageProfileReader().readAll(new File(profileDirectory));
return LanguageDetectorBuilder.create(NgramExtractors.standard())
.alpha(alpha)
.seed(seed)
.shortTextAlgorithm(50)
.withProfiles(languageProfiles)
.build();
}
}
GenProfile.java 0000664 0000000 0000000 00000006237 12775654761 0034053 0 ustar 00root root 0000000 0000000 language-detector-language-detector-0.6/src/main/java/com/optimaize/langdetect/cybozu /*
* Copyright 2011 Nakatani Shuyo
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.optimaize.langdetect.cybozu;
import com.optimaize.langdetect.cybozu.util.TagExtractor;
import com.optimaize.langdetect.cybozu.util.LangProfile;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import java.io.*;
import java.util.zip.GZIPInputStream;
/**
* Load Wikipedia's abstract XML as corpus and generate its language profile in JSON format.
*
* @author Nakatani Shuyo
*/
public class GenProfile {
private static final Logger logger = LoggerFactory.getLogger(GenProfile.class);
/**
* Load Wikipedia abstract database file and generate its language profile
* @param lang target language name
* @param file target database file path
* @return Language profile instance
*/
public static LangProfile load(String lang, File file) {
LangProfile profile = new LangProfile(lang);
try (InputStream is = file.getName().endsWith(".gz") ?
new GZIPInputStream(new BufferedInputStream(new FileInputStream(file))) :
new BufferedInputStream(new FileInputStream(file))) {
TagExtractor tagextractor = new TagExtractor("abstract", 100);
XMLStreamReader reader = null;
try {
XMLInputFactory factory = XMLInputFactory.newInstance();
reader = factory.createXMLStreamReader(is);
while (reader.hasNext()) {
switch (reader.next()) {
case XMLStreamReader.START_ELEMENT:
tagextractor.setTag(reader.getName().toString());
break;
case XMLStreamReader.CHARACTERS:
tagextractor.add(reader.getText());
break;
case XMLStreamReader.END_ELEMENT:
tagextractor.closeTag(profile);
break;
}
}
} catch (XMLStreamException e) {
throw new RuntimeException("Training database file '" + file.getName() + "' is an invalid XML.", e);
} finally {
try {
if (reader != null) reader.close();
} catch (XMLStreamException e) { /* ignore exception */ }
}
logger.info(lang + ":" + tagextractor.count());
} catch (IOException e) {
throw new RuntimeException("Can't open training database file '" + file.getName() + "'", e);
}
return profile;
}
}
language-detector-language-detector-0.6/src/main/java/com/optimaize/langdetect/cybozu/package.html 0000664 0000000 0000000 00000000150 12775654761 0033502 0 ustar 00root root 0000000 0000000
Original language detection classes from https://code.google.com/p/language-detection/
language-detector-language-detector-0.6/src/main/java/com/optimaize/langdetect/cybozu/util/ 0000775 0000000 0000000 00000000000 12775654761 0032202 5 ustar 00root root 0000000 0000000 CharNormalizer.java 0000664 0000000 0000000 00000021217 12775654761 0035711 0 ustar 00root root 0000000 0000000 language-detector-language-detector-0.6/src/main/java/com/optimaize/langdetect/cybozu/util package com.optimaize.langdetect.cybozu.util;
import java.util.HashMap;
import java.util.Map;
/**
* Some character normalization (and exclusion) functionality.
*
* This functionality was previously embedded in the NGram class.
*
* @author Nakatani Shuyo
* @author Fabian Kessler
*/
public class CharNormalizer {
/**
* Character Normalization (and exclusion).
* @return Normalized character, the space to exclude the character.
*/
public static char normalize(char ch) {
Character.UnicodeBlock block = Character.UnicodeBlock.of(ch);
if (block == Character.UnicodeBlock.BASIC_LATIN) {
if (ch<'A' || (ch<'a' && ch >'Z') || ch>'z') ch = ' ';
} else if (block == Character.UnicodeBlock.LATIN_1_SUPPLEMENT) {
if (LATIN1_EXCLUDED.indexOf(ch)>=0) ch = ' ';
} else if (block == Character.UnicodeBlock.GENERAL_PUNCTUATION) {
ch = ' ';
} else if (block == Character.UnicodeBlock.ARABIC) {
if (ch == '\u06cc') ch = '\u064a';
} else if (block == Character.UnicodeBlock.LATIN_EXTENDED_ADDITIONAL) {
if (ch >= '\u1ea0') ch = '\u1ec3';
} else if (block == Character.UnicodeBlock.HIRAGANA) {
ch = '\u3042';
} else if (block == Character.UnicodeBlock.KATAKANA) {
ch = '\u30a2';
} else if (block == Character.UnicodeBlock.BOPOMOFO || block == Character.UnicodeBlock.BOPOMOFO_EXTENDED) {
ch = '\u3105';
} else if (block == Character.UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS) {
if (cjk_map.containsKey(ch)) ch = cjk_map.get(ch);
} else if (block == Character.UnicodeBlock.HANGUL_SYLLABLES) {
ch = '\uac00';
}
return ch;
}
private static final String LATIN1_EXCLUDED = Messages.getString("NGram.LATIN1_EXCLUDE");
private static final Map cjk_map;
/**
* CJK Kanji Normalization Mapping
*/
private static final String[] CJK_CLASS = {
Messages.getString("NGram.KANJI_1_0"),
Messages.getString("NGram.KANJI_1_2"),
Messages.getString("NGram.KANJI_1_4"),
Messages.getString("NGram.KANJI_1_8"),
Messages.getString("NGram.KANJI_1_11"),
Messages.getString("NGram.KANJI_1_12"),
Messages.getString("NGram.KANJI_1_13"),
Messages.getString("NGram.KANJI_1_14"),
Messages.getString("NGram.KANJI_1_16"),
Messages.getString("NGram.KANJI_1_18"),
Messages.getString("NGram.KANJI_1_22"),
Messages.getString("NGram.KANJI_1_27"),
Messages.getString("NGram.KANJI_1_29"),
Messages.getString("NGram.KANJI_1_31"),
Messages.getString("NGram.KANJI_1_35"),
Messages.getString("NGram.KANJI_2_0"),
Messages.getString("NGram.KANJI_2_1"),
Messages.getString("NGram.KANJI_2_4"),
Messages.getString("NGram.KANJI_2_9"),
Messages.getString("NGram.KANJI_2_10"),
Messages.getString("NGram.KANJI_2_11"),
Messages.getString("NGram.KANJI_2_12"),
Messages.getString("NGram.KANJI_2_13"),
Messages.getString("NGram.KANJI_2_15"),
Messages.getString("NGram.KANJI_2_16"),
Messages.getString("NGram.KANJI_2_18"),
Messages.getString("NGram.KANJI_2_21"),
Messages.getString("NGram.KANJI_2_22"),
Messages.getString("NGram.KANJI_2_23"),
Messages.getString("NGram.KANJI_2_28"),
Messages.getString("NGram.KANJI_2_29"),
Messages.getString("NGram.KANJI_2_30"),
Messages.getString("NGram.KANJI_2_31"),
Messages.getString("NGram.KANJI_2_32"),
Messages.getString("NGram.KANJI_2_35"),
Messages.getString("NGram.KANJI_2_36"),
Messages.getString("NGram.KANJI_2_37"),
Messages.getString("NGram.KANJI_2_38"),
Messages.getString("NGram.KANJI_3_1"),
Messages.getString("NGram.KANJI_3_2"),
Messages.getString("NGram.KANJI_3_3"),
Messages.getString("NGram.KANJI_3_4"),
Messages.getString("NGram.KANJI_3_5"),
Messages.getString("NGram.KANJI_3_8"),
Messages.getString("NGram.KANJI_3_9"),
Messages.getString("NGram.KANJI_3_11"),
Messages.getString("NGram.KANJI_3_12"),
Messages.getString("NGram.KANJI_3_13"),
Messages.getString("NGram.KANJI_3_15"),
Messages.getString("NGram.KANJI_3_16"),
Messages.getString("NGram.KANJI_3_18"),
Messages.getString("NGram.KANJI_3_19"),
Messages.getString("NGram.KANJI_3_22"),
Messages.getString("NGram.KANJI_3_23"),
Messages.getString("NGram.KANJI_3_27"),
Messages.getString("NGram.KANJI_3_29"),
Messages.getString("NGram.KANJI_3_30"),
Messages.getString("NGram.KANJI_3_31"),
Messages.getString("NGram.KANJI_3_32"),
Messages.getString("NGram.KANJI_3_35"),
Messages.getString("NGram.KANJI_3_36"),
Messages.getString("NGram.KANJI_3_37"),
Messages.getString("NGram.KANJI_3_38"),
Messages.getString("NGram.KANJI_4_0"),
Messages.getString("NGram.KANJI_4_9"),
Messages.getString("NGram.KANJI_4_10"),
Messages.getString("NGram.KANJI_4_16"),
Messages.getString("NGram.KANJI_4_17"),
Messages.getString("NGram.KANJI_4_18"),
Messages.getString("NGram.KANJI_4_22"),
Messages.getString("NGram.KANJI_4_24"),
Messages.getString("NGram.KANJI_4_28"),
Messages.getString("NGram.KANJI_4_34"),
Messages.getString("NGram.KANJI_4_39"),
Messages.getString("NGram.KANJI_5_10"),
Messages.getString("NGram.KANJI_5_11"),
Messages.getString("NGram.KANJI_5_12"),
Messages.getString("NGram.KANJI_5_13"),
Messages.getString("NGram.KANJI_5_14"),
Messages.getString("NGram.KANJI_5_18"),
Messages.getString("NGram.KANJI_5_26"),
Messages.getString("NGram.KANJI_5_29"),
Messages.getString("NGram.KANJI_5_34"),
Messages.getString("NGram.KANJI_5_39"),
Messages.getString("NGram.KANJI_6_0"),
Messages.getString("NGram.KANJI_6_3"),
Messages.getString("NGram.KANJI_6_9"),
Messages.getString("NGram.KANJI_6_10"),
Messages.getString("NGram.KANJI_6_11"),
Messages.getString("NGram.KANJI_6_12"),
Messages.getString("NGram.KANJI_6_16"),
Messages.getString("NGram.KANJI_6_18"),
Messages.getString("NGram.KANJI_6_20"),
Messages.getString("NGram.KANJI_6_21"),
Messages.getString("NGram.KANJI_6_22"),
Messages.getString("NGram.KANJI_6_23"),
Messages.getString("NGram.KANJI_6_25"),
Messages.getString("NGram.KANJI_6_28"),
Messages.getString("NGram.KANJI_6_29"),
Messages.getString("NGram.KANJI_6_30"),
Messages.getString("NGram.KANJI_6_32"),
Messages.getString("NGram.KANJI_6_34"),
Messages.getString("NGram.KANJI_6_35"),
Messages.getString("NGram.KANJI_6_37"),
Messages.getString("NGram.KANJI_6_39"),
Messages.getString("NGram.KANJI_7_0"),
Messages.getString("NGram.KANJI_7_3"),
Messages.getString("NGram.KANJI_7_6"),
Messages.getString("NGram.KANJI_7_7"),
Messages.getString("NGram.KANJI_7_9"),
Messages.getString("NGram.KANJI_7_11"),
Messages.getString("NGram.KANJI_7_12"),
Messages.getString("NGram.KANJI_7_13"),
Messages.getString("NGram.KANJI_7_16"),
Messages.getString("NGram.KANJI_7_18"),
Messages.getString("NGram.KANJI_7_19"),
Messages.getString("NGram.KANJI_7_20"),
Messages.getString("NGram.KANJI_7_21"),
Messages.getString("NGram.KANJI_7_23"),
Messages.getString("NGram.KANJI_7_25"),
Messages.getString("NGram.KANJI_7_28"),
Messages.getString("NGram.KANJI_7_29"),
Messages.getString("NGram.KANJI_7_32"),
Messages.getString("NGram.KANJI_7_33"),
Messages.getString("NGram.KANJI_7_35"),
Messages.getString("NGram.KANJI_7_37"),
};
static {
cjk_map = new HashMap<>();
for (String cjk_list : CJK_CLASS) {
char representative = cjk_list.charAt(0);
for (int i=0;i freq = new HashMap<>();
/**
* Tells how many occurrences of n-grams exist per gram length.
* When making 1grams, 2grams and 3grams (currently) then this contains 3 entries where
* element 0 = number occurrences of 1-grams
* element 1 = number occurrences of 2-grams
* element 2 = number occurrences of 3-grams
* Example: if there are 57 1-grams (English language has about that many) and the training text is
* fairly long, then this number is in the millions.
*/
private int[] nWords = new int[NGram.N_GRAM];
/**
* Constructor for JSONIC
*/
public LangProfile() {}
/**
* Normal Constructor
* @param name language name
*/
public LangProfile(String name) {
this.setName(name);
}
/**
* Add n-gram to profile
* @param gram
*/
public void add(@NotNull String gram) {
if (name == null) throw new IllegalStateException();
int len = gram.length();
if (len < 1 || len > NGram.N_GRAM) {
throw new IllegalArgumentException("ngram length must be 1-3 but was "+len+": >>>"+gram+"<< keys = freq.keySet();
int roman = 0;
for(Iterator i = keys.iterator(); i.hasNext(); ){
String key = i.next();
int count = freq.get(key);
if (count <= threshold) {
nWords[key.length()-1] -= count;
i.remove();
} else {
if (key.matches("^[A-Za-z]$")) {
roman += count;
}
}
}
// roman check
if (roman < nWords[0] / 3) {
Set keys2 = freq.keySet();
for(Iterator i = keys2.iterator(); i.hasNext(); ){
String key = i.next();
if (key.matches(".*[A-Za-z].*")) {
nWords[key.length()-1] -= freq.get(key);
i.remove();
}
}
}
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public Map getFreq() {
return freq;
}
public void setFreq(Map freq) {
this.freq = freq;
}
public int[] getNWords() {
return nWords;
}
public void setNWords(int[] nWords) {
this.nWords = nWords;
}
}
Messages.java 0000664 0000000 0000000 00000002505 12775654761 0034537 0 ustar 00root root 0000000 0000000 language-detector-language-detector-0.6/src/main/java/com/optimaize/langdetect/cybozu/util /*
* Copyright 2011 Nakatani Shuyo
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.optimaize.langdetect.cybozu.util;
import java.util.MissingResourceException;
import java.util.ResourceBundle;
/**
* This is {@link Messages} class generated by Eclipse automatically.
* Users don't use this class directly.
*
* @author Nakatani Shuyo
*/
public class Messages {
private static final String BUNDLE_NAME = "com.optimaize.langdetect.cybozu.util.messages"; //$NON-NLS-1$
private static final ResourceBundle RESOURCE_BUNDLE = ResourceBundle.getBundle(BUNDLE_NAME);
private Messages() {
}
public static String getString(String key) {
try {
return RESOURCE_BUNDLE.getString(key);
} catch (MissingResourceException e) {
return '!' + key + '!';
}
}
}
NGram.java 0000664 0000000 0000000 00000004675 12775654761 0034006 0 ustar 00root root 0000000 0000000 language-detector-language-detector-0.6/src/main/java/com/optimaize/langdetect/cybozu/util /*
* Copyright 2011 Nakatani Shuyo
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.optimaize.langdetect.cybozu.util;
import org.jetbrains.annotations.Nullable;
/**
* TODO document.
*
* Users don't use this class directly.
*
* TODO this class treats a word as "upper case" if the first 2 characters are upper case. That seems like a simplification,
* would need documentation.
*
* @author Nakatani Shuyo
*/
public class NGram {
/**
* ngrams are created from 1gram to this amount, currently 2grams and 3grams.
*/
public static final int N_GRAM = 3;
private StringBuilder grams_;
private boolean capitalword_;
public NGram() {
grams_ = new StringBuilder(" ");
capitalword_ = false;
}
public void addChar(char ch) {
ch = CharNormalizer.normalize(ch);
char lastChar = grams_.charAt(grams_.length() - 1);
if (lastChar == ' ') {
grams_ = new StringBuilder(" ");
capitalword_ = false;
if (ch==' ') return;
} else if (grams_.length() >= N_GRAM) {
grams_.deleteCharAt(0);
}
grams_.append(ch);
if (Character.isUpperCase(ch)){
if (Character.isUpperCase(lastChar)) capitalword_ = true;
} else {
capitalword_ = false;
}
}
/**
* TODO this method has some weird, undocumented behavior to ignore ngrams with upper case.
*
* Get n-Gram
* @param n length of n-gram
* @return n-Gram String (null if it is invalid)
*/
@Nullable
public String get(int n) {
if (capitalword_) return null;
int len = grams_.length();
if (n < 1 || n > N_GRAM || len < n) return null;
if (n == 1) {
char ch = grams_.charAt(len - 1);
if (ch == ' ') return null;
return Character.toString(ch);
} else {
return grams_.substring(len - n, len);
}
}
}
TagExtractor.java 0000664 0000000 0000000 00000004134 12775654761 0035377 0 ustar 00root root 0000000 0000000 language-detector-language-detector-0.6/src/main/java/com/optimaize/langdetect/cybozu/util /*
* Copyright 2011 Nakatani Shuyo
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.optimaize.langdetect.cybozu.util;
import com.optimaize.langdetect.text.CommonTextObjectFactories;
import com.optimaize.langdetect.text.TextObjectFactory;
/**
* {@link TagExtractor} is a class which extracts inner texts of specified tag.
* Users don't use this class directly.
* @author Nakatani Shuyo
*/
public class TagExtractor {
private static final TextObjectFactory textObjectFactory = CommonTextObjectFactories.forIndexing();
/* package scope */ String target_;
/* package scope */ int threshold_;
/* package scope */ StringBuilder buf_;
/* package scope */ String tag_;
private int count_;
public TagExtractor(String tag, int threshold) {
target_ = tag;
threshold_ = threshold;
count_ = 0;
clear();
}
public int count() {
return count_;
}
public void clear() {
buf_ = new StringBuilder(" ");
tag_ = null;
}
public void setTag(String tag){
tag_ = tag;
}
public void add(String line) {
if (tag_ != null && tag_.equals(target_) && line != null) {
buf_.append(line);
}
}
public void closeTag(LangProfile profile) {
if ((profile != null) && tag_.equals(target_) && (buf_.length() > threshold_) && !isSpace()) {
Util.addCharSequence(profile, textObjectFactory.forText(buf_));
++count_;
}
clear();
}
private boolean isSpace() {
return (buf_.length()==1 && buf_.toString().equals(" "));
}
}
language-detector-language-detector-0.6/src/main/java/com/optimaize/langdetect/cybozu/util/Util.java0000664 0000000 0000000 00000007057 12775654761 0033773 0 ustar 00root root 0000000 0000000 package com.optimaize.langdetect.cybozu.util;
import com.optimaize.langdetect.i18n.LdLocale;
import com.optimaize.langdetect.ngram.NgramExtractor;
import com.optimaize.langdetect.ngram.NgramExtractors;
import com.optimaize.langdetect.ngram.OldNgramExtractor;
import org.jetbrains.annotations.NotNull;
import java.util.Formatter;
import java.util.List;
import java.util.Map;
/**
* A place for sharing code.
*
* @author Fabian Kessler
*/
public class Util {
private static final NgramExtractor ngramExtractor = NgramExtractors.standard();
public static void addCharSequence(LangProfile langProfile, CharSequence text) {
//TODO replace with new code.
// List old = OldNgramExtractor.extractNGrams(text, null);
// List nuu = ngramExtractor.extractGrams(text);
//
// Set oldSet = new HashSet<>(old);
// Set nuuSet = new HashSet<>(nuu);
//
// ArrayList justNuu = new ArrayList<>(nuu);
// justNuu.removeAll(old);
//
// ArrayList justOld = new ArrayList<>(old);
// justOld.removeAll(nuu);
//
// System.out.println(text);
// for (String s : ngramExtractor.extractGrams(text)) {
// langProfile.add(s);
// }
for (String s : OldNgramExtractor.extractNGrams(text, null)) {
langProfile.add(s);
}
}
/**
* unicode encoding (for verbose mode)
*/
public static String unicodeEncode(String s) {
StringBuilder buf = new StringBuilder();
for (int i = 0; i < s.length(); ++i) {
char ch = s.charAt(i);
if (ch >= '\u0080') {
String st = Integer.toHexString(0x10000 + (int) ch);
while (st.length() < 4) st = "0" + st;
buf.append("\\u").append(st.subSequence(1, 5));
} else {
buf.append(ch);
}
}
return buf.toString();
}
/**
* normalize probabilities and check convergence by the maximum probability
* @return maximum of probabilities
*/
public static double normalizeProb(double[] prob) {
double maxp = 0, sump = 0;
for(int i=0;i langlist) {
Formatter formatter = new Formatter();
for(int j=0;j=0.00001) {
formatter.format(" %s:%.5f", langlist.get(j), p);
}
}
return formatter.toString();
}
/**
*/
public static double[] makeInternalPrioMap(@NotNull Map langWeightingMap,
@NotNull List langlist) {
assert !langWeightingMap.isEmpty();
double[] priorMap = new double[langlist.size()];
double sump = 0;
for (int i=0;i=0 : "Prior probability must be non-negative!";
priorMap[i] = p;
sump += p;
}
}
assert sump > 0 : "Sum must be greater than zero!";
for (int i=0;i
Provides the utility classes for language detection.
Users don't use this package's classes directly.