diff --git a/src/main/java/org/codelibs/fess/helper/QueryHelper.java b/src/main/java/org/codelibs/fess/helper/QueryHelper.java
index 1d4cf97f7b420a6b4227bf92e0eb0a33ab30e85d..489509d3a26eb01c6c899ef957e1da07a9f3ecd3 100644
--- a/src/main/java/org/codelibs/fess/helper/QueryHelper.java
+++ b/src/main/java/org/codelibs/fess/helper/QueryHelper.java
@@ -569,19 +569,17 @@ public class QueryHelper {
builder.apply(fessConfig.getIndexFieldContent(), fessConfig.getQueryBoostContentAsDecimal().floatValue());
boolQuery.should(contentQuery);
getQueryLanguages().ifPresent(
- langs -> {
- stream(langs).of(
- stream -> stream.forEach(lang -> {
- final QueryBuilder titleLangQuery =
- builder.apply(fessConfig.getIndexFieldTitle() + "_" + lang, fessConfig
- .getQueryBoostTitleLangAsDecimal().floatValue());
- boolQuery.should(titleLangQuery);
- final QueryBuilder contentLangQuery =
- builder.apply(fessConfig.getIndexFieldContent() + "_" + lang, fessConfig
- .getQueryBoostContentLangAsDecimal().floatValue());
- boolQuery.should(contentLangQuery);
- }));
- });
+ langs -> stream(langs).of(
+ stream -> stream.forEach(lang -> {
+ final QueryBuilder titleLangQuery =
+ builder.apply(fessConfig.getIndexFieldTitle() + "_" + lang, fessConfig
+ .getQueryBoostTitleLangAsDecimal().floatValue());
+ boolQuery.should(titleLangQuery);
+ final QueryBuilder contentLangQuery =
+ builder.apply(fessConfig.getIndexFieldContent() + "_" + lang, fessConfig
+ .getQueryBoostContentLangAsDecimal().floatValue());
+ boolQuery.should(contentLangQuery);
+ })));
return boolQuery;
}
diff --git a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java
index 4d70efb176d986b40fad628026e0d3d151444ccb..d0f513065d20c5e94b9df988ae72e3dbf01b441c 100644
--- a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java
+++ b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java
@@ -124,7 +124,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/** The key of the configuration. e.g. license.properties */
String SUPPORTED_UPLOADED_FILES = "supported.uploaded.files";
- /** The key of the configuration. e.g. ar,bg,ca,da,de,el,en,es,eu,fa,fi,fr,ga,gl,hi,hu,hy,id,it,ja,lv,ko,nl,no,pt,ro,ru,sv,th,tr,zh_CN,zh_TW,zh */
+ /** The key of the configuration. e.g. ar,bg,bn,ca,ckb_IQ,cs,da,de,el,en,en_IE,es,et,eu,fa,fi,fr,gl,gu,he,hi,hr,hu,hy,id,it,ja,ko,lt,lv,mk,ml,nl,no,pa,pl,pt,pt_BR,ro,ru,si,sq,sv,ta,te,th,tl,tr,uk,ur,vi,zh_CN,zh_TW,zh */
String SUPPORTED_LANGUAGES = "supported.languages";
/** The key of the configuration. e.g. 60 */
@@ -538,12 +538,15 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
bg=bg
bn=bn
ca=ca
+ ckb-iq=ckb-iq
+ ckb_IQ=ckb-iq
cs=cs
da=da
de=de
el=el
en=en
en-ie=en-ie
+ en_IE=en-ie
es=es
et=et
eu=eu
@@ -571,6 +574,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
pl=pl
pt=pt
pt-br=pt-br
+ pt_BR=pt-br
ro=ro
ru=ru
si=si
@@ -584,9 +588,11 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
uk=uk
ur=ur
vi=vi
- zh=zh-cn
zh-cn=zh-cn
+ zh_CN=zh-cn
zh-tw=zh-tw
+ zh_TW=zh-tw
+ zh=zh
*/
String QUERY_LANGUAGE_MAPPING = "query.language.mapping";
@@ -1469,7 +1475,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/**
* Get the value for the key 'supported.languages'. <br>
- * The value is, e.g. ar,bg,ca,da,de,el,en,es,eu,fa,fi,fr,ga,gl,hi,hu,hy,id,it,ja,lv,ko,nl,no,pt,ro,ru,sv,th,tr,zh_CN,zh_TW,zh <br>
+ * The value is, e.g. ar,bg,bn,ca,ckb_IQ,cs,da,de,el,en,en_IE,es,et,eu,fa,fi,fr,gl,gu,he,hi,hr,hu,hy,id,it,ja,ko,lt,lv,mk,ml,nl,no,pa,pl,pt,pt_BR,ro,ru,si,sq,sv,ta,te,th,tl,tr,uk,ur,vi,zh_CN,zh_TW,zh <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getSupportedLanguages();
@@ -2884,12 +2890,15 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
bg=bg
bn=bn
ca=ca
+ ckb-iq=ckb-iq
+ ckb_IQ=ckb-iq
cs=cs
da=da
de=de
el=el
en=en
en-ie=en-ie
+ en_IE=en-ie
es=es
et=et
eu=eu
@@ -2917,6 +2926,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
pl=pl
pt=pt
pt-br=pt-br
+ pt_BR=pt-br
ro=ro
ru=ru
si=si
@@ -2930,9 +2940,11 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
uk=uk
ur=ur
vi=vi
- zh=zh-cn
zh-cn=zh-cn
+ zh_CN=zh-cn
zh-tw=zh-tw
+ zh_TW=zh-tw
+ zh=zh
<br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
@@ -7270,8 +7282,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
defaultMap.put(FessConfig.SUPPORTED_UPLOADED_CSS_EXTENTIONS, "css");
defaultMap.put(FessConfig.SUPPORTED_UPLOADED_MEDIA_EXTENTIONS, "jpg,jpeg,gif,png,swf");
defaultMap.put(FessConfig.SUPPORTED_UPLOADED_FILES, "license.properties");
- defaultMap.put(FessConfig.SUPPORTED_LANGUAGES,
- "ar,bg,ca,da,de,el,en,es,eu,fa,fi,fr,ga,gl,hi,hu,hy,id,it,ja,lv,ko,nl,no,pt,ro,ru,sv,th,tr,zh_CN,zh_TW,zh");
+ defaultMap
+ .put(FessConfig.SUPPORTED_LANGUAGES,
+ "ar,bg,bn,ca,ckb_IQ,cs,da,de,el,en,en_IE,es,et,eu,fa,fi,fr,gl,gu,he,hi,hr,hu,hy,id,it,ja,ko,lt,lv,mk,ml,nl,no,pa,pl,pt,pt_BR,ro,ru,si,sq,sv,ta,te,th,tl,tr,uk,ur,vi,zh_CN,zh_TW,zh");
defaultMap.put(FessConfig.API_ACCESS_TOKEN_LENGTH, "60");
defaultMap.put(FessConfig.API_ACCESS_TOKEN_REQUIRED, "false");
defaultMap.put(FessConfig.API_ADMIN_ACCESS_PERMISSIONS, "Radmin-api");
@@ -7411,7 +7424,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
defaultMap.put(FessConfig.QUERY_DEFAULT_LANGUAGES, "");
defaultMap
.put(FessConfig.QUERY_LANGUAGE_MAPPING,
- "ar=ar\nbg=bg\nbn=bn\nca=ca\ncs=cs\nda=da\nde=de\nel=el\nen=en\nen-ie=en-ie\nes=es\net=et\neu=eu\nfa=fa\nfi=fi\nfr=fr\ngl=gl\ngu=gu\nhe=he\nhi=hi\nhr=hr\nhu=hu\nhy=hy\nid=id\nit=it\nja=ja\nko=ko\nlt=lt\nlv=lv\nmk=mk\nml=ml\nnl=nl\nno=no\npa=pa\npl=pl\npt=pt\npt-br=pt-br\nro=ro\nru=ru\nsi=si\nsq=sq\nsv=sv\nta=ta\nte=te\nth=th\ntl=tl\ntr=tr\nuk=uk\nur=ur\nvi=vi\nzh=zh-cn\nzh-cn=zh-cn\nzh-tw=zh-tw\n");
+ "ar=ar\nbg=bg\nbn=bn\nca=ca\nckb-iq=ckb-iq\nckb_IQ=ckb-iq\ncs=cs\nda=da\nde=de\nel=el\nen=en\nen-ie=en-ie\nen_IE=en-ie\nes=es\net=et\neu=eu\nfa=fa\nfi=fi\nfr=fr\ngl=gl\ngu=gu\nhe=he\nhi=hi\nhr=hr\nhu=hu\nhy=hy\nid=id\nit=it\nja=ja\nko=ko\nlt=lt\nlv=lv\nmk=mk\nml=ml\nnl=nl\nno=no\npa=pa\npl=pl\npt=pt\npt-br=pt-br\npt_BR=pt-br\nro=ro\nru=ru\nsi=si\nsq=sq\nsv=sv\nta=ta\nte=te\nth=th\ntl=tl\ntr=tr\nuk=uk\nur=ur\nvi=vi\nzh-cn=zh-cn\nzh_CN=zh-cn\nzh-tw=zh-tw\nzh_TW=zh-tw\nzh=zh\n");
defaultMap.put(FessConfig.QUERY_BOOST_TITLE, "0.2");
defaultMap.put(FessConfig.QUERY_BOOST_TITLE_LANG, "1.0");
defaultMap.put(FessConfig.QUERY_BOOST_CONTENT, "0.1");
diff --git a/src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java b/src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java
index 1165ce826d6e745490149657670d1821192ed378..17d0d425154d722d27b8bf6ad9f6265f5f58adc1 100644
--- a/src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java
+++ b/src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java
@@ -68,6 +68,8 @@ import org.lastaflute.web.validation.theme.typed.LongTypeValidator;
public interface FessProp {
+ public static final String QUERY_DEFAULT_LANGUAGES = "queryDefaultLanguages";
+
public static final String HTML_PROXY = "httpProxy";
public static final String CRAWLER_FAILURE_URL_STATUS_CODES = "crawlerFailureUrlStatusCodes";
@@ -848,55 +850,67 @@ public interface FessProp {
String getQueryLanguageMapping();
+ public default String[] normalizeQueryLanguages(final String[] langs) {
+ @SuppressWarnings("unchecked")
+ Map<String, String> params = (Map<String, String>) propMap.get(QUERY_LANGUAGE_MAPPING);
+ if (params == null) {
+ params = stream(getQueryLanguageMapping().split("\n")).get(stream -> stream.filter(StringUtil::isNotBlank).map(v -> {
+ final String[] values = v.split("=");
+ if (values.length == 2) {
+ return new Pair<>(values[0], values[1]);
+ }
+ return null;
+ }).collect(Collectors.toMap(Pair::getFirst, d -> d.getSecond())));
+ propMap.put(QUERY_LANGUAGE_MAPPING, params);
+ }
+ final Map<String, String> mapping = params;
+ return stream(langs).get(stream -> stream.map(s -> {
+ if (StringUtil.isBlank(s)) {
+ return null;
+ }
+ final String lang1 = mapping.get(s);
+ if (lang1 != null) {
+ return lang1;
+ }
+ final String lang2 = mapping.get(s.split("[\\-_]")[0]);
+ if (lang2 != null) {
+ return lang2;
+ }
+ return null;
+ }).filter(StringUtil::isNotBlank).distinct().toArray(n -> new String[n]));
+ }
+
public default String[] getQueryLanguages(final Enumeration<Locale> locales, final String[] requestLangs) {
+ // requestLangs > default > browser
if (StringUtil.isNotBlank(getQueryDefaultLanguages())) {
- String[] langs = (String[]) propMap.get("queryDefaultLanguages");
+ String[] langs = (String[]) propMap.get(QUERY_DEFAULT_LANGUAGES);
if (langs == null) {
langs = split(getQueryDefaultLanguages(), ",").get(stream -> stream.map(s -> s.trim()).toArray(n -> new String[n]));
- propMap.put("queryDefaultLanguages", langs);
+ propMap.put(QUERY_DEFAULT_LANGUAGES, langs);
}
- return langs;
+ return normalizeQueryLanguages(langs);
}
if (requestLangs != null && requestLangs.length != 0) {
- return requestLangs;
+ return normalizeQueryLanguages(requestLangs);
}
if (locales == null) {
return StringUtil.EMPTY_STRINGS;
}
- @SuppressWarnings("unchecked")
- Map<String, String> params = (Map<String, String>) propMap.get(QUERY_LANGUAGE_MAPPING);
- if (params == null) {
- params = stream(getQueryLanguageMapping().split("\n")).get(stream -> stream.filter(StringUtil::isNotBlank).map(v -> {
- final String[] values = v.split("=");
- if (values.length == 2) {
- return new Pair<>(values[0], values[1]);
- }
- return null;
- }).collect(Collectors.toMap(Pair::getFirst, d -> d.getSecond())));
- propMap.put(QUERY_LANGUAGE_MAPPING, params);
- }
-
- final Map<String, String> mapping = params;
- return Collections.list(locales).stream().map(locale -> {
+ return normalizeQueryLanguages(Collections.list(locales).stream().map(locale -> {
final String language = locale.getLanguage();
final String country = locale.getCountry();
if (StringUtil.isNotBlank(language)) {
if (StringUtil.isNotBlank(country)) {
- final String lang = language.toLowerCase(Locale.ROOT) + "-" + country.toLowerCase(Locale.ROOT);
- if (mapping.containsKey(lang)) {
- return mapping.get(lang);
- }
- }
- if (mapping.containsKey(language)) {
- return mapping.get(language);
+ return language.toLowerCase(Locale.ROOT) + "-" + country.toLowerCase(Locale.ROOT);
}
+ return language.toLowerCase(Locale.ROOT);
}
return null;
- }).filter(l -> l != null).distinct().toArray(n -> new String[n]);
+ }).toArray(n -> new String[n]));
}
String getSupportedUploadedFiles();
diff --git a/src/main/resources/fess_config.properties b/src/main/resources/fess_config.properties
index 233e5c918ae293971597bbd98cade0b41b323ebb..74049c6056b3e67c8ef9f8c9faf1db98ae9b05cc 100644
--- a/src/main/resources/fess_config.properties
+++ b/src/main/resources/fess_config.properties
@@ -75,7 +75,7 @@ supported.uploaded.js.extentions=js
supported.uploaded.css.extentions=css
supported.uploaded.media.extentions=jpg,jpeg,gif,png,swf
supported.uploaded.files=license.properties
-supported.languages=ar,bg,ca,da,de,el,en,es,eu,fa,fi,fr,ga,gl,hi,hu,hy,id,it,ja,lv,ko,nl,no,pt,ro,ru,sv,th,tr,zh_CN,zh_TW,zh
+supported.languages=ar,bg,bn,ca,ckb_IQ,cs,da,de,el,en,en_IE,es,et,eu,fa,fi,fr,gl,gu,he,hi,hr,hu,hy,id,it,ja,ko,lt,lv,mk,ml,nl,no,pa,pl,pt,pt_BR,ro,ru,si,sq,sv,ta,te,th,tl,tr,uk,ur,vi,zh_CN,zh_TW,zh
api.access.token.length=60
api.access.token.required=false
api.admin.access.permissions=Radmin-api
@@ -246,12 +246,15 @@ ar=ar\n\
bg=bg\n\
bn=bn\n\
ca=ca\n\
+ckb-iq=ckb-iq\n\
+ckb_IQ=ckb-iq\n\
cs=cs\n\
da=da\n\
de=de\n\
el=el\n\
en=en\n\
en-ie=en-ie\n\
+en_IE=en-ie\n\
es=es\n\
et=et\n\
eu=eu\n\
@@ -279,6 +282,7 @@ pa=pa\n\
pl=pl\n\
pt=pt\n\
pt-br=pt-br\n\
+pt_BR=pt-br\n\
ro=ro\n\
ru=ru\n\
si=si\n\
@@ -292,9 +296,11 @@ tr=tr\n\
uk=uk\n\
ur=ur\n\
vi=vi\n\
-zh=zh-cn\n\
zh-cn=zh-cn\n\
+zh_CN=zh-cn\n\
zh-tw=zh-tw\n\
+zh_TW=zh-tw\n\
+zh=zh\n\
# boost
query.boost.title=0.2
diff --git a/src/test/java/org/codelibs/fess/mylasta/direction/FessPropTest.java b/src/test/java/org/codelibs/fess/mylasta/direction/FessPropTest.java
index 0c247904d9d3da0582cdf6e26d183edf3abf3b9c..bfd3d4b08fc87de3ff732ca3cc5d70f603f544d5 100644
--- a/src/test/java/org/codelibs/fess/mylasta/direction/FessPropTest.java
+++ b/src/test/java/org/codelibs/fess/mylasta/direction/FessPropTest.java
@@ -17,6 +17,7 @@ package org.codelibs.fess.mylasta.direction;
import java.io.File;
import java.io.IOException;
+import java.util.Arrays;
import java.util.HashMap;
import org.codelibs.core.io.FileUtil;
@@ -119,4 +120,31 @@ public class FessPropTest extends UnitFessTestCase {
assertEquals(12288, spaceChars[1]);
}
+ public void test_normalizeQueryLanguages() {
+ FessProp.propMap.clear();
+ FessConfig fessConfig = new FessConfig.SimpleImpl() {
+ @Override
+ public String getQueryLanguageMapping() {
+ return "ja=ja\nzh_cn=zh-cn\nzh_TW=zh-tw\nzh=zh-cn";
+ }
+ };
+
+ assertArrays(new String[] {}, fessConfig.normalizeQueryLanguages(new String[] {}));
+ assertArrays(new String[] {}, fessConfig.normalizeQueryLanguages(new String[] { "unknown" }));
+ assertArrays(new String[] { "ja" }, fessConfig.normalizeQueryLanguages(new String[] { "ja" }));
+ assertArrays(new String[] { "ja" }, fessConfig.normalizeQueryLanguages(new String[] { "ja", "ja" }));
+ assertArrays(new String[] { "ja" }, fessConfig.normalizeQueryLanguages(new String[] { "ja-jp" }));
+ assertArrays(new String[] { "ja" }, fessConfig.normalizeQueryLanguages(new String[] { "ja_JP" }));
+ assertArrays(new String[] { "ja", "zh-cn" }, fessConfig.normalizeQueryLanguages(new String[] { "ja", "zh" }));
+ assertArrays(new String[] { "ja", "zh-cn" }, fessConfig.normalizeQueryLanguages(new String[] { "ja", "zh_CN" }));
+ assertArrays(new String[] { "ja", "zh-cn" }, fessConfig.normalizeQueryLanguages(new String[] { "ja", "zh-cn" }));
+ assertArrays(new String[] { "zh-cn" }, fessConfig.normalizeQueryLanguages(new String[] { "zh", "zh-cn" }));
+ assertArrays(new String[] { "zh-tw" }, fessConfig.normalizeQueryLanguages(new String[] { "zh_TW" }));
+ }
+
+ private void assertArrays(final String[] expected, final String[] actual) {
+ Arrays.sort(expected);
+ Arrays.sort(actual);
+ assertEquals(String.join(",", expected), String.join(",", actual));
+ }
}