From 4ea814bf6f81dd5f02c741008f50cc788a07250b Mon Sep 17 00:00:00 2001
From: y_fujita <uu055650@CA0057.local>
Date: Tue, 15 Dec 2015 07:56:47 +0900
Subject: [PATCH] fix oom at suggest job
---
pom.xml | 2 +-
.../codelibs/fess/exec/SuggestCreator.java | 6 +++++
.../codelibs/fess/helper/SuggestHelper.java | 22 +++++++++----------
3 files changed, 18 insertions(+), 12 deletions(-)
diff --git a/pom.xml b/pom.xml
index 7f34468e3..3b74ff491 100644
--- a/pom.xml
+++ b/pom.xml
@@ -62,7 +62,7 @@
<crawler.version>1.0.0</crawler.version>
<!-- Suggest -->
- <suggest.version>2.0.1-SNAPSHOT</suggest.version>
+ <suggest.version>2.1.0-SNAPSHOT</suggest.version>
<!-- Elasticsearch -->
<elasticsearch.version>2.1.0</elasticsearch.version>
diff --git a/src/main/java/org/codelibs/fess/exec/SuggestCreator.java b/src/main/java/org/codelibs/fess/exec/SuggestCreator.java
index a33e968de..406ab98e4 100644
--- a/src/main/java/org/codelibs/fess/exec/SuggestCreator.java
+++ b/src/main/java/org/codelibs/fess/exec/SuggestCreator.java
@@ -118,6 +118,12 @@ public class SuggestCreator implements Serializable {
}
private int create() {
+ final DynamicProperties crawlerProperties = ComponentUtil.getCrawlerProperties();
+ if(!Constants.TRUE.equals(crawlerProperties.getProperty(Constants.SUGGEST_DOCUMENTS_PROPERTY, Constants.TRUE))) {
+ logger.info("Skip create suggest document.");
+ return 0;
+ }
+
logger.info("Start create suggest document.");
final AtomicInteger result = new AtomicInteger(1);
diff --git a/src/main/java/org/codelibs/fess/helper/SuggestHelper.java b/src/main/java/org/codelibs/fess/helper/SuggestHelper.java
index c9a44f2be..8236ff4ad 100644
--- a/src/main/java/org/codelibs/fess/helper/SuggestHelper.java
+++ b/src/main/java/org/codelibs/fess/helper/SuggestHelper.java
@@ -39,7 +39,6 @@ import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.suggest.Suggester;
import org.codelibs.fess.suggest.constants.FieldNames;
import org.codelibs.fess.suggest.entity.SuggestItem;
-import org.codelibs.fess.suggest.index.contents.document.DocumentReader;
import org.codelibs.fess.suggest.index.contents.document.ESSourceReader;
import org.codelibs.fess.suggest.settings.SuggestSettings;
import org.codelibs.fess.suggest.util.SuggestUtil;
@@ -69,6 +68,10 @@ public class SuggestHelper {
public String[] contentsIndexFieldNames = { "content", "title" };
+ public long updateRequestIntervalMills = 1;
+
+ public int sourceReaderScrollSize = 1;
+
private static final String TEXT_SEP = " ";
protected Suggester suggester;
@@ -138,17 +141,14 @@ public class SuggestHelper {
}
final FessConfig fessConfig = ComponentUtil.getFessConfig();
- final DocumentReader reader =
+ final ESSourceReader reader =
new ESSourceReader(fessEsClient, suggester.settings(), fessConfig.getIndexDocumentIndex(),
fessConfig.getIndexDocumentType());
-
- suggester.indexer().indexFromDocument(reader, 2, 100).then(response -> {
+ reader.setScrollSize(sourceReaderScrollSize);
+ suggester.indexer().indexFromDocument(reader, 2, updateRequestIntervalMills).then(response -> {
suggester.refresh();
-
- //TODO delete old doc
-
- success.accept(true);
- }).error(t -> error.accept(t));
+ success.accept(true);
+ }).error(t -> error.accept(t));
}
public void purgeDocumentSuggest(final LocalDateTime time) {
@@ -214,11 +214,11 @@ public class SuggestHelper {
public void addElevateWord(final String word, final String reading, final String[] tags, final String roles, final float boost,
final boolean commit) {
- final List<String> labelList = new ArrayList<String>();
+ final List<String> labelList = new ArrayList<>();
for (final String label : tags) {
labelList.add(label);
}
- final List<String> roleList = new ArrayList<String>();
+ final List<String> roleList = new ArrayList<>();
if (StringUtil.isNotBlank(roles)) {
final String[] array = roles.trim().split(",");
for (final String role : array) {
--
GitLab