From 5193d958cbf267d13b68b18f9223c6ed7e6abbde Mon Sep 17 00:00:00 2001
From: Shinsuke Sugaya <shinsuke@yahoo.co.jp>
Date: Thu, 22 Oct 2015 22:04:25 +0900
Subject: [PATCH] improve log handling
---
.../freegen/elasticsearch/AbstractBehavior.vm | 21 +-
.../fess/app/service/SearchService.java | 38 +++-
.../fess/app/web/search/SearchAction.java | 2 +-
.../fess/entity/SearchRenderData.java | 10 +
.../fess/es/bsbhv/AbstractBehavior.java | 21 +-
.../codelibs/fess/helper/SearchLogHelper.java | 211 ++++++++++++++++-
.../fess/helper/impl/SearchLogHelperImpl.java | 214 ------------------
src/main/resources/fess.xml | 2 +-
src/main/resources/fess_indices/fess_log.json | 2 +-
9 files changed, 266 insertions(+), 255 deletions(-)
delete mode 100644 src/main/java/org/codelibs/fess/helper/impl/SearchLogHelperImpl.java
diff --git a/dbflute_fess/freegen/elasticsearch/AbstractBehavior.vm b/dbflute_fess/freegen/elasticsearch/AbstractBehavior.vm
index 8bf7e2f15..fe13d41dd 100644
--- a/dbflute_fess/freegen/elasticsearch/AbstractBehavior.vm
+++ b/dbflute_fess/freegen/elasticsearch/AbstractBehavior.vm
@@ -337,19 +337,24 @@ public abstract class AbstractBehavior<ENTITY extends Entity, CB extends Conditi
bulkBuilder.add((DeleteRequestBuilder) builder);
}
}
- RequestOptionCall<BulkRequestBuilder> builderCall = bulkList.getCall();
+ final RequestOptionCall<BulkRequestBuilder> builderCall = bulkList.getCall();
if (builderCall != null) {
builderCall.callback(bulkBuilder);
}
- BulkResponse response = bulkBuilder.execute().actionGet();
- List<Integer> resultList = new ArrayList<>();
- for (BulkItemResponse itemResponse : response.getItems()) {
- resultList.add(itemResponse.isFailed() ? 0 : 1);
+ final BulkResponse response = bulkBuilder.execute().actionGet();
+ final BulkItemResponse[] itemResponses = response.getItems();
+ if (itemResponses.length != entityList.size()) {
+ throw new IllegalStateException("Invalid response size: " + itemResponses.length + " != " + entityList.size());
}
- int[] results = new int[resultList.size()];
- for (int i = 0; i < resultList.size(); i++) {
- results[i] = resultList.get(i);
+ final int[] results = new int[itemResponses.length];
+ for (int i = 0; i < itemResponses.length; i++) {
+ final BulkItemResponse itemResponse = itemResponses[i];
+ final Entity entity = entityList.get(i);
+ if (entity instanceof AbstractEntity) {
+ ((AbstractEntity) entity).asDocMeta().id(itemResponse.getId());
+ }
+ results[i] = itemResponse.isFailed() ? 0 : 1;
}
return results;
}
diff --git a/src/main/java/org/codelibs/fess/app/service/SearchService.java b/src/main/java/org/codelibs/fess/app/service/SearchService.java
index c8a409c95..306907b6b 100644
--- a/src/main/java/org/codelibs/fess/app/service/SearchService.java
+++ b/src/main/java/org/codelibs/fess/app/service/SearchService.java
@@ -7,6 +7,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.ExecutionException;
import java.util.function.Consumer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -22,6 +23,7 @@ import org.codelibs.fess.entity.SearchRenderData;
import org.codelibs.fess.entity.SearchRequestParams;
import org.codelibs.fess.es.client.FessEsClient;
import org.codelibs.fess.es.client.FessEsClient.SearchConditionBuilder;
+import org.codelibs.fess.es.client.FessEsClientException;
import org.codelibs.fess.es.exentity.SearchLog;
import org.codelibs.fess.es.exentity.UserInfo;
import org.codelibs.fess.helper.FieldHelper;
@@ -32,16 +34,15 @@ import org.codelibs.fess.helper.UserInfoHelper;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.QueryResponseList;
import org.dbflute.optional.OptionalEntity;
+import org.elasticsearch.action.bulk.BulkRequestBuilder;
+import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.index.query.QueryBuilders;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
public class SearchService {
// ===================================================================================
// Constant
//
- private static final Logger logger = LoggerFactory.getLogger(SearchService.class);
protected static final Pattern FIELD_EXTRACTION_PATTERN = Pattern.compile("^([a-zA-Z0-9_]+):.*");
@@ -71,6 +72,8 @@ public class SearchService {
// ==============
public void search(final HttpServletRequest request, final SearchRequestParams params, final SearchRenderData data) {
+ final long requestedTime = systemHelper.getCurrentTimeAsLong();
+
final long startTime = System.currentTimeMillis();
final boolean searchLogSupport =
Constants.TRUE.equals(crawlerProperties.getProperty(Constants.SEARCH_LOG_PROPERTY, Constants.TRUE));
@@ -136,7 +139,7 @@ public class SearchService {
// search log
if (searchLogSupport) {
- storeSearchLog(request, query, pageStart, pageSize, queryResponseList);
+ storeSearchLog(request, requestedTime, query, pageStart, pageSize, queryResponseList);
}
queryResponseList.setExecTime(System.currentTimeMillis() - startTime);
@@ -163,11 +166,11 @@ public class SearchService {
data.setPartialResults(queryResponseList.isPartialResults());
data.setQueryTime(queryResponseList.getQueryTime());
data.setSearchQuery(query);
+ data.setRequestedTime(requestedTime);
}
- protected void storeSearchLog(final HttpServletRequest request, final String query, final int pageStart, final int pageSize,
- final QueryResponseList queryResponseList) {
- final long now = systemHelper.getCurrentTimeAsLong();
+ protected void storeSearchLog(final HttpServletRequest request, final long requestedTime, final String query, final int pageStart,
+ final int pageSize, final QueryResponseList queryResponseList) {
final SearchLogHelper searchLogHelper = ComponentUtil.getSearchLogHelper();
final SearchLog searchLog = new SearchLog();
@@ -178,8 +181,8 @@ public class SearchService {
if (StringUtil.isNotBlank(userCode)) {
final UserInfo userInfo = new UserInfo();
userInfo.setCode(userCode);
- userInfo.setCreatedTime(now);
- userInfo.setUpdatedTime(now);
+ userInfo.setCreatedTime(requestedTime);
+ userInfo.setUpdatedTime(requestedTime);
searchLog.setUserInfo(OptionalEntity.of(userInfo));
}
}
@@ -188,7 +191,7 @@ public class SearchService {
searchLog.setResponseTime(Integer.valueOf((int) queryResponseList.getExecTime()));
searchLog.setSearchWord(StringUtils.abbreviate(query, 1000));
searchLog.setSearchQuery(StringUtils.abbreviate(queryResponseList.getSearchQuery(), 1000));
- searchLog.setRequestedTime(now);
+ searchLog.setRequestedTime(requestedTime);
searchLog.setQueryOffset(pageStart);
searchLog.setQueryPageSize(pageSize);
@@ -321,4 +324,19 @@ public class SearchService {
public boolean update(final String id, final String field, final Object value) {
return fessEsClient.update(fieldHelper.docIndex, fieldHelper.docType, id, field, value);
}
+
+ public boolean bulkUpdate(Consumer<BulkRequestBuilder> consumer) {
+ BulkRequestBuilder builder = fessEsClient.prepareBulk();
+ consumer.accept(builder);
+ try {
+ BulkResponse response = builder.execute().get();
+ if (response.hasFailures()) {
+ throw new FessEsClientException(response.buildFailureMessage());
+ } else {
+ return true;
+ }
+ } catch (InterruptedException | ExecutionException e) {
+ throw new FessEsClientException("Failed to update bulk data.", e);
+ }
+ }
}
diff --git a/src/main/java/org/codelibs/fess/app/web/search/SearchAction.java b/src/main/java/org/codelibs/fess/app/web/search/SearchAction.java
index d6c967578..01ea48b2f 100644
--- a/src/main/java/org/codelibs/fess/app/web/search/SearchAction.java
+++ b/src/main/java/org/codelibs/fess/app/web/search/SearchAction.java
@@ -134,6 +134,7 @@ public class SearchAction extends FessSearchAction {
try {
final WebRenderData renderData = new WebRenderData(data);
searchService.search(request, form, renderData);
+ form.rt = Long.toString(renderData.getRequestedTime());
// favorite or screenshot
if (favoriteSupport || screenShotManager != null) {
final String searchQuery = renderData.getSearchQuery();
@@ -157,7 +158,6 @@ public class SearchAction extends FessSearchAction {
messages.addErrorsResultSizeExceeded(GLOBAL);
}, () -> asHtml(path_ErrorJsp));
}
- form.rt = Long.toString(systemHelper.getCurrentTimeAsLong());
data.register("displayQuery", getDisplayQuery(form, labelTypeHelper.getLabelTypeItemList()));
data.register("pagingQuery", getPagingQuery(form));
});
diff --git a/src/main/java/org/codelibs/fess/entity/SearchRenderData.java b/src/main/java/org/codelibs/fess/entity/SearchRenderData.java
index addb3b8c4..df5d22152 100644
--- a/src/main/java/org/codelibs/fess/entity/SearchRenderData.java
+++ b/src/main/java/org/codelibs/fess/entity/SearchRenderData.java
@@ -39,6 +39,8 @@ public class SearchRenderData {
private long queryTime;
+ private long requestedTime;
+
public void setDocumentItems(final List<Map<String, Object>> documentItems) {
this.documentItems = documentItems;
}
@@ -103,6 +105,10 @@ public class SearchRenderData {
this.searchQuery = searchQuery;
}
+ public void setRequestedTime(final long requestedTime) {
+ this.requestedTime = requestedTime;
+ }
+
public List<Map<String, Object>> getDocumentItems() {
return documentItems;
}
@@ -167,4 +173,8 @@ public class SearchRenderData {
return queryTime;
}
+ public long getRequestedTime() {
+ return requestedTime;
+ }
+
}
diff --git a/src/main/java/org/codelibs/fess/es/bsbhv/AbstractBehavior.java b/src/main/java/org/codelibs/fess/es/bsbhv/AbstractBehavior.java
index a6a517131..fdc8b985e 100644
--- a/src/main/java/org/codelibs/fess/es/bsbhv/AbstractBehavior.java
+++ b/src/main/java/org/codelibs/fess/es/bsbhv/AbstractBehavior.java
@@ -339,19 +339,24 @@ public abstract class AbstractBehavior<ENTITY extends Entity, CB extends Conditi
bulkBuilder.add((DeleteRequestBuilder) builder);
}
}
- RequestOptionCall<BulkRequestBuilder> builderCall = bulkList.getCall();
+ final RequestOptionCall<BulkRequestBuilder> builderCall = bulkList.getCall();
if (builderCall != null) {
builderCall.callback(bulkBuilder);
}
- BulkResponse response = bulkBuilder.execute().actionGet();
- List<Integer> resultList = new ArrayList<>();
- for (BulkItemResponse itemResponse : response.getItems()) {
- resultList.add(itemResponse.isFailed() ? 0 : 1);
+ final BulkResponse response = bulkBuilder.execute().actionGet();
+ final BulkItemResponse[] itemResponses = response.getItems();
+ if (itemResponses.length != entityList.size()) {
+ throw new IllegalStateException("Invalid response size: " + itemResponses.length + " != " + entityList.size());
}
- int[] results = new int[resultList.size()];
- for (int i = 0; i < resultList.size(); i++) {
- results[i] = resultList.get(i);
+ final int[] results = new int[itemResponses.length];
+ for (int i = 0; i < itemResponses.length; i++) {
+ final BulkItemResponse itemResponse = itemResponses[i];
+ final Entity entity = entityList.get(i);
+ if (entity instanceof AbstractEntity) {
+ ((AbstractEntity) entity).asDocMeta().id(itemResponse.getId());
+ }
+ results[i] = itemResponse.isFailed() ? 0 : 1;
}
return results;
}
diff --git a/src/main/java/org/codelibs/fess/helper/SearchLogHelper.java b/src/main/java/org/codelibs/fess/helper/SearchLogHelper.java
index c08a90747..e80358cc3 100644
--- a/src/main/java/org/codelibs/fess/helper/SearchLogHelper.java
+++ b/src/main/java/org/codelibs/fess/helper/SearchLogHelper.java
@@ -16,23 +16,48 @@
package org.codelibs.fess.helper;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.stream.Stream;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
+import org.codelibs.core.beans.util.BeanUtil;
import org.codelibs.core.collection.LruHashMap;
+import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.DynamicProperties;
+import org.codelibs.fess.Constants;
+import org.codelibs.fess.app.service.SearchService;
+import org.codelibs.fess.es.exbhv.ClickLogBhv;
+import org.codelibs.fess.es.exbhv.FavoriteLogBhv;
+import org.codelibs.fess.es.exbhv.SearchFieldLogBhv;
+import org.codelibs.fess.es.exbhv.SearchLogBhv;
+import org.codelibs.fess.es.exbhv.UserInfoBhv;
import org.codelibs.fess.es.exentity.ClickLog;
+import org.codelibs.fess.es.exentity.SearchFieldLog;
import org.codelibs.fess.es.exentity.SearchLog;
+import org.codelibs.fess.es.exentity.UserInfo;
+import org.codelibs.fess.util.ComponentUtil;
+import org.codelibs.fess.util.DocumentUtil;
+import org.elasticsearch.action.update.UpdateRequest;
+import org.lastaflute.di.core.SingletonLaContainer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-public abstract class SearchLogHelper {
+public class SearchLogHelper {
+ private static final Logger logger = LoggerFactory.getLogger(SearchLogHelper.class);
@Resource
protected DynamicProperties crawlerProperties;
+ @Resource
+ protected FieldHelper fieldHelper;
+
public long userCheckInterval = 5 * 60 * 1000;// 5 min
public int userInfoCacheSize = 1000;
@@ -48,8 +73,6 @@ public abstract class SearchLogHelper {
userInfoCache = new LruHashMap<String, Long>(userInfoCacheSize);
}
- public abstract void updateUserInfo(final String userCode);
-
public void addSearchLog(final SearchLog searchLog) {
searchLogQueue.add(searchLog);
}
@@ -73,21 +96,185 @@ public abstract class SearchLogHelper {
}
public int getClickCount(final String url) {
- // TODO
- return 0;
+ final ClickLogBhv clickLogBhv = ComponentUtil.getComponent(ClickLogBhv.class);
+ return clickLogBhv.selectCount(cb -> {
+ cb.query().setUrl_Equal(url);
+ });
}
public long getFavoriteCount(final String url) {
- // TODO
- return 0;
+ final FavoriteLogBhv favoriteLogBhv = ComponentUtil.getComponent(FavoriteLogBhv.class);
+ return favoriteLogBhv.selectCount(cb -> {
+ cb.query().setUrl_Equal(url);
+ });
+ }
+
+ public void updateUserInfo(final String userCode) {
+ final long current = System.currentTimeMillis();
+ final Long time = userInfoCache.get(userCode);
+ if (time == null || current - time.longValue() > userCheckInterval) {
+
+ final UserInfoBhv userInfoBhv = ComponentUtil.getComponent(UserInfoBhv.class);
+
+ final List<UserInfo> list = userInfoBhv.selectList(cb -> {
+ cb.query().setCode_Equal(userCode);
+ cb.query().addOrderBy_UpdatedTime_Desc();
+ });
+
+ final UserInfo userInfo;
+ final long now = ComponentUtil.getSystemHelper().getCurrentTimeAsLong();
+ if (list.isEmpty()) {
+ userInfo = new UserInfo();
+ userInfo.setCode(userCode);
+ userInfo.setCreatedTime(now);
+ } else {
+ userInfo = list.get(0);
+ }
+ userInfo.setUpdatedTime(now);
+ new Thread(() -> {
+ userInfoBhv.insertOrUpdate(userInfo);
+ }).start();
+ userInfoCache.put(userCode, current);
+ }
+ }
+
+ protected void processSearchLogQueue(final Queue<SearchLog> queue) {
+ final String value = crawlerProperties.getProperty(Constants.PURGE_BY_BOTS_PROPERTY, StringUtil.EMPTY);
+ String[] botNames;
+ if (StringUtil.isBlank(value)) {
+ botNames = StringUtil.EMPTY_STRINGS;
+ } else {
+ botNames = value.split(",");
+ }
+
+ final List<SearchLog> searchLogList = new ArrayList<>();
+ final Map<String, UserInfo> userInfoMap = new HashMap<>();
+ queue.stream().forEach(searchLog -> {
+ final String userAgent = searchLog.getUserAgent();
+ boolean isBot = userAgent != null && Stream.of(botNames).anyMatch(botName -> userAgent.indexOf(botName) >= 0);
+ if (!isBot) {
+ searchLog.getUserInfo().ifPresent(userInfo -> {
+ final String code = userInfo.getCode();
+ final UserInfo oldUserInfo = userInfoMap.get(code);
+ if (oldUserInfo != null) {
+ userInfo.setCreatedTime(oldUserInfo.getCreatedTime());
+ }
+ userInfoMap.put(code, userInfo);
+ });
+ searchLogList.add(searchLog);
+ }
+ });
+
+ if (!userInfoMap.isEmpty()) {
+ final List<UserInfo> insertList = new ArrayList<>(userInfoMap.values());
+ final List<UserInfo> updateList = new ArrayList<>();
+ final UserInfoBhv userInfoBhv = SingletonLaContainer.getComponent(UserInfoBhv.class);
+ final List<UserInfo> list = userInfoBhv.selectList(cb -> {
+ cb.query().setCode_InScope(userInfoMap.keySet());
+ });
+ for (final UserInfo userInfo : list) {
+ final String code = userInfo.getCode();
+ final UserInfo entity = userInfoMap.get(code);
+ BeanUtil.copyBeanToBean(userInfo, entity, option -> option.include("id", "createdTime"));
+ updateList.add(entity);
+ insertList.remove(entity);
+ }
+ userInfoBhv.batchInsert(insertList);
+ userInfoBhv.batchUpdate(updateList);
+ searchLogList.stream().forEach(searchLog -> {
+ searchLog.getUserInfo().ifPresent(userInfo -> {
+ final UserInfo entity = userInfoMap.get(userInfo.getCode());
+ searchLog.setUserInfoId(entity.getId());
+ });
+ });
+ }
+
+ if (!searchLogList.isEmpty()) {
+ storeSearchLogList(searchLogList);
+ }
}
- protected abstract void processSearchLogQueue(Queue<SearchLog> queue);
+ private void storeSearchLogList(final List<SearchLog> searchLogList) {
+ final SearchLogBhv searchLogBhv = ComponentUtil.getComponent(SearchLogBhv.class);
+ final SearchFieldLogBhv searchFieldLogBhv = ComponentUtil.getComponent(SearchFieldLogBhv.class);
+ searchLogBhv.batchUpdate(searchLogList);
+ searchLogList.stream().forEach(searchLog -> {
+ final List<SearchFieldLog> fieldLogList = new ArrayList<>();
+ searchLog.getSearchFieldLogList().stream().forEach(fieldLog -> {
+ fieldLog.setSearchLogId(searchLog.getId());
+ fieldLogList.add(fieldLog);
+ });
+ searchFieldLogBhv.batchInsert(fieldLogList);
+ });
+ }
- protected abstract void processClickLogQueue(Queue<ClickLog> queue);
+ protected void processClickLogQueue(final Queue<ClickLog> queue) {
+ final Map<String, Long> clickCountMap = new HashMap<>();
+ final List<ClickLog> clickLogList = new ArrayList<>();
+ for (final ClickLog clickLog : queue) {
+ try {
+ final SearchLogBhv searchLogBhv = SingletonLaContainer.getComponent(SearchLogBhv.class);
+ searchLogBhv.selectEntity(cb -> {
+ cb.query().setRequestedTime_Equal(clickLog.getQueryRequestedTime());
+ cb.query().setUserSessionId_Equal(clickLog.getUserSessionId());
+ }).ifPresent(entity -> {
+ clickLog.setSearchLogId(entity.getId());
+ clickLogList.add(clickLog);
+ }).orElse(() -> {
+ logger.warn("Not Found for SearchLog: " + clickLog);
+ });
- public boolean addfavoriteLog(final String userCode, final String favoriteUrl) {
- // TODO Auto-generated method stub
- return false;
+ final String docId = clickLog.getDocId();
+ Long countObj = clickCountMap.get(docId);
+ if (countObj == null) {
+ final long clickCount = clickLog.getClickCount();
+ countObj = Long.valueOf(clickCount + 1);
+ } else {
+ countObj = countObj.longValue() + 1;
+ }
+ clickCountMap.put(docId, countObj);
+ } catch (final Exception e) {
+ logger.warn("Failed to process: " + clickLog, e);
+ }
+ }
+ if (!clickLogList.isEmpty()) {
+ try {
+ final ClickLogBhv clickLogBhv = SingletonLaContainer.getComponent(ClickLogBhv.class);
+ clickLogBhv.batchInsert(clickLogList);
+ } catch (final Exception e) {
+ logger.warn("Failed to insert: " + clickLogList, e);
+ }
+ }
+
+ if (!clickCountMap.isEmpty()) {
+ final SearchService searchService = ComponentUtil.getComponent(SearchService.class);
+ try {
+ final Map<String, String> docIdMap = new HashMap<>();
+ searchService
+ .getDocumentListByDocIds(clickCountMap.keySet().toArray(new String[clickCountMap.size()]),
+ new String[] { fieldHelper.idField, fieldHelper.docIdField })
+ .stream()
+ .forEach(
+ doc -> {
+ docIdMap.put(DocumentUtil.getValue(doc, fieldHelper.docIdField, String.class),
+ DocumentUtil.getValue(doc, fieldHelper.idField, String.class));
+ });
+ searchService.bulkUpdate(builder -> {
+ clickCountMap
+ .entrySet()
+ .stream()
+ .forEach(
+ entry -> {
+ String id = docIdMap.get(entry.getKey());
+ if (id != null) {
+ builder.add(new UpdateRequest(fieldHelper.docIndex, fieldHelper.docType, id).doc(
+ fieldHelper.clickCountField, entry.getValue()));
+ }
+ });
+ });
+ } catch (final Exception e) {
+ logger.warn("Failed to update clickCounts", e);
+ }
+ }
}
}
diff --git a/src/main/java/org/codelibs/fess/helper/impl/SearchLogHelperImpl.java b/src/main/java/org/codelibs/fess/helper/impl/SearchLogHelperImpl.java
deleted file mode 100644
index 9e525ee15..000000000
--- a/src/main/java/org/codelibs/fess/helper/impl/SearchLogHelperImpl.java
+++ /dev/null
@@ -1,214 +0,0 @@
-/*
- * Copyright 2009-2015 the CodeLibs Project and the Others.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
- * either express or implied. See the License for the specific language
- * governing permissions and limitations under the License.
- */
-
-package org.codelibs.fess.helper.impl;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Queue;
-
-import org.codelibs.core.beans.util.BeanUtil;
-import org.codelibs.core.lang.StringUtil;
-import org.codelibs.fess.Constants;
-import org.codelibs.fess.es.client.FessEsClient;
-import org.codelibs.fess.es.exbhv.ClickLogBhv;
-import org.codelibs.fess.es.exbhv.SearchLogBhv;
-import org.codelibs.fess.es.exbhv.UserInfoBhv;
-import org.codelibs.fess.es.exentity.ClickLog;
-import org.codelibs.fess.es.exentity.SearchLog;
-import org.codelibs.fess.es.exentity.UserInfo;
-import org.codelibs.fess.helper.FieldHelper;
-import org.codelibs.fess.helper.SearchLogHelper;
-import org.codelibs.fess.util.ComponentUtil;
-import org.lastaflute.di.core.SingletonLaContainer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class SearchLogHelperImpl extends SearchLogHelper {
- private static final Logger logger = LoggerFactory // NOPMD
- .getLogger(SearchLogHelperImpl.class);
-
- @Override
- public void updateUserInfo(final String userCode) {
- final long current = System.currentTimeMillis();
- final Long time = userInfoCache.get(userCode);
- if (time == null || current - time.longValue() > userCheckInterval) {
-
- final UserInfoBhv userInfoBhv = ComponentUtil.getComponent(UserInfoBhv.class);
-
- final List<UserInfo> list = userInfoBhv.selectList(cb -> {
- cb.query().setCode_Equal(userCode);
- cb.query().addOrderBy_UpdatedTime_Desc();
- });
-
- final UserInfo userInfo;
- final long now = ComponentUtil.getSystemHelper().getCurrentTimeAsLong();
- if (list.isEmpty()) {
- userInfo = new UserInfo();
- userInfo.setCode(userCode);
- userInfo.setCreatedTime(now);
- } else {
- userInfo = list.get(0);
- }
- userInfo.setUpdatedTime(now);
- new Thread(() -> {
- userInfoBhv.insertOrUpdate(userInfo);
- }).start();
- userInfoCache.put(userCode, current);
- }
- }
-
- @Override
- protected void processSearchLogQueue(final Queue<SearchLog> queue) {
- final List<SearchLog> searchLogList = new ArrayList<SearchLog>();
- final String value = crawlerProperties.getProperty(Constants.PURGE_BY_BOTS_PROPERTY, StringUtil.EMPTY);
- String[] botNames;
- if (StringUtil.isBlank(value)) {
- botNames = StringUtil.EMPTY_STRINGS;
- } else {
- botNames = value.split(",");
- }
-
- Constants.TRUE.equals(crawlerProperties.getProperty(Constants.SUGGEST_SEARCH_LOG_PROPERTY, Constants.TRUE));
- final String dayForCleanupStr = crawlerProperties.getProperty(Constants.PURGE_SUGGEST_SEARCH_LOG_DAY_PROPERTY, "30");
- try {
- Integer.parseInt(dayForCleanupStr);
- } catch (final NumberFormatException e) {}
-
- final Map<String, UserInfo> userInfoMap = new HashMap<String, UserInfo>();
- for (final SearchLog searchLog : queue) {
- boolean add = true;
- for (final String botName : botNames) {
- if (searchLog.getUserAgent() != null && searchLog.getUserAgent().indexOf(botName) >= 0) {
- add = false;
- break;
- }
- }
- if (add) {
- final UserInfo userInfo = searchLog.getUserInfo().orElse(null);// TODO
- if (userInfo != null) {
- final String code = userInfo.getCode();
- final UserInfo oldUserInfo = userInfoMap.get(code);
- if (oldUserInfo != null) {
- userInfo.setCreatedTime(oldUserInfo.getCreatedTime());
- }
- userInfoMap.put(code, userInfo);
- }
- searchLogList.add(searchLog);
-
- // TODO
- // if (suggestAvailable && searchLog.getHitCount() > 0) {
- // final List<SearchFieldLog> searchFieldLogList = searchLog.getSearchFieldLogList();
- // for (final SearchFieldLog searchFieldLog : searchFieldLogList) {
- // if ("solrQuery".equals(searchFieldLog.getName())) {
- // suggestService.addSolrParams(searchFieldLog.getValue(), dayForCleanup);
- // addedSuggest = true;
- // }
- // }
- // }
- }
- }
- // if (addedSuggest) {
- // suggestService.commit();
- // }
-
- if (!userInfoMap.isEmpty()) {
- final List<UserInfo> insertList = new ArrayList<UserInfo>(userInfoMap.values());
- final List<UserInfo> updateList = new ArrayList<UserInfo>();
- final UserInfoBhv userInfoBhv = SingletonLaContainer.getComponent(UserInfoBhv.class);
- final List<UserInfo> list = userInfoBhv.selectList(cb -> {
- cb.query().setCode_InScope(userInfoMap.keySet());
- });
- for (final UserInfo userInfo : list) {
- final String code = userInfo.getCode();
- final UserInfo entity = userInfoMap.get(code);
- BeanUtil.copyBeanToBean(userInfo, entity, option -> option.include("id", "createdTime"));
- updateList.add(entity);
- insertList.remove(entity);
- }
- userInfoBhv.batchInsert(insertList);
- userInfoBhv.batchUpdate(updateList);
- for (final SearchLog searchLog : searchLogList) {
- final UserInfo userInfo = searchLog.getUserInfo().orElse(null);//TODO
- if (userInfo != null) {
- final UserInfo entity = userInfoMap.get(userInfo.getCode());
- searchLog.setUserInfoId(entity.getId());
- }
- }
- }
-
- if (!searchLogList.isEmpty()) {
- final SearchLogBhv searchLogBhv = ComponentUtil.getComponent(SearchLogBhv.class);
- searchLogBhv.batchUpdate(searchLogList);
- // TODO SearchLogValue
- }
- }
-
- @Override
- protected void processClickLogQueue(final Queue<ClickLog> queue) {
- final Map<String, Long> clickCountMap = new HashMap<String, Long>();
- final List<ClickLog> clickLogList = new ArrayList<ClickLog>();
- for (final ClickLog clickLog : queue) {
- try {
- final SearchLogBhv searchLogBhv = SingletonLaContainer.getComponent(SearchLogBhv.class);
- final SearchLog entity = searchLogBhv.selectEntity(cb -> {
- cb.query().setRequestedTime_Equal(clickLog.getQueryRequestedTime());
- cb.query().setUserSessionId_Equal(clickLog.getUserSessionId());
- }).orElse(null);//TODO
- if (entity != null) {
- clickLog.setSearchLogId(entity.getId());
- clickLogList.add(clickLog);
- } else {
- logger.warn("Not Found[ClickLog]: " + clickLog);
- }
-
- final String docId = clickLog.getDocId();
- Long countObj = clickCountMap.get(docId);
- final long clickCount = clickLog.getClickCount();
- if (countObj == null) {
- countObj = Long.valueOf(clickCount);
- } else {
- countObj = Math.max(countObj.longValue(), clickCount) + 1;
- }
- clickCountMap.put(docId, countObj);
- } catch (final Exception e) {
- logger.warn("Failed to process: " + clickLog, e);
- }
- }
- if (!clickLogList.isEmpty()) {
- try {
- final ClickLogBhv clickLogBhv = SingletonLaContainer.getComponent(ClickLogBhv.class);
- clickLogBhv.batchInsert(clickLogList);
- } catch (final Exception e) {
- logger.warn("Failed to insert: " + clickLogList, e);
- }
- }
-
- final FessEsClient fessEsClient = ComponentUtil.getElasticsearchClient();
- final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
- for (final Map.Entry<String, Long> entry : clickCountMap.entrySet()) {
- try {
- // TODO buik update
- fessEsClient.update(fieldHelper.docIndex, fieldHelper.docType, entry.getKey(), fieldHelper.clickCountField,
- entry.getValue() + 1);
- } catch (final Exception e) {
- logger.warn("Failed to update a clickCount(" + entry.getValue() + ") for " + entry.getKey(), e);
- }
- }
- }
-}
diff --git a/src/main/resources/fess.xml b/src/main/resources/fess.xml
index 186e40225..0bbf083c0 100644
--- a/src/main/resources/fess.xml
+++ b/src/main/resources/fess.xml
@@ -6,7 +6,7 @@
<include path="fess_ds.xml"/>
<include path="fess_es.xml"/>
- <component name="searchLogHelper" class="org.codelibs.fess.helper.impl.SearchLogHelperImpl">
+ <component name="searchLogHelper" class="org.codelibs.fess.helper.SearchLogHelper">
<!--
<property name="userCheckInterval">5 * 60 * 1000</property>
<property name="userInfoCacheSize">1000</property>
diff --git a/src/main/resources/fess_indices/fess_log.json b/src/main/resources/fess_indices/fess_log.json
index 4000a8acb..e3657f0d6 100644
--- a/src/main/resources/fess_indices/fess_log.json
+++ b/src/main/resources/fess_indices/fess_log.json
@@ -1,7 +1,7 @@
{
"settings": {
"index": {
- "refresh_interval": "60s",
+ "refresh_interval": "1s",
"number_of_shards": 10,
"number_of_replicas": 0
}
--
GitLab