diff --git a/src/main/java/org/codelibs/fess/app/web/admin/storage/AdminStorageAction.java b/src/main/java/org/codelibs/fess/app/web/admin/storage/AdminStorageAction.java index 22a1441b713ac625dcd79f5f18115dda16475961..5697d88cce7943ba12dfb4058627e9cbd93a4922 100644 --- a/src/main/java/org/codelibs/fess/app/web/admin/storage/AdminStorageAction.java +++ b/src/main/java/org/codelibs/fess/app/web/admin/storage/AdminStorageAction.java @@ -34,14 +34,17 @@ import org.codelibs.fess.exception.StorageException; import org.codelibs.fess.mylasta.direction.FessConfig; import org.codelibs.fess.util.ComponentUtil; import org.codelibs.fess.util.RenderDataUtil; +import org.dbflute.optional.OptionalThing; import org.lastaflute.web.Execute; import org.lastaflute.web.response.ActionResponse; import org.lastaflute.web.response.HtmlResponse; +import org.lastaflute.web.ruts.multipart.MultipartFormFile; import org.lastaflute.web.ruts.process.ActionRuntime; import io.minio.MinioClient; import io.minio.Result; import io.minio.messages.Item; +import org.lastaflute.web.servlet.request.stream.WrittenStreamOut; /** * @author shinsuke @@ -67,39 +70,36 @@ public class AdminStorageAction extends FessAdminAction { // public HtmlResponse create() { // } + @Execute + public ActionResponse list(final OptionalThing<String> id) { + saveToken(); + if (id.isPresent() && id.get() != null) { + return asListHtml(decodePath(id.get())); + } + return redirect(getClass()); + } + @Execute public HtmlResponse upload(final ItemForm form) { validate(form, messages -> {}, () -> asListHtml(form.path)); if (form.uploadFile == null) { throwValidationError(messages -> messages.addErrorsStorageNoUploadFile(GLOBAL), () -> asListHtml(form.path)); } - logger.debug("form.path = {}", form.path); verifyToken(() -> asListHtml(form.path)); - final String objectName = getObjectName(form.path, form.uploadFile.getFileName()); - try (final InputStream in = form.uploadFile.getInputStream()) { - final MinioClient minioClient = createClient(fessConfig); - minioClient.putObject(fessConfig.getStorageBucket(), objectName, in, (long) form.uploadFile.getFileSize(), null, null, - "application/octet-stream"); - } catch (final Exception e) { + try { + uploadObject(getObjectName(form.path, form.uploadFile.getFileName()), form.uploadFile); + } catch (final StorageException e) { if (logger.isDebugEnabled()) { - logger.debug("Failed to upload {}", objectName, e); + logger.debug("Failed to upload {}", form.uploadFile.getFileName(), e); } - throwValidationError(messages -> messages.addErrorsStorageFileUploadFailure(GLOBAL, e.getLocalizedMessage()), - () -> asListHtml(form.path)); + throwValidationError(messages -> messages.addErrorsStorageFileUploadFailure(GLOBAL, form.uploadFile.getFileName()), + () -> asListHtml(encodeId(form.path))); + } saveInfo(messages -> messages.addSuccessUploadFileToStorage(GLOBAL, form.uploadFile.getFileName())); - if (StringUtil.isEmpty(form.path)) { - return redirect(getClass()); - } return redirectWith(getClass(), moreUrl("list/" + encodeId(form.path))); } - @Execute - public ActionResponse list(final String id) { - saveToken(); - return asListHtml(decodePath(id)); - } - @Execute public ActionResponse download(final String id) { final String[] values = decodeId(id); @@ -108,13 +108,13 @@ public class AdminStorageAction extends FessAdminAction { } return asStream(values[1]).contentTypeOctetStream().stream( out -> { - try (InputStream in = createClient(fessConfig).getObject(fessConfig.getStorageBucket(), values[0] + values[1])) { - out.write(in); - } catch (final Exception e) { + try { + downloadObject(getObjectName(values[0], values[1]), out); + } catch (final StorageException e) { if (logger.isDebugEnabled()) { - logger.debug("Failed to access {}", fessConfig.getStorageEndpoint(), e); + logger.debug("Failed to download {}", values[1], e); } - throwValidationError(messages -> messages.addErrorsStorageAccessError(GLOBAL, e.getLocalizedMessage()), + throwValidationError(messages -> messages.addErrorsStorageFileDownloadFailure(GLOBAL, values[1]), () -> asListHtml(encodeId(values[0]))); } }); @@ -126,20 +126,14 @@ public class AdminStorageAction extends FessAdminAction { if (StringUtil.isEmpty(values[1])) { throwValidationError(messages -> messages.addErrorsStorageFileNotFound(GLOBAL), () -> asListHtml(encodeId(values[0]))); } - logger.debug("values[0] = {}, values[1] = {}", values[0], values[1]); final String objectName = getObjectName(values[0], values[1]); try { - final MinioClient minioClient = createClient(fessConfig); - minioClient.removeObject(fessConfig.getStorageBucket(), objectName); - } catch (final Exception e) { + deleteObject(objectName); + } catch (final StorageException e) { logger.debug("Failed to delete {}", values[1], e); - throwValidationError(messages -> messages.addErrorsFailedToDeleteFile(GLOBAL, e.getLocalizedMessage()), - () -> asListHtml(encodeId(values[0]))); + throwValidationError(messages -> messages.addErrorsFailedToDeleteFile(GLOBAL, values[1]), () -> asListHtml(encodeId(values[0]))); } saveInfo(messages -> messages.addSuccessDeleteFile(GLOBAL, values[1])); - if (StringUtil.isEmpty(values[0])) { - return redirect(getClass()); - } return redirectWith(getClass(), moreUrl("list/" + encodeId(values[0]))); } @@ -152,6 +146,44 @@ public class AdminStorageAction extends FessAdminAction { return redirectWith(getClass(), moreUrl("list/" + encodeId(getObjectName(form.path, form.name)))); } + public static void uploadObject(final String objectName, final MultipartFormFile uploadFile) { + try (final InputStream in = uploadFile.getInputStream()) { + final FessConfig fessConfig = ComponentUtil.getFessConfig(); + final MinioClient minioClient = createClient(fessConfig); + minioClient.putObject(fessConfig.getStorageBucket(), objectName, in, (long) uploadFile.getFileSize(), null, null, + "application/octet-stream"); + } catch (final Exception e) { + throw new StorageException("Failed to upload " + objectName, e); + } + } + + public static void downloadObject(final String objectName, final WrittenStreamOut out) { + final FessConfig fessConfig = ComponentUtil.getFessConfig(); + try (InputStream in = createClient(fessConfig).getObject(fessConfig.getStorageBucket(), objectName)) { + out.write(in); + } catch (final Exception e) { + throw new StorageException("Failed to download " + objectName, e); + } + } + + public static void deleteObject(final String objectName) { + try { + final FessConfig fessConfig = ComponentUtil.getFessConfig(); + final MinioClient minioClient = createClient(fessConfig); + minioClient.removeObject(fessConfig.getStorageBucket(), objectName); + } catch (final Exception e) { + throw new StorageException("Failed to delete " + objectName, e); + } + } + + protected static MinioClient createClient(final FessConfig fessConfig) { + try { + return new MinioClient(fessConfig.getStorageEndpoint(), fessConfig.getStorageAccessKey(), fessConfig.getStorageSecretKey()); + } catch (final Exception e) { + throw new StorageException("Failed to create MinioClient: " + fessConfig.getStorageEndpoint(), e); + } + } + public static List<Map<String, Object>> getFileItems(final String prefix) { final FessConfig fessConfig = ComponentUtil.getFessConfig(); final ArrayList<Map<String, Object>> list = new ArrayList<>(); @@ -162,7 +194,7 @@ public class AdminStorageAction extends FessAdminAction { final Map<String, Object> map = new HashMap<>(); final Item item = result.get(); final String objectName = item.objectName(); - map.put("id", URLEncoder.encode(objectName, Constants.UTF_8_CHARSET)); + map.put("id", encodeId(objectName)); map.put("name", getName(objectName)); map.put("hashCode", item.hashCode()); map.put("size", item.objectSize()); @@ -191,16 +223,7 @@ public class AdminStorageAction extends FessAdminAction { return values[values.length - 1]; } - protected static MinioClient createClient(final FessConfig fessConfig) { - try { - return new MinioClient(fessConfig.getStorageEndpoint(), fessConfig.getStorageAccessKey(), fessConfig.getStorageSecretKey()); - } catch (final Exception e) { - throw new StorageException("Failed to create MinioClient: " + fessConfig.getStorageEndpoint(), e); - } - - } - - protected static String decodePath(final String id) { + public static String decodePath(final String id) { final String[] values = decodeId(id); if (StringUtil.isEmpty(values[0]) && StringUtil.isEmpty(values[1])) { return StringUtil.EMPTY; @@ -211,8 +234,8 @@ public class AdminStorageAction extends FessAdminAction { } } - protected static String[] decodeId(final String id) { - final String value = URLDecoder.decode(id, Constants.UTF_8_CHARSET); + public static String[] decodeId(final String id) { + final String value = urlDecode(urlDecode(id)); final String[] values = split(value, "/").get(stream -> stream.filter(StringUtil::isNotEmpty).toArray(n -> new String[n])); if (values.length == 0) { // invalid? @@ -269,19 +292,26 @@ public class AdminStorageAction extends FessAdminAction { return StringUtil.isEmpty(path) ? StringUtil.EMPTY : path + "/"; } - protected static String getObjectName(final String path, final String name) { + public static String getObjectName(final String path, final String name) { return getPathPrefix(path) + name; } protected static String urlEncode(final String str) { if (str == null) { - return null; + return StringUtil.EMPTY; } return URLEncoder.encode(str, Constants.UTF_8_CHARSET); } - protected static String encodeId(final String str) { - return urlEncode(urlEncode(str)); + protected static String urlDecode(final String str) { + if (str == null) { + return StringUtil.EMPTY; + } + return URLDecoder.decode(str, Constants.UTF_8_CHARSET); + } + + protected static String encodeId(final String objectName) { + return urlEncode(urlEncode(objectName)); } private HtmlResponse asListHtml(final String prefix) { diff --git a/src/main/java/org/codelibs/fess/app/web/api/ApiResult.java b/src/main/java/org/codelibs/fess/app/web/api/ApiResult.java index dfff8614bcada47eec6b8dea1d229e3ed8fd678d..7d04a3addd07d337ec965a2afc5c9eb62bc0e2db 100644 --- a/src/main/java/org/codelibs/fess/app/web/api/ApiResult.java +++ b/src/main/java/org/codelibs/fess/app/web/api/ApiResult.java @@ -361,4 +361,19 @@ public class ApiResult { return new ApiResult(this); } } + + public static class ApiStorageResponse extends ApiResponse { + protected List<Map<String, Object>> items; + + public ApiStorageResponse items(final List<Map<String, Object>> items) { + this.items = items; + return this; + } + + @Override + public ApiResult result() { + return new ApiResult(this); + } + } + } diff --git a/src/main/java/org/codelibs/fess/app/web/api/admin/storage/ApiAdminStorageAction.java b/src/main/java/org/codelibs/fess/app/web/api/admin/storage/ApiAdminStorageAction.java new file mode 100644 index 0000000000000000000000000000000000000000..ffdff990b58f589093a1c0fbed05adeea0de958f --- /dev/null +++ b/src/main/java/org/codelibs/fess/app/web/api/admin/storage/ApiAdminStorageAction.java @@ -0,0 +1,113 @@ +/* + * Copyright 2012-2019 CodeLibs Project and the Others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package org.codelibs.fess.app.web.api.admin.storage; + +import static org.codelibs.fess.app.web.admin.storage.AdminStorageAction.decodeId; +import static org.codelibs.fess.app.web.admin.storage.AdminStorageAction.decodePath; +import static org.codelibs.fess.app.web.admin.storage.AdminStorageAction.getFileItems; +import static org.codelibs.fess.app.web.admin.storage.AdminStorageAction.deleteObject; +import static org.codelibs.fess.app.web.admin.storage.AdminStorageAction.downloadObject; +import static org.codelibs.fess.app.web.admin.storage.AdminStorageAction.getObjectName; +import static org.codelibs.fess.app.web.admin.storage.AdminStorageAction.uploadObject; + +import java.util.List; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.codelibs.core.lang.StringUtil; +import org.codelibs.fess.app.web.api.ApiResult; +import org.codelibs.fess.app.web.api.admin.FessApiAdminAction; +import org.codelibs.fess.exception.ResultOffsetExceededException; +import org.codelibs.fess.exception.StorageException; +import org.dbflute.optional.OptionalThing; +import org.lastaflute.web.Execute; +import org.lastaflute.web.response.JsonResponse; +import org.lastaflute.web.response.StreamResponse; + +public class ApiAdminStorageAction extends FessApiAdminAction { + + private static final Logger logger = LogManager.getLogger(ApiAdminStorageAction.class); + + // GET /api/admin/storage/list/{id} + // POST /api/admin/storage/list/{id} + @Execute + public JsonResponse<ApiResult> list(final OptionalThing<String> id) { + final List<Map<String, Object>> list = getFileItems(id.isPresent() ? decodePath(id.get()) : null); + try { + return asJson(new ApiResult.ApiStorageResponse().items(list).status(ApiResult.Status.OK).result()); + } catch (final ResultOffsetExceededException e) { + if (logger.isDebugEnabled()) { + logger.debug(e.getMessage(), e); + } + throwValidationErrorApi(messages -> messages.addErrorsResultSizeExceeded(GLOBAL)); + } + + return null; + } + + // GET /api/admin/storage/download/{id}/ + @Execute + public StreamResponse get$download(final String id) { + final String[] values = decodeId(id); + if (StringUtil.isEmpty(values[1])) { + throwValidationErrorApi(messages -> messages.addErrorsStorageFileNotFound(GLOBAL)); + } + return asStream(values[1]).contentTypeOctetStream().stream(out -> { + try { + downloadObject(getObjectName(values[0], values[1]), out); + } catch (final StorageException e) { + throwValidationErrorApi(messages -> messages.addErrorsStorageFileDownloadFailure(GLOBAL, values[1])); + } + }); + } + + // DELETE /api/admin/storage/delete/{id}/ + @Execute + public JsonResponse<ApiResult> delete$delete(final String id) { + final String[] values = decodeId(id); + if (StringUtil.isEmpty(values[1])) { + throwValidationErrorApi(messages -> messages.addErrorsStorageAccessError(GLOBAL, "id is invalid")); + } + final String objectName = getObjectName(values[0], values[1]); + try { + deleteObject(objectName); + saveInfo(messages -> messages.addSuccessDeleteFile(GLOBAL, values[1])); + return asJson(new ApiResult.ApiResponse().status(ApiResult.Status.OK).result()); + } catch (final StorageException e) { + throwValidationErrorApi(messages -> messages.addErrorsFailedToDeleteFile(GLOBAL, values[1])); + } + return null; + } + + // POST /api/admin/storage/upload/{pathId}/ + @Execute + public JsonResponse<ApiResult> post$upload(final String pathId, final UploadForm form) { + validateApi(form, messages -> {}); + if (form.uploadFile == null) { + throwValidationErrorApi(messages -> messages.addErrorsStorageNoUploadFile(GLOBAL)); + } + try { + uploadObject(getObjectName(decodeId(pathId)[0], form.uploadFile.getFileName()), form.uploadFile); + saveInfo(messages -> messages.addSuccessUploadFileToStorage(GLOBAL, form.uploadFile.getFileName())); + return asJson(new ApiResult.ApiResponse().status(ApiResult.Status.OK).result()); + } catch (final StorageException e) { + throwValidationErrorApi(messages -> messages.addErrorsStorageFileUploadFailure(GLOBAL, form.uploadFile.getFileName())); + } + return null; + } + +} diff --git a/src/main/java/org/codelibs/fess/app/web/api/admin/storage/UploadForm.java b/src/main/java/org/codelibs/fess/app/web/api/admin/storage/UploadForm.java new file mode 100644 index 0000000000000000000000000000000000000000..3c12ec5bf84bb7f9d1b7e28fbf53ed3eace02e60 --- /dev/null +++ b/src/main/java/org/codelibs/fess/app/web/api/admin/storage/UploadForm.java @@ -0,0 +1,27 @@ +/* + * Copyright 2012-2019 CodeLibs Project and the Others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package org.codelibs.fess.app.web.api.admin.storage; + +import org.codelibs.fess.app.web.admin.storage.ItemForm; +import org.lastaflute.web.ruts.multipart.MultipartFormFile; +import org.lastaflute.web.validation.Required; + +public class UploadForm extends ItemForm { + + @Required + public MultipartFormFile uploadFile; + +} diff --git a/src/main/webapp/WEB-INF/view/admin/storage/admin_storage.jsp b/src/main/webapp/WEB-INF/view/admin/storage/admin_storage.jsp index fb4be739d7e2af5f7c9253f0047b826895debf84..e122163d020c3f153f7dee976e804775a23b6f68 100644 --- a/src/main/webapp/WEB-INF/view/admin/storage/admin_storage.jsp +++ b/src/main/webapp/WEB-INF/view/admin/storage/admin_storage.jsp @@ -151,7 +151,7 @@ </c:if> <c:if test="${data.directory.booleanValue()}"> <tr - data-href="${contextPath}/admin/storage/list/${f:u(data.id)}/"> + data-href="${contextPath}/admin/storage/list/${f:h(data.id)}/"> <td> <em class="fa fa-folder-open" style="color:#F7C502;"></em> ${f:h(data.name)} @@ -162,7 +162,7 @@ <td> <c:if test="${not data.directory}"> <a class="btn btn-primary btn-xs" role="button" name="download" data-toggle="modal" - href="${contextPath}/admin/storage/download/${f:u(data.id)}/" download="${f:u(data.name)}" + href="${contextPath}/admin/storage/download/${f:h(data.id)}/" download="${f:u(data.name)}" value="<la:message key="labels.design_download_button" />" > <em class="fa fa-download"></em> @@ -197,7 +197,7 @@ <button type="button" class="btn btn-outline pull-left" data-dismiss="modal"> <la:message key="labels.crud_button_cancel" /> </button> - <la:form action="${contextPath}/admin/storage/delete/${f:u(data.id)}/" styleClass="form-horizontal"> + <la:form action="${contextPath}/admin/storage/delete/${f:h(data.id)}/" styleClass="form-horizontal"> <button type="submit" class="btn btn-outline btn-danger" name="delete" value="<la:message key="labels.crud_button_delete" />" > diff --git a/src/test/java/org/codelibs/fess/it/admin/StorageTests.java b/src/test/java/org/codelibs/fess/it/admin/StorageTests.java new file mode 100644 index 0000000000000000000000000000000000000000..7456ffe8c49bc9bf4031354e3fea712b97a1b023 --- /dev/null +++ b/src/test/java/org/codelibs/fess/it/admin/StorageTests.java @@ -0,0 +1,86 @@ +/* + * Copyright 2012-2019 CodeLibs Project and the Others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package org.codelibs.fess.it.admin; + +import static org.hamcrest.Matchers.equalTo; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import org.codelibs.fess.it.CrudTestBase; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +@Tag("it") +public class StorageTests extends CrudTestBase { + + private static final String NAME_PREFIX = "storageTests_"; + private static final String API_PATH = "/api/admin/storage"; + private static final String LIST_ENDPOINT_SUFFIX = "list"; + private static final String ITEM_ENDPOINT_SUFFIX = ""; + + private static final String KEY_PROPERTY = ""; + + @Override + protected String getNamePrefix() { + return NAME_PREFIX; + } + + @Override + protected String getApiPath() { + return API_PATH; + } + + @Override + protected String getKeyProperty() { + return KEY_PROPERTY; + } + + @Override + protected String getListEndpointSuffix() { + return LIST_ENDPOINT_SUFFIX; + } + + @Override + protected String getItemEndpointSuffix() { + return ITEM_ENDPOINT_SUFFIX; + } + + @Override + protected Map<String, Object> createTestParam(int id) { + final Map<String, Object> requestBody = new HashMap<>(); + return requestBody; + } + + @Override + protected Map<String, Object> getUpdateMap() { + final Map<String, Object> updateMap = new HashMap<>(); + return updateMap; + } + + @AfterEach + protected void tearDown() { + // do nothing + } + + @Test + void testList_ok() { + checkGetMethod(Collections.emptyMap(), getListEndpointSuffix() + "/").then().body("response.status", equalTo(0)); + } + +}