1. fix sanitize query for js data source

jerry-goodman · jerry-goodman · commit 9b68c087e96e · 2023-01-17T19:09:18.000+08:00
2. fix issue that library query doesn't show when imported from other workspaces in SAAS mode
3. fix: none exist datasource permission check for library-query list and application save
diff --git a/server/api-service/openblocks-domain/src/main/java/com/openblocks/domain/datasource/repository/DatasourceRepository.java b/server/api-service/openblocks-domain/src/main/java/com/openblocks/domain/datasource/repository/DatasourceRepository.java
@@ -3,6 +3,12 @@
 import static com.openblocks.sdk.util.JsonUtils.fromJsonMap;
 import static com.openblocks.sdk.util.JsonUtils.toJson;
 
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.function.Function;
+
+import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Repository;
 
@@ -16,6 +22,7 @@
 import com.openblocks.domain.plugin.service.DatasourceMetaInfoService;
 import com.openblocks.infra.mongo.MongoUpsertHelper;
 import com.openblocks.sdk.models.DatasourceConnectionConfig;
+import com.openblocks.sdk.models.HasIdAndAuditing;
 import com.openblocks.sdk.models.JsDatasourceConnectionConfig;
 import com.openblocks.sdk.util.JsonUtils;
 
@@ -77,6 +84,23 @@ public Mono<Boolean> markDatasourceAsDeleted(String datasourceId) {
         return mongoUpsertHelper.updateById(datasource, datasourceId);
     }
 
+    public Flux<String> retainNoneExistAndNonCurrentOrgDatasourceIds(Collection<String> datasourceIds, String orgId) {
+        if (CollectionUtils.isEmpty(datasourceIds)) {
+            return Flux.empty();
+        }
+        return repository.findAllById(new HashSet<>(datasourceIds))
+                .collectList()
+                .map(existDatasources -> {
+                    Set<String> result = new HashSet<>(datasourceIds);
+                    existDatasources.stream()
+                            .filter(datasource -> datasource.getOrganizationId().equals(orgId))
+                            .map(HasIdAndAuditing::getId)
+                            .forEach(result::remove);
+                    return result;
+                })
+                .flatMapIterable(Function.identity());
+    }
+
     public Mono<Long> countByOrganizationId(String orgId) {
         return repository.countByOrganizationId(orgId);
     }
diff --git a/server/api-service/openblocks-domain/src/main/java/com/openblocks/domain/datasource/service/DatasourceService.java b/server/api-service/openblocks-domain/src/main/java/com/openblocks/domain/datasource/service/DatasourceService.java
@@ -1,5 +1,7 @@
 package com.openblocks.domain.datasource.service;
 
+import java.util.Collection;
+
 import com.openblocks.domain.datasource.model.Datasource;
 import com.openblocks.sdk.models.DatasourceTestResult;
 
@@ -25,4 +27,6 @@ public interface DatasourceService {
     Mono<Long> countByOrganizationId(String orgId);
 
     Mono<Datasource> findWorkspacePredefinedDatasource(String organizationId, String datasourceType);
+
+    Flux<String> retainNoneExistAndNonCurrentOrgDatasourceIds(Collection<String> datasourceIds, String organizationId);
 }
diff --git a/server/api-service/openblocks-domain/src/main/java/com/openblocks/domain/datasource/service/impl/DatasourceServiceImpl.java b/server/api-service/openblocks-domain/src/main/java/com/openblocks/domain/datasource/service/impl/DatasourceServiceImpl.java
@@ -6,12 +6,14 @@
 import static org.apache.commons.collections4.CollectionUtils.isNotEmpty;
 
 import java.time.Duration;
+import java.util.Collection;
 import java.util.Locale;
 import java.util.Set;
 import java.util.stream.Collectors;
 
 import javax.annotation.Nonnull;
 
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.ObjectUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -220,6 +222,14 @@ public Mono<Datasource> findWorkspacePredefinedDatasource(String organizationId,
         return repository.findWorkspacePredefinedDatasourceByOrgIdAndType(organizationId, type);
     }
 
+    @Override
+    public Flux<String> retainNoneExistAndNonCurrentOrgDatasourceIds(Collection<String> datasourceIds, String orgId) {
+        if (CollectionUtils.isEmpty(datasourceIds)) {
+            return Flux.empty();
+        }
+        return repository.retainNoneExistAndNonCurrentOrgDatasourceIds(datasourceIds, orgId);
+    }
+
     @Override
     public Mono<Boolean> delete(String datasourceId) {
         return stillUsedInApplications(datasourceId)
diff --git a/server/api-service/openblocks-domain/src/main/java/com/openblocks/domain/plugin/service/impl/DatasourceMetaInfoServiceImpl.java b/server/api-service/openblocks-domain/src/main/java/com/openblocks/domain/plugin/service/impl/DatasourceMetaInfoServiceImpl.java
@@ -158,7 +158,9 @@ public List<DatasourceMetaInfo> getJavaBasedSupportedDatasourceMetaInfos() {
 
     @Override
     public boolean isJavaDatasourcePlugin(String type) {
-        return getJavaBasedSupportedDatasourceMetaInfos()
+        return "majiangInternal".equals(type)
+                || "openblocksApi".equals(type)
+                || getJavaBasedSupportedDatasourceMetaInfos()
                 .stream()
                 .anyMatch(datasourceMetaInfo -> datasourceMetaInfo.getType().equals(type));
     }
diff --git a/server/api-service/openblocks-server/src/main/java/com/openblocks/api/application/ApplicationApiService.java b/server/api-service/openblocks-server/src/main/java/com/openblocks/api/application/ApplicationApiService.java
@@ -31,6 +31,7 @@
 import org.springframework.stereotype.Service;
 
 import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
 import com.openblocks.api.application.ApplicationController.CreateApplicationRequest;
 import com.openblocks.api.application.view.ApplicationInfoView;
 import com.openblocks.api.application.view.ApplicationPermissionView;
@@ -47,6 +48,7 @@
 import com.openblocks.domain.application.service.ApplicationService;
 import com.openblocks.domain.bizthreshold.AbstractBizThresholdChecker;
 import com.openblocks.domain.datasource.model.Datasource;
+import com.openblocks.domain.datasource.service.DatasourceService;
 import com.openblocks.domain.group.service.GroupService;
 import com.openblocks.domain.interaction.UserApplicationInteractionService;
 import com.openblocks.domain.organization.model.Organization;
@@ -68,6 +70,7 @@
 import com.openblocks.sdk.constants.Authentication;
 import com.openblocks.sdk.exception.BizError;
 import com.openblocks.sdk.exception.BizException;
+import com.openblocks.sdk.plugin.common.QueryExecutor;
 import com.openblocks.sdk.util.ExceptionUtils;
 
 import lombok.extern.slf4j.Slf4j;
@@ -128,6 +131,8 @@ public class ApplicationApiService {
     private TemplateService templateService;
     @Autowired
     private PermissionHelper permissionHelper;
+    @Autowired
+    private DatasourceService datasourceService;
 
     public Mono<ApplicationView> create(CreateApplicationRequest createApplicationRequest) {
 
@@ -517,7 +522,12 @@ private Map<String, Object> doSanitizeQuery(Object query) {
                 VIEW_DATASOURCE_TYPE.equalsIgnoreCase(datasourceType)) {
             return queryMap;
         }
-        var queryExecutor = datasourceMetaInfoService.getQueryExecutor(datasourceType);
+        QueryExecutor<?, Object, ?> queryExecutor;
+        try {
+            queryExecutor = datasourceMetaInfoService.getQueryExecutor(datasourceType);
+        } catch (Exception e) {
+            return queryMap;
+        }
         Object comp = queryMap.get("comp");
         if (!(comp instanceof Map<?, ?> queryConfig)) {
             return queryMap;
@@ -551,10 +561,15 @@ private Mono<Void> checkDatasourcePermissions(Application application) {
                 return Mono.empty();
             }
 
+            String organizationId = application.getOrganizationId();
             return sessionUserService.getVisitorId()
-                    .flatMap(userId -> resourcePermissionService.haveAllEnoughPermissions(userId, datasourceIds, USE_DATASOURCES))
-                    .flatMap(havePermissions -> {
-                        if (havePermissions) {
+                    .flatMap(userId -> resourcePermissionService.getMaxMatchingPermission(userId, datasourceIds, USE_DATASOURCES))
+                    .zipWith(datasourceService.retainNoneExistAndNonCurrentOrgDatasourceIds(datasourceIds, organizationId).collectList())
+                    .flatMap(tuple -> {
+                        Set<String> hasPermissionDatasourceIds = tuple.getT1().keySet();
+                        List<String> noneExistDatasourceIds = tuple.getT2();
+
+                        if (Sets.union(hasPermissionDatasourceIds, new HashSet<>(noneExistDatasourceIds)).containsAll(datasourceIds)) {
                             return Mono.empty();
                         }
                         return ExceptionUtils.ofError(BizError.NOT_AUTHORIZED, "APPLICATION_EDIT_ERROR_LACK_OF_DATASOURCE_PERMISSIONS");
diff --git a/server/api-service/openblocks-server/src/main/java/com/openblocks/api/query/LibraryQueryApiService.java b/server/api-service/openblocks-server/src/main/java/com/openblocks/api/query/LibraryQueryApiService.java
@@ -103,21 +103,27 @@ private Flux<LibraryQuery> getByOrgIdWithDatasourcePermissions(String orgId) {
         Flux<LibraryQuery> libraryQueryFlux = libraryQueryService.getByOrganizationId(orgId)
                 .cache();
 
-        Mono<HashSet<String>> datasourceIdSetWithPermissions = libraryQueryFlux.map(libraryQuery -> libraryQuery.getQuery().getDatasourceId())
+        Mono<List<String>> datasourceIdListMono = libraryQueryFlux.map(libraryQuery -> libraryQuery.getQuery().getDatasourceId())
                 .filter(StringUtils::isNotBlank)
                 .collectList()
+                .cache();
+
+        Mono<HashSet<String>> datasourceIdSetWithPermissionsOrNoneExists = datasourceIdListMono
                 .zipWith(sessionUserService.getVisitorId())
                 .flatMapMany(tuple -> {
                     List<String> datasourceIds = tuple.getT1();
                     String userId = tuple.getT2();
                     return resourcePermissionService.filterResourceWithPermission(userId, datasourceIds, ResourceAction.USE_DATASOURCES);
                 })
+                .concatWith(datasourceIdListMono.flatMapMany(
+                        datasourceIds -> datasourceService.retainNoneExistAndNonCurrentOrgDatasourceIds(datasourceIds, orgId)))
                 .collectList()
                 .map(HashSet::new)
                 .cache();
 
         return libraryQueryFlux
-                .filterWhen(libraryQuery -> datasourceIdSetWithPermissions.map(set -> set.contains(libraryQuery.getQuery().getDatasourceId())));
+                .filterWhen(libraryQuery -> datasourceIdSetWithPermissionsOrNoneExists.map(
+                        set -> set.contains(libraryQuery.getQuery().getDatasourceId())));
     }
 
     public Mono<LibraryQueryView> create(LibraryQuery libraryQuery) {
@@ -275,7 +281,7 @@ public Mono<QueryExecutionResult> executeLibraryQuery(ServerWebExchange exchange
         Mono<BaseQuery> baseQueryMono = libraryQueryService.getEditingBaseQueryByLibraryQueryId(
                 queryExecutionRequest.getLibraryQueryCombineId().libraryQueryId()).cache();
         Mono<Datasource> datasourceMono = baseQueryMono.flatMap(query -> datasourceService.getById(query.getDatasourceId())
-                        .switchIfEmpty(deferredError(BizError.DATASOURCE_NOT_FOUND, "DATASOURCE_NOT_FOUND", query.getDatasourceId()))).cache();
+                .switchIfEmpty(deferredError(BizError.DATASOURCE_NOT_FOUND, "DATASOURCE_NOT_FOUND", query.getDatasourceId()))).cache();
 
         return orgDevChecker.checkCurrentOrgDev()
                 .then(Mono.zip(sessionUserService.getVisitorOrgMemberCache(),
diff --git a/server/api-service/openblocks-server/src/main/resources/application-openblocks.yml b/server/api-service/openblocks-server/src/main/resources/application-openblocks.yml
@@ -35,7 +35,7 @@ common:
   security:
     cors-allowed-domains:
       - '*'
-  version: 1.1.3
+  version: 1.1.4
   block-hound-enable: false
 
 material:
diff --git a/server/api-service/openblocks-server/src/main/resources/selfhost/ce/application.yml b/server/api-service/openblocks-server/src/main/resources/selfhost/ce/application.yml
@@ -27,7 +27,7 @@ common:
   domain:
     default-value: openblocks.dev
   cloud: false
-  version: 1.1.3
+  version: 1.1.4
   block-hound-enable: false
 
 material:
