Merge pull request #204 from FederatedAI/dev-2.1.7

Dev 2.1.7
FederatedAI · Nov 9, 2023 · c7602ab · c7602ab
2 parents d50cb78 + 6a9d9de
commit c7602ab
Show file tree

Hide file tree

Showing 62 changed files with 1,108 additions and 268 deletions.
diff --git a/document/docs/service/adapter.md b/document/docs/service/adapter.md
@@ -40,8 +40,10 @@ Context为上下文信息，用于传递请求所需参数，featureIds用于传
 #在host方的配置文件serving-server.properties中将其配置成自定义的类的全路径，如下所示
 feature.single.adaptor=com.webank.ai.fate.serving.adaptor.dataaccess.CustomAdapter
 feature.batch.adaptor=com.webank.ai.fate.serving.adaptor.dataaccess.CustomBatchAdapter
+feature.batch.single.adatpor=com.webank.ai.fate.serving.adaptor.dataaccess.CustomAdapter
 ```
 可以根据需要实现Adapter中的逻辑，并修改serving-server.properties中feature.single.adaptor或feature.batch.adaptor配置项为新增Adapter的全类名即可。可以参考源码中的MockAdaptor
+注: feature.batch.single.adatpor与feature.batch.adatpor配套使用，feature.batch.single.adatpor可根据用户场景自行实现,fate-serving中目前支持httpAdaptor
 
 ## fate-serving-extension
 为了更好的代码解耦合，代码中将自定义adapter分离到fate-serving-extension模块中。用户可在此模块中开发自定义的adapter。
@@ -69,7 +71,7 @@ x0:1,x1:5,x2:13,x3:58,x4:95,x5:352,x6:418,x7:833,x8:888,x9:937,x10:32776
 
 #### HttpAdapter
 在serving-server.properties文件中配置属性feature.single.adaptor和http.adapter.url，feature.single.adaptor为继承AbstractSingleFeatureDataAdaptor
-接口，url为调用获取数据接口地址。  
+接口，url为调用获取数据接口地址。http.adapter.url中标明的用户接口，返回格式请定义为 {"code": 200, "data": xxx}标准格式即可，httpAdapter中会根据接口返回状态码是否为200判断用户数据拉取接口是否执行成功。
 ```yaml
 feature.single.adaptor=com.webank.ai.fate.serving.adaptor.dataaccess.HttpAdapter
 http.adapter.url=http://127.0.0.1:9380/v1/http/adapter/getFeature

diff --git a/document/docs/service/admin.md b/document/docs/service/admin.md
@@ -5,6 +5,8 @@ serving-admin提供了FATE-Serving集群的可视化操作界面，依赖zookeep
 ### 功能介绍
 #### 用户管理
 默认用户：admin，默认密码：admin，用户可在[conf/application.properties](config/admin.md)中修改预设用户。    
+除此之外serving-admin提供一个基本的登录密码加解密功能，用户可在[conf/application.properties](config/application.properties)
+中通过设置admin.isEncrypt参数为true（默认为false关闭），同时根据spring.security中的BCryptPasswordEncoder库对密码进行提前处理并预设为默认密码。
 serving-admin仅实现简单的用户登录，用户可业务需求，自行实现登录逻辑，或接入第三方平台。
 
 #### 节点管理

diff --git a/fate-serving-admin-ui/README.md b/fate-serving-admin-ui/README.md
@@ -4,7 +4,7 @@ fate-serving定制联邦服务管理端
 ## Build Setup
 
 # Install dependencies
-npm install
+npm install --force
 
 # Serve with hot reload at localhost:8080
 npm run serve

diff --git a/fate-serving-admin-ui/package.json b/fate-serving-admin-ui/package.json
@@ -48,6 +48,7 @@
     "svg-sprite-loader": "^3.9.2",
     "vue-template-compiler": "^2.6.10",
     "webpack-bundle-analyzer": ">=3.3.2",
-    "webpack-cli": "^3.2.3"
+    "webpack-cli": "^3.2.3",
+    "webpack": "^4.0.0"
   }
 }
diff --git a/fate-serving-admin-ui/pom.xml b/fate-serving-admin-ui/pom.xml
@@ -55,7 +55,7 @@
                             <goal>install-node-and-npm</goal>
                         </goals>
                         <configuration>
-                            <nodeVersion>v9.11.1</nodeVersion>
+                            <nodeVersion>v16.20.2</nodeVersion>
                         </configuration>
                     </execution>
                     <!-- Install all project dependencies -->
@@ -68,7 +68,7 @@
                         <phase>generate-resources</phase>
                         <!-- Optional configuration which provides for running any npm command -->
                         <configuration>
-                            <arguments>install</arguments>
+                            <arguments>install --force</arguments>
                         </configuration>
                     </execution>
                     <!-- Build and minify static files -->

diff --git a/fate-serving-admin/bin/service.sh b/fate-serving-admin/bin/service.sh
@@ -24,7 +24,7 @@ basepath=$(cd `dirname $0`;pwd)
 configpath=$(cd $basepath/conf;pwd)
 module=serving-admin
 main_class=com.webank.ai.fate.serving.admin.Bootstrap
-module_version=2.1.6
+module_version=2.1.7
 
 case "$1" in
     start)

diff --git a/fate-serving-admin/pom.xml b/fate-serving-admin/pom.xml
@@ -51,6 +51,11 @@
             <version>${fate.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-security</artifactId>
+        </dependency>
+
 
 
         <!--<dependency>

diff --git a/fate-serving-admin/src/main/java/com/webank/ai/fate/serving/admin/Bootstrap.java b/fate-serving-admin/src/main/java/com/webank/ai/fate/serving/admin/Bootstrap.java
@@ -119,6 +119,7 @@ public void stop() {
             } catch (InterruptedException e) {
                 e.printStackTrace();
             }
+            tryNum++;
         }
     }
 }
diff --git a/...rving-admin/src/main/java/com/webank/ai/fate/serving/admin/bean/ServiceConfiguration.java b/...rving-admin/src/main/java/com/webank/ai/fate/serving/admin/bean/ServiceConfiguration.java
@@ -44,8 +44,7 @@ public static boolean isAllowModify(String project, String config) {
             return Boolean.FALSE;
         }
 
-        boolean match = Arrays.asList(value.config).contains(config);
-        return match;
+        return Arrays.asList(value.config).contains(config);
     }
 
 }
diff --git a/fate-serving-admin/src/main/java/com/webank/ai/fate/serving/admin/config/WebConfig.java b/fate-serving-admin/src/main/java/com/webank/ai/fate/serving/admin/config/WebConfig.java
@@ -72,8 +72,7 @@ public Cache cache() {
         Integer maxSize = MetaInfo.PROPERTY_LOCAL_CACHE_MAXSIZE;
         Integer expireTime = MetaInfo.PROPERTY_LOCAL_CACHE_EXPIRE;
         Integer interval = MetaInfo.PROPERTY_LOCAL_CACHE_INTERVAL;
-        ExpiringLRUCache lruCache = new ExpiringLRUCache(maxSize, expireTime, interval);
-        return lruCache;
+        return new ExpiringLRUCache(maxSize, expireTime, interval);
     }
 
 

diff --git a/...-admin/src/main/java/com/webank/ai/fate/serving/admin/controller/ComponentController.java b/...-admin/src/main/java/com/webank/ai/fate/serving/admin/controller/ComponentController.java
@@ -16,12 +16,14 @@
 
 package com.webank.ai.fate.serving.admin.controller;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Maps;
 import com.webank.ai.fate.api.networking.common.CommonServiceGrpc;
 import com.webank.ai.fate.api.networking.common.CommonServiceProto;
 import com.webank.ai.fate.serving.admin.bean.ServiceConfiguration;
 import com.webank.ai.fate.serving.admin.services.ComponentService;
+import com.webank.ai.fate.serving.admin.utils.NetAddressChecker;
 import com.webank.ai.fate.serving.core.bean.*;
 import com.webank.ai.fate.serving.core.constant.StatusCode;
 import com.webank.ai.fate.serving.core.exceptions.RemoteRpcException;
@@ -53,24 +55,22 @@ public class ComponentController {
 
     private static final Logger logger = LoggerFactory.getLogger(ComponentController.class);
 
+    private final ObjectMapper objectMapper = new ObjectMapper();
+
     @Autowired
     ComponentService componentServices;
     GrpcConnectionPool grpcConnectionPool = GrpcConnectionPool.getPool();
 
     @GetMapping("/component/list")
     public ReturnResult list() {
         ComponentService.NodeData cachedNodeData = componentServices.getCachedNodeData();
-        return ReturnResult.build(StatusCode.SUCCESS, Dict.SUCCESS, JsonUtil.json2Object(JsonUtil.object2Json(cachedNodeData), Map.class));
+        Map<String, Object> cachedNodeDataMap = objectMapper.convertValue(cachedNodeData, Map.class);
+        return ReturnResult.build(StatusCode.SUCCESS, Dict.SUCCESS, cachedNodeDataMap);
     }
 
     @GetMapping("/component/listProps")
     public ReturnResult listProps(String host, int port, String keyword) {
-        if (!NetUtils.isValidAddress(host + ":" + port)) {
-            throw new SysException("invalid address");
-        }
-        if (!componentServices.isAllowAccess(host, port)) {
-            throw new RemoteRpcException("no allow access, target: " + host + ":" + port);
-        }
+        NetAddressChecker.check(host, port);
         ManagedChannel managedChannel = grpcConnectionPool.getManagedChannel(host, port);
         CommonServiceGrpc.CommonServiceBlockingStub blockingStub = CommonServiceGrpc.newBlockingStub(managedChannel);
         blockingStub = blockingStub.withDeadlineAfter(MetaInfo.PROPERTY_GRPC_TIMEOUT, TimeUnit.MILLISECONDS);
@@ -102,17 +102,15 @@ public ReturnResult listProps(String host, int port, String keyword) {
 
     @PostMapping("/component/updateConfig")
     public ReturnResult updateConfig(@RequestBody RequestParamWrapper requestParams) {
-        Preconditions.checkArgument(StringUtils.isNotBlank(requestParams.getFilePath()), "file path is blank");
-        Preconditions.checkArgument(StringUtils.isNotBlank(requestParams.getData()), "data is blank");
+        String filePath = requestParams.getFilePath();
+        String data = requestParams.getData();
+        Preconditions.checkArgument(StringUtils.isNotBlank(filePath), "file path is blank");
+        Preconditions.checkArgument(StringUtils.isNotBlank(data), "data is blank");
 
         String host = requestParams.getHost();
         int port = requestParams.getPort();
+        NetAddressChecker.check(host, port);
 
-        if (!componentServices.isAllowAccess(host, port)) {
-            throw new RemoteRpcException("no allow access, target: " + host + ":" + port);
-        }
-
-        String filePath = requestParams.getFilePath();
         String fileName = filePath.substring(filePath.lastIndexOf(File.separator) + 1);
 
         String project = componentServices.getProject(host, port);
@@ -124,8 +122,8 @@ public ReturnResult updateConfig(@RequestBody RequestParamWrapper requestParams)
         CommonServiceGrpc.CommonServiceBlockingStub blockingStub = CommonServiceGrpc.newBlockingStub(managedChannel)
                 .withDeadlineAfter(MetaInfo.PROPERTY_GRPC_TIMEOUT, TimeUnit.MILLISECONDS);
         CommonServiceProto.UpdateConfigRequest.Builder builder = CommonServiceProto.UpdateConfigRequest.newBuilder();
-        builder.setFilePath(requestParams.getFilePath());
-        builder.setData(requestParams.getData());
+        builder.setFilePath(filePath);
+        builder.setData(data);
 
         CommonServiceProto.CommonResponse response = blockingStub.updateConfig(builder.build());
         return ReturnResult.build(response.getStatusCode(), response.getMessage());

diff --git a/...admin/src/main/java/com/webank/ai/fate/serving/admin/controller/DynamicLogController.java b/...admin/src/main/java/com/webank/ai/fate/serving/admin/controller/DynamicLogController.java
@@ -0,0 +1,42 @@
+package com.webank.ai.fate.serving.admin.controller;
+
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.web.bind.annotation.*;
+
+/**
+ * @author hcy
+ */
+@RequestMapping("/admin")
+@RestController
+public class DynamicLogController {
+    private static final Logger logger = LoggerFactory.getLogger(DynamicLogController.class);
+
+    @GetMapping("/alterSysLogLevel/{level}")
+    public String alterSysLogLevel(@PathVariable String level){
+        try {
+            LoggerContext context = (LoggerContext) LogManager.getContext(false);
+            context.getLogger("ROOT").setLevel(Level.valueOf(level));
+            return "ok";
+        } catch (Exception ex) {
+            logger.error("admin alterSysLogLevel failed : " + ex);
+            return "failed";
+        }
+
+    }
+
+    @GetMapping("/alterPkgLogLevel")
+    public String alterPkgLogLevel(@RequestParam String level, @RequestParam String pkgName){
+        try {
+            LoggerContext context = (LoggerContext) LogManager.getContext(false);
+            context.getLogger(pkgName).setLevel(Level.valueOf(level));
+            return "ok";
+        } catch (Exception ex) {
+            logger.error("admin alterPkgLogLevel failed : " + ex);
+            return "failed";
+        }
+    }
+}
diff --git a/...ving-admin/src/main/java/com/webank/ai/fate/serving/admin/controller/LoginController.java b/...ving-admin/src/main/java/com/webank/ai/fate/serving/admin/controller/LoginController.java
@@ -30,6 +30,7 @@
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
+import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder;
 
 import javax.servlet.http.HttpServletRequest;
 import java.util.Arrays;
@@ -51,7 +52,10 @@ public class LoginController {
     private String username;
 
     @Value("${admin.password}")
-    private String password;
+    private String hashedPassword;
+
+    @Value("${admin.isEncrypt}")
+    private Boolean isEncrypt;
 
     @Autowired
     private Cache cache;
@@ -60,12 +64,20 @@ public class LoginController {
     public ReturnResult login(@RequestBody RequestParamWrapper requestParams) {
         String username = requestParams.getUsername();
         String password = requestParams.getPassword();
+        boolean passwordIfCorrect;
 
         Preconditions.checkArgument(StringUtils.isNotBlank(username), "parameter username is blank");
         Preconditions.checkArgument(StringUtils.isNotBlank(password), "parameter password is blank");
 
         ReturnResult result = new ReturnResult();
-        if (username.equals(this.username) && password.equals(this.password)) {
+
+        if (isEncrypt) {
+             passwordIfCorrect = new BCryptPasswordEncoder().matches(password, this.hashedPassword);
+        } else {
+             passwordIfCorrect = password.equals(this.hashedPassword);
+        }
+
+        if (username.equals(this.username) && passwordIfCorrect) {
             String userInfo = StringUtils.join(Arrays.asList(username, password), "_");
             String token = EncryptUtils.encrypt(Dict.USER_CACHE_KEY_PREFIX + userInfo, EncryptMethod.MD5);
             cache.put(token, userInfo, MetaInfo.PROPERTY_CACHE_TYPE.equalsIgnoreCase("local") ? MetaInfo.PROPERTY_LOCAL_CACHE_EXPIRE : MetaInfo.PROPERTY_REDIS_EXPIRE);
@@ -77,7 +89,7 @@ public ReturnResult login(@RequestBody RequestParamWrapper requestParams) {
             result.setRetcode(StatusCode.SUCCESS);
             result.setData(data);
         } else {
-            logger.info("user {} login failure, username or password {} is wrong.", username,password);
+            logger.error("user {} login failure, username or password {} is wrong.", username,password);
             result.setRetcode(StatusCode.PARAM_ERROR);
             result.setRetmsg("username or password is wrong");
         }
@@ -94,7 +106,7 @@ public ReturnResult logout(HttpServletRequest request) {
             cache.delete(sessionToken);
             result.setRetcode(StatusCode.SUCCESS);
         } else {
-            logger.info("Session token unavailable");
+            logger.error("Session token unavailable");
             result.setRetcode(StatusCode.PARAM_ERROR);
             result.setRetmsg("Session token unavailable");
         }

diff --git a/...ving-admin/src/main/java/com/webank/ai/fate/serving/admin/controller/ModelController.java b/...ving-admin/src/main/java/com/webank/ai/fate/serving/admin/controller/ModelController.java
@@ -23,6 +23,7 @@
 import com.webank.ai.fate.api.mlmodel.manager.ModelServiceGrpc;
 import com.webank.ai.fate.api.mlmodel.manager.ModelServiceProto;
 import com.webank.ai.fate.serving.admin.services.ComponentService;
+import com.webank.ai.fate.serving.admin.utils.NetAddressChecker;
 import com.webank.ai.fate.serving.core.bean.GrpcConnectionPool;
 import com.webank.ai.fate.serving.core.bean.MetaInfo;
 import com.webank.ai.fate.serving.core.bean.RequestParamWrapper;
@@ -66,12 +67,15 @@ public ReturnResult queryModel(String host, Integer port, String serviceId,Strin
         Preconditions.checkArgument(StringUtils.isNotBlank(host), "parameter host is blank");
         Preconditions.checkArgument(port != 0, "parameter port is blank");
 
-        if (page == null || page < 0) {
-            page = 1;
+        int defaultPage = 1;
+        int defaultPageSize = 10;
+
+        if (page == null || page <= 0) {
+            page = defaultPage;
         }
 
-        if (pageSize == null) {
-            pageSize = 10;
+        if (pageSize == null || pageSize <= 0) {
+            pageSize = defaultPageSize;
         }
 
         if (logger.isDebugEnabled()) {
@@ -153,7 +157,6 @@ public Callable<ReturnResult> transfer(@RequestBody RequestParamWrapper requestP
         return () -> {
             String host = requestParams.getHost();
             Integer port = requestParams.getPort();
-            List<String> serviceIds = requestParams.getServiceIds();
             String tableName = requestParams.getTableName();
             String namespace = requestParams.getNamespace();
 
@@ -176,13 +179,6 @@ public Callable<ReturnResult> transfer(@RequestBody RequestParamWrapper requestP
                     //.setServiceId()
                     .setNamespace(namespace).setTableName(tableName).setSourceIp(host).setSourcePort(port).build();
 
-
-
-            ModelServiceProto.UnloadRequest unloadRequest = ModelServiceProto.UnloadRequest.newBuilder()
-                    .setTableName(tableName)
-                    .setNamespace(namespace)
-                    .build();
-
             ListenableFuture<ModelServiceProto.FetchModelResponse> future = futureStub.fetchModel(fetchModelRequest);
             ModelServiceProto.FetchModelResponse response = future.get(MetaInfo.PROPERTY_GRPC_TIMEOUT, TimeUnit.MILLISECONDS);
 
@@ -281,13 +277,7 @@ private ModelServiceGrpc.ModelServiceBlockingStub getModelServiceBlockingStub(St
         Preconditions.checkArgument(StringUtils.isNotBlank(host), "parameter host is blank");
         Preconditions.checkArgument(port != null && port.intValue() != 0, "parameter port was wrong");
 
-        if (!NetUtils.isValidAddress(host + ":" + port)) {
-            throw new SysException("invalid address");
-        }
-
-        if (!componentService.isAllowAccess(host, port)) {
-            throw new RemoteRpcException("no allow access, target: " + host + ":" + port);
-        }
+        NetAddressChecker.check(host, port);
 
         ManagedChannel managedChannel = grpcConnectionPool.getManagedChannel(host, port);
         ModelServiceGrpc.ModelServiceBlockingStub blockingStub = ModelServiceGrpc.newBlockingStub(managedChannel);
@@ -299,17 +289,10 @@ private ModelServiceGrpc.ModelServiceFutureStub getModelServiceFutureStub(String
         Preconditions.checkArgument(StringUtils.isNotBlank(host), "parameter host is blank");
         Preconditions.checkArgument(port != null && port.intValue() != 0, "parameter port was wrong");
 
-        if (!NetUtils.isValidAddress(host + ":" + port)) {
-            throw new SysException("invalid address");
-        }
-
-        if (!componentService.isAllowAccess(host, port)) {
-            throw new RemoteRpcException("no allow access, target: " + host + ":" + port);
-        }
+        NetAddressChecker.check(host, port);
 
         ManagedChannel managedChannel = grpcConnectionPool.getManagedChannel(host, port);
-        ModelServiceGrpc.ModelServiceFutureStub futureStub = ModelServiceGrpc.newFutureStub(managedChannel);
-        return futureStub;
+        return ModelServiceGrpc.newFutureStub(managedChannel);
     }
 
     public void parseComponentInfo(ModelServiceProto.QueryModelResponse response){