support s3 env

be/CMakeLists.txt

@@ -186,6 +186,12 @@ if (WITH_HDFS)
     set_target_properties(hdfs PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libhdfs.a)
 endif()
 
+if (WITH_AWS)
+    set(AWSSDK_ROOT_DIR ${THIRDPARTY_DIR})
+    find_package(AWSSDK REQUIRED COMPONENTS core s3 transfer)
+    include_directories(${AWSSDK_INCLUDE_DIRS})
+endif()
+
 add_library(libevent STATIC IMPORTED)
 set_target_properties(libevent PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libevent.a)
 
@@ -323,6 +329,10 @@ if (WITH_HDFS)
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DSTARROCKS_WITH_HDFS")
 endif()
 
+if (WITH_AWS)
+    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DSTARROCKS_WITH_AWS")
+endif()
+
 if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0)
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -faligned-new")
 endif()
@@ -455,6 +465,13 @@ if (WITH_HDFS)
         )
 endif()
 
+if (WITH_AWS)
+    set(STARROCKS_DEPENDENCIES ${STARROCKS_DEPENDENCIES}
+        ${AWSSDK_LINK_LIBRARIES}
+        ${AWSSDK_PLATFORM_DEPS}
+        )
+endif()
+
 # Set thirdparty libraries
 set(STARROCKS_DEPENDENCIES
     ${STARROCKS_DEPENDENCIES}

be/src/env/CMakeLists.txt

@@ -39,6 +39,13 @@ if (WITH_HDFS)
     )
 endif()
 
+if (WITH_AWS)
+    set(EXEC_FILES ${EXEC_FILES}
+        env_s3.cpp
+        s3_client.cpp
+    )
+endif()
+
 add_library(Env STATIC
     ${EXEC_FILES}
 )

be/src/env/cloud_storage_client.h

@@ -0,0 +1,53 @@
+// This file is licensed under the Elastic License 2.0. Copyright 2021 StarRocks Limited.
+
+#pragma once
+
+#include <string>
+
+#include "common/status.h"
+
+namespace starrocks {
+
+// base class for cloud storage client
+class CloudStorageClient {
+public:
+    CloudStorageClient() = default;
+    virtual ~CloudStorageClient() = default;
+
+    /*
+     *  Bucket Operation
+     */
+    virtual Status create_bucket(const std::string& bucket_name) = 0;
+
+    virtual Status delete_bucket(const std::string& bucket_name) = 0;
+
+    /*
+     *  Object Operation
+     */
+    virtual Status put_object(const std::string& bucket_name, const std::string& object_key,
+                              const std::string& object_path) = 0;
+
+    virtual Status put_string_object(const std::string& bucket_name, const std::string& object_key,
+                                     const std::string& object_value) = 0;
+
+    virtual Status get_object(const std::string& bucket_name, const std::string& object_key,
+                              const std::string& object_path) = 0;
+
+    virtual Status get_object_range(const std::string& bucket_name, const std::string& object_key,
+                                    std::string* object_value, size_t offset, size_t length, size_t* read_bytes) = 0;
+
+    // `object_value` should already be allocated at least `length` bytes
+    virtual Status get_object_range(const std::string& bucket_name, const std::string& object_key,
+                                    char* object_value, size_t offset, size_t length, size_t* read_bytes) = 0;
+
+    virtual Status exist_object(const std::string& bucket_name, const std::string& object_key) = 0;
+
+    virtual Status get_object_size(const std::string& bucket_name, const std::string& object_key, size_t* size) = 0;
+
+    virtual Status delete_object(const std::string& bucket_name, const std::string& object_key) = 0;
+
+    virtual Status list_objects(const std::string& bucket_name, const std::string& object_prefix,
+                                std::vector<std::string>* result) = 0;
+};
+
+} // namespace starrocks