Skip to content

Commit

Permalink
Displaying only my custom bucket
Browse files Browse the repository at this point in the history
  • Loading branch information
@adhithya-nair
adhithya-nair committed Oct 19, 2023
1 parent d6f4816 commit 66ef7e4
Show file tree
Hide file tree
Showing 2 changed files with 122 additions and 23 deletions.
91 changes: 68 additions & 23 deletions jupyterlab_s3_browser/handlers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@
"""
import base64
import json
import os
import logging
from .logger import log
from pathlib import Path

import boto3
Expand All @@ -13,6 +15,11 @@
from jupyter_server.base.handlers import APIHandler
from jupyter_server.utils import url_path_join

for key, value in os.environ.items():
print(f'{key}={value}')
myCustomBucket = os.getenv('S3_BROWSER_BUCKET')
print(f"My Custom Bucket -> {myCustomBucket}")


class DirectoryNotEmptyException(Exception):
"""Raise for attempted deletions of non-empty directories"""
Expand All @@ -21,9 +28,9 @@ class DirectoryNotEmptyException(Exception):


def create_s3fs(config):

print("inside create_s3fs")
if config.endpoint_url and config.client_id and config.client_secret:

print(f"key -> {config.client_id} , secret -> {config.client_secret} , token -> {config.session_token} , client_kwargs-> {config.endpoint_url}")
return s3fs.S3FileSystem(
key=config.client_id,
secret=config.client_secret,
Expand All @@ -36,7 +43,7 @@ def create_s3fs(config):


def create_s3_resource(config):

print("inside create_s3_resource")
if config.endpoint_url and config.client_id and config.client_secret:

return boto3.resource(
Expand All @@ -55,6 +62,7 @@ def _test_aws_s3_role_access():
"""
Checks if we have access to AWS S3 through role-based access
"""
print("inside _test_aws_s3_role_access")
test = boto3.resource("s3")
all_buckets = test.buckets.all()
result = [
Expand All @@ -68,7 +76,7 @@ def has_aws_s3_role_access():
"""
Returns true if the user has access to an aws S3 bucket
"""

print("inside has_aws_s3_role_access")
# avoid making requests to AWS if the user's ~/.aws/credentials file has credentials for a different provider,
# e.g. https://cloud.ibm.com/docs/cloud-object-storage?topic=cloud-object-storage-aws-cli#aws-cli-config
aws_credentials_file = Path("{}/.aws/credentials".format(Path.home()))
Expand All @@ -82,7 +90,7 @@ def has_aws_s3_role_access():
"AKIA"
) and not access_key_id.startswith("ASIA"):
# if any keys are not valid AWS keys, don't try to authenticate
logging.info(
log.info(
"Found invalid AWS aws_access_key_id in ~/.aws/credentials file, "
"will not attempt to authenticate through ~/.aws/credentials."
)
Expand All @@ -94,7 +102,7 @@ def has_aws_s3_role_access():
except NoCredentialsError:
return False
except Exception as e:
logging.error(e)
log.error(e)
return False


Expand All @@ -103,6 +111,7 @@ def test_s3_credentials(endpoint_url, client_id, client_secret, session_token):
Checks if we're able to list buckets with these credentials.
If not, it throws an exception.
"""
print("inside test_s3_credentials")
test = boto3.resource(
"s3",
aws_access_key_id=client_id,
Expand All @@ -111,12 +120,21 @@ def test_s3_credentials(endpoint_url, client_id, client_secret, session_token):
aws_session_token=session_token,
)
all_buckets = test.buckets.all()
logging.debug(
[
{"name": bucket.name + "/", "path": bucket.name + "/", "type": "directory"}
for bucket in all_buckets
]
)
# logging.debug(
# [
# {"name": bucket.name + "/", "path": bucket.name + "/", "type": "directory"}
# for bucket in all_buckets
# ]
# )

# data = [
# {"name": bucket.name + "/", "path": bucket.name + "/", "type": "directory"}
# for bucket in all_buckets
# ]

print("Creating connection to my bucket")
print(f"List of buckets -> {all_buckets}")
# print(json.dumps(data, indent=4))


class AuthHandler(APIHandler): # pylint: disable=abstract-method
Expand All @@ -126,6 +144,7 @@ class AuthHandler(APIHandler): # pylint: disable=abstract-method

@property
def config(self):
print("inside AuthHandler -> config")
return self.settings["s3_config"]

@tornado.web.authenticated
Expand All @@ -134,9 +153,13 @@ def get(self, path=""):
Checks if the user is already authenticated
against an s3 instance.
"""
print("inside AuthHandler -> get")
print(self.settings["s3_config"])
print(dir(self.settings["s3_config"]))
authenticated = False
if has_aws_s3_role_access():
authenticated = True
print(authenticated)

if not authenticated:

Expand All @@ -149,7 +172,8 @@ def get(self, path=""):
config.client_secret,
config.session_token,
)
logging.debug("...successfully authenticated")
print("")
log.info("...successfully authenticated")

# If no exceptions were encountered during testS3Credentials,
# then assume we're authenticated
Expand All @@ -159,8 +183,8 @@ def get(self, path=""):
# If an exception was encountered,
# assume that we're not yet authenticated
# or invalid credentials were provided
logging.debug("...failed to authenticate")
logging.debug(err)
log.info("...failed to authenticate")
log.error(err)

self.finish(json.dumps({"authenticated": authenticated}))

Expand All @@ -169,7 +193,7 @@ def post(self, path=""):
"""
Sets s3 credentials.
"""

print("inside AuthHandler -> post")
try:
req = json.loads(self.request.body)
endpoint_url = req["endpoint_url"]
Expand All @@ -186,7 +210,7 @@ def post(self, path=""):

self.finish(json.dumps({"success": True}))
except Exception as err:
logging.info("unable to authenticate using credentials")
log.info("unable to authenticate using credentials")
self.finish(json.dumps({"success": False, "message": str(err)}))


Expand Down Expand Up @@ -220,6 +244,7 @@ def get(self, path=""):
Takes a path and returns lists of files/objects
and directories/prefixes based on the path.
"""
print("inside S3Handler -> get")
path = path[1:]
# logging.info("GET {}".format(path))

Expand All @@ -238,19 +263,37 @@ def get(self, path=""):
"type": "file",
"content": base64.encodebytes(f.read()).decode("ascii"),
}
print("inside if")
print(result)
else:
raw_result = list(
map(convertS3FStoJupyterFormat, self.s3fs.listdir(path))
)
result = list(filter(lambda x: x["name"] != "", raw_result))
print(f"Raw result -> {raw_result}")
print(f"Length of raw_result -> {len(raw_result)}")

checkBucket = raw_result[0]['path'].split('/')[0]
print(f"Check bucket name for filtering -> {checkBucket}")

if checkBucket != myCustomBucket:
filtered_result = [item for item in raw_result if item['name'] == myCustomBucket]
result = list(filter(lambda x: x["name"] != "", filtered_result))

else:
result = list(filter(lambda x: x["name"] != "", raw_result))

# print(f"Filtered List -> {filtered_result}")
# result = list(filter(lambda x: x["name"] != "", filtered_result))
print("inside else")
print(result)

except S3ResourceNotFoundException as e:
result = {
"error": 404,
"message": "The requested resource could not be found.",
}
except Exception as e:
logging.error("Exception encountered during GET {}: {}".format(path, e))
log.error("Exception encountered during GET {}: {}".format(path, e))
result = {"error": 500, "message": str(e)}

self.finish(json.dumps(result))
Expand All @@ -261,6 +304,7 @@ def put(self, path=""):
Takes a path and returns lists of files/objects
and directories/prefixes based on the path.
"""
print("inside S3Handler -> put")
path = path[1:]

result = {}
Expand Down Expand Up @@ -323,7 +367,7 @@ def put(self, path=""):
"message": "The requested resource could not be found.",
}
except Exception as e:
logging.error(e)
log.error(e)
result = {"error": 500, "message": str(e)}

self.finish(json.dumps(result))
Expand All @@ -334,6 +378,7 @@ def delete(self, path=""):
Takes a path and returns lists of files/objects
and directories/prefixes based on the path.
"""
print("inside S3Handler -> delete")
path = path[1:]
# logging.info("DELETE: {}".format(path))

Expand Down Expand Up @@ -372,7 +417,7 @@ def delete(self, path=""):
self.s3fs.rm(path)

except S3ResourceNotFoundException as e:
logging.error(e)
log.error(e)
result = {
"error": 404,
"message": "The requested resource could not be found.",
Expand All @@ -381,8 +426,8 @@ def delete(self, path=""):
# logging.info("Attempted to delete non-empty directory")
result = {"error": 400, "error": "DIR_NOT_EMPTY"}
except Exception as e:
logging.error("error while deleting")
logging.error(e)
log.error("error while deleting")
log.error(e)
result = {"error": 500, "message": str(e)}

self.finish(json.dumps(result))
Expand Down
54 changes: 54 additions & 0 deletions jupyterlab_s3_browser/logger.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
import datetime
import json
import logging
import os
import sys

class CustomFormatter(logging.Formatter):

grey = "\x1b[38;20m"
darkGrey = '\033[1;30m'
yellow = "\x1b[33;20m"
red = "\x1b[31;20m"
boldRed = "\x1b[31;1m"
green = '\033[0;32m'
blue = '\033[0;34m'
purple = '\033[0;35m'
reset = "\x1b[0m"
if os.getenv("NOTEBOOK", None) or os.getenv("API_ENV", None):
format = '[%(asctime)s] [%(levelname)s] %(message)s'
else:
format = '[%(asctime)s] [%(levelname)s] [%(funcName)s] [%(lineno)d] %(message)s'

FORMATS = {
logging.DEBUG: green + format + reset,
logging.INFO: blue + format + reset,
logging.WARNING: yellow + format + reset,
logging.ERROR: boldRed + format + reset,
logging.CRITICAL: boldRed + format + reset
}

def format(self, record):

log_fmt = self.FORMATS.get(record.levelno)
formatter = logging.Formatter(log_fmt)
return formatter.format(record)

class Logger:

def __init__(self):
logs = logging.getLogger(__name__)
logs.propagate = False
logs.setLevel(10 if os.getenv("VERBOSE") == "True" else 20)
handler_stream = logging.StreamHandler(sys.stdout)
handler_stream.setFormatter(CustomFormatter())
logs.addHandler(handler_stream)
self.DataLog = logs

def getLogger(self):
return self.DataLog




log = Logger().DataLog

0 comments on commit 66ef7e4

Please sign in to comment.