Skip to content

Commit

Permalink
fix: revert function as no longer used
Browse files Browse the repository at this point in the history
  • Loading branch information
MDavidson17 committed Aug 23, 2023
1 parent 2410930 commit 8e456b6
Showing 1 changed file with 4 additions and 3 deletions.
7 changes: 4 additions & 3 deletions scripts/files/fs_s3.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from linz_logger import get_log

from scripts.aws.aws_helper import get_session, parse_path
from scripts.files.files_helper import is_json
from scripts.logging.time_helper import time_in_ms


Expand Down Expand Up @@ -163,7 +164,7 @@ def prefix_from_path(path: str) -> str:
return path.replace(f"s3://{bucket_name}/", "")


def list_uri(uri: str, s3_client: Optional[boto3.client], extension: Optional[str] = None) -> List[str]:
def list_uri(uri: str, s3_client: Optional[boto3.client]) -> List[str]:
"""Get the `JSON` files from a s3 path
Args:
Expand All @@ -182,8 +183,8 @@ def list_uri(uri: str, s3_client: Optional[boto3.client], extension: Optional[st
for response in response_iterator:
for contents_data in response["Contents"]:
key = contents_data["Key"]
if extension not in key:
get_log().trace("skipping file not specified extension", file=key, extension=extension, reason="skip")
if not is_json(key):
get_log().trace("skipping file not JSON", file=key, action="collection_from_items", reason="skip")
continue
files.append(key)
get_log().info("Files Listed", number_of_files=len(files))
Expand Down

0 comments on commit 8e456b6

Please sign in to comment.