diff --git a/scripts/files/fs_s3.py b/scripts/files/fs_s3.py index 42221c129..eedc7b08c 100644 --- a/scripts/files/fs_s3.py +++ b/scripts/files/fs_s3.py @@ -7,6 +7,7 @@ from linz_logger import get_log from scripts.aws.aws_helper import get_session, parse_path +from scripts.files.files_helper import is_json from scripts.logging.time_helper import time_in_ms @@ -163,7 +164,7 @@ def prefix_from_path(path: str) -> str: return path.replace(f"s3://{bucket_name}/", "") -def list_uri(uri: str, s3_client: Optional[boto3.client], extension: Optional[str] = None) -> List[str]: +def list_uri(uri: str, s3_client: Optional[boto3.client]) -> List[str]: """Get the `JSON` files from a s3 path Args: @@ -182,8 +183,8 @@ def list_uri(uri: str, s3_client: Optional[boto3.client], extension: Optional[st for response in response_iterator: for contents_data in response["Contents"]: key = contents_data["Key"] - if extension not in key: - get_log().trace("skipping file not specified extension", file=key, extension=extension, reason="skip") + if not is_json(key): + get_log().trace("skipping file not JSON", file=key, action="collection_from_items", reason="skip") continue files.append(key) get_log().info("Files Listed", number_of_files=len(files))