From 77bb3beecc16bd3e905d832ecccc5568863b1ae7 Mon Sep 17 00:00:00 2001
From: chrystinne <chrystinne@gmail.com>
Date: Tue, 24 Oct 2023 00:12:39 -0400
Subject: [PATCH] Configure server access logging for an S3 bucket. It enables
 access logging, directing logs to the Remove unnecessary methods. Raise an
 exception if the S3 credentials are undefined.

---
 .env.example                                  |   2 +
 .../console/manage_published_project.html     |   2 +-
 physionet-django/console/views.py             |  17 +-
 physionet-django/physionet/settings/base.py   |   3 +
 physionet-django/project/cloud/s3.py          | 435 +++---------------
 physionet-django/project/views.py             |   2 -
 6 files changed, 80 insertions(+), 381 deletions(-)

diff --git a/.env.example b/.env.example
index 3ebac56023..2f2a3f46b3 100644
--- a/.env.example
+++ b/.env.example
@@ -67,6 +67,8 @@ AWS_PROFILE=
 AWS_SHARED_CREDENTIALS_FILE=
 # The default bucket name to store projects with an 'OPEN' access policy.
 S3_OPEN_ACCESS_BUCKET=
+# The default bucket name to store logs and metrics related to project usage.
+S3_SERVER_ACCESS_LOGS=
 
 # Datacite
 # Used to assign the DOIs
diff --git a/physionet-django/console/templates/console/manage_published_project.html b/physionet-django/console/templates/console/manage_published_project.html
index 7700cdda73..fda1899ba0 100644
--- a/physionet-django/console/templates/console/manage_published_project.html
+++ b/physionet-django/console/templates/console/manage_published_project.html
@@ -417,7 +417,7 @@ <h5 class="card-title mt-3 mb-1">AWS</h5>
             are set.)
           </p>
         {% elif not project.aws.bucket_name %}
-          <p>Create a bucket on AWS to store the files associated with this project. dsjkbjak {{project.id}} {{project.aws.bucket_name}} {{project.aws.sent_files}}</p>
+          <p>Create a bucket on AWS to store the files associated with this project.</p>
           <form action="" method="post">
             {% csrf_token %}
             <button class="btn btn-primary" name="aws-bucket" value='{{project.slug}}'
diff --git a/physionet-django/console/views.py b/physionet-django/console/views.py
index 0e17841566..a32404cb17 100644
--- a/physionet-django/console/views.py
+++ b/physionet-django/console/views.py
@@ -83,8 +83,6 @@
     get_bucket_name,
     check_s3_bucket_exists,
     update_bucket_policy,
-    get_bucket_name_and_prefix,
-    check_s3_bucket_with_prefix_exists,
     has_s3_credentials,
 )
 
@@ -797,13 +795,13 @@ def send_files_to_aws(pid):
     """
     project = PublishedProject.objects.get(id=pid)
     # Create or get the associated AWS object
-    aws_instance = AWS.objects.get_or_create(project=project)
+    AWS.objects.get_or_create(project=project)
     upload_project_to_S3(project)
-    aws_instance.sent_files = True
-    aws_instance.finished_datetime = timezone.now()
+    project.aws.sent_files = True
+    project.aws.finished_datetime = timezone.now()
     if project.compressed_storage_size:
-        aws_instance.sent_zip = True
-    aws_instance.save()
+        project.aws.sent_zip = True
+    project.aws.save()
 
 
 @associated_task(PublishedProject, "pid", read_only=True)
@@ -902,8 +900,6 @@ def manage_published_project(request, project_slug, version):
         project = PublishedProject.objects.get(slug=project_slug, version=version)
     except PublishedProject.DoesNotExist:
         raise Http404()
-    # s3_bucket_exists = None
-    # s3_bucket_name = None
     user = request.user
     passphrase = ''
     anonymous_url = project.get_anonymous_url()
@@ -911,9 +907,6 @@ def manage_published_project(request, project_slug, version):
     topic_form.set_initial()
     deprecate_form = None if project.deprecated_files else forms.DeprecateFilesForm()
     has_credentials = bool(settings.GOOGLE_APPLICATION_CREDENTIALS)
-    # if has_s3_credentials():
-    #     s3_bucket_exists = check_s3_bucket_with_prefix_exists(project)
-    #     s3_bucket_name = get_bucket_name_and_prefix(project)
     data_access_form = forms.DataAccessForm(project=project)
     contact_form = forms.PublishedProjectContactForm(project=project,
                                                      instance=project.contact)
diff --git a/physionet-django/physionet/settings/base.py b/physionet-django/physionet/settings/base.py
index 134afc5d37..29b8a3bc06 100644
--- a/physionet-django/physionet/settings/base.py
+++ b/physionet-django/physionet/settings/base.py
@@ -237,6 +237,9 @@
 # Bucket name for the S3 bucket containing the open access data
 S3_OPEN_ACCESS_BUCKET = config('S3_OPEN_ACCESS_BUCKET', default=False)
 
+# Bucket name to store logs and metrics related to project usage.
+S3_SERVER_ACCESS_LOGS = config('S3_SERVER_ACCESS_LOGS', default=False)
+
 # Header tags for the AWS lambda function that grants access to S3 storage
 AWS_HEADER_KEY = config('AWS_KEY', default=False)
 AWS_HEADER_VALUE = config('AWS_VALUE', default=False)
diff --git a/physionet-django/project/cloud/s3.py b/physionet-django/project/cloud/s3.py
index 165195d6d9..ef55306511 100644
--- a/physionet-django/project/cloud/s3.py
+++ b/physionet-django/project/cloud/s3.py
@@ -32,6 +32,7 @@ def has_s3_credentials():
     return all([
         settings.AWS_PROFILE,
         settings.S3_OPEN_ACCESS_BUCKET,
+        settings.S3_SERVER_ACCESS_LOGS,
     ])
 
 
@@ -51,8 +52,10 @@ def create_s3_client():
     - The AWS_PROFILE should be defined in the settings module.
 
     Returns:
-        botocore.client.S3 or None: An initialized AWS S3 client
-        object if AWS_PROFILE is not None, otherwise None.
+        botocore.client.S3: An initialized AWS S3 client object.
+
+    Raises:
+        botocore.exceptions.NoCredentialsError: If S3 credentials are undefined.
     """
     if has_s3_credentials():
         session = boto3.Session(
@@ -60,8 +63,7 @@ def create_s3_client():
         )
         s3 = session.client("s3", region_name="us-east-1")
         return s3
-    else:
-        return None
+    raise botocore.exceptions.NoCredentialsError("S3 credentials are undefined.")
 
 
 def create_s3_resource():
@@ -69,8 +71,11 @@ def create_s3_resource():
     Creates and returns an AWS S3 resource if valid credentials are available.
 
     Returns:
-        boto3.resources.base.ServiceResource or None:
-            An S3 resource if credentials are valid, otherwise None.
+        boto3.resources.base.ServiceResource:
+            An S3 resource if credentials are valid.
+
+    Raises:
+        botocore.exceptions.NoCredentialsError: If S3 credentials are undefined.
     """
     if has_s3_credentials():
         session = boto3.Session(
@@ -78,8 +83,7 @@ def create_s3_resource():
         )
         s3 = session.resource("s3", region_name="us-east-1")
         return s3
-    else:
-        return None
+    raise botocore.exceptions.NoCredentialsError("S3 credentials are undefined.")
 
 
 def get_bucket_name(project):
@@ -112,44 +116,8 @@ def get_bucket_name(project):
 
     if project.access_policy == AccessPolicy.OPEN and has_S3_open_data_bucket_name():
         bucket_name = settings.S3_OPEN_ACCESS_BUCKET
-    elif (
-        project.access_policy == AccessPolicy.RESTRICTED
-        or project.access_policy == AccessPolicy.CREDENTIALED
-        or project.access_policy == AccessPolicy.CONTRIBUTOR_REVIEW
-    ):
-        bucket_name = project.slug
-    return bucket_name
-
-
-def get_bucket_name_and_prefix(project):
-    """
-    Determine the S3 bucket name and optional prefix for a
-    given project.
-
-    This function calculates the S3 bucket name based on the
-    project's attributes. If the project has an 'OPEN' access policy
-    and a non-empty prefix defined, the bucket name includes the prefix.
-    For all other access policies or if no prefix is defined, the
-    bucket name consists of the calculated project-specific name only.
-
-    Args:
-        project (project.models.Project): The project for which to
-        determine the S3 bucket name and prefix.
-
-    Returns:
-        str: The S3 bucket name and an optional prefix associated
-        with the project.
-
-    Note:
-    - This function does not create or verify the existence of the
-    actual S3 bucket; it only providesthe calculated bucket name
-    and prefix based on project attributes.
-    """
-    prefix = get_prefix_open_project(project)
-    if project.access_policy == AccessPolicy.OPEN and prefix is not None:
-        bucket_name = get_bucket_name(project) + "/" + prefix
     else:
-        bucket_name = get_bucket_name(project)
+        bucket_name = project.slug
     return bucket_name
 
 
@@ -225,7 +193,7 @@ def get_prefix_open_project(project):
     if project.access_policy != AccessPolicy.OPEN:
         return None
     else:
-        target_prefix = project.slug + "-" + project.version + "/"
+        target_prefix = project.slug + "/"
         matching_prefix = find_matching_prefix(get_all_prefixes(project), target_prefix)
     return matching_prefix
 
@@ -255,26 +223,6 @@ def find_matching_prefix(prefix_list, target_prefix):
     return None
 
 
-def get_list_prefixes(common_prefixes):
-    """
-    Extract and return a list of prefixes from a given list of
-    common prefixes.
-
-    This function takes a list of common prefixes (often obtained
-    from S3 bucket listings) and extracts the 'Prefix' values from
-    each element, returning them as a list.
-
-    Args:
-        common_prefixes (list): A list of common prefixes, typically
-        from an S3 bucket listing.
-
-    Returns:
-        list: A list of extracted prefixes.
-    """
-    perfixes = [prefix.get("Prefix") for prefix in common_prefixes]
-    return perfixes
-
-
 def check_s3_bucket_exists(project):
     """
     Check the existence of an S3 bucket associated with the
@@ -312,40 +260,6 @@ def check_s3_bucket_exists(project):
         return False
 
 
-def check_s3_bucket_with_prefix_exists(project):
-    """
-    Check the existence of an S3 bucket, considering the optional
-    prefix for open projects.
-
-    This function assesses whether an S3 bucket associated with the
-      given project exists.
-    If the project has an 'OPEN' access policy and a non-empty
-    prefix defined, the function checks for the existence of the bucket
-    along with the prefix. For all other access policies
-    or if no prefix is defined, it checks only the existence
-    of the bucket.
-
-    Args:
-        project (project.models.Project): The project for which to
-        check the S3 bucket's existence.
-
-    Returns:
-        bool: True if the S3 bucket (and optional prefix) exists and
-        is accessible, False otherwise.
-
-    Note:
-    - Make sure that AWS credentials (Access Key and Secret Key)
-    are properly configured for this function to work.
-    - The S3 bucket should exist and be accessible based on the
-    project's configuration.
-    """
-    prefix = get_prefix_open_project(project)
-    if project.access_policy == AccessPolicy.OPEN and prefix is None:
-        return False
-    else:
-        return check_s3_bucket_exists(project)
-
-
 def create_s3_bucket(s3, bucket_name):
     """
     Create a new Amazon S3 bucket with the specified name.
@@ -370,6 +284,43 @@ def create_s3_bucket(s3, bucket_name):
     s3.create_bucket(Bucket=bucket_name)
 
 
+def put_bucket_logging(s3, bucket_name, target_bucket, target_prefix):
+    """
+    Configures server access logging for an Amazon S3 bucket.
+
+    Args:
+        s3 (boto3.client): The Amazon S3 client.
+        bucket_name (str): The name of the source bucket.
+        target_bucket (str): The name of the bucket where log
+        files will be stored.
+        target_prefix (str): The prefix for log file names within the
+        target bucket.
+
+    Returns:
+        None
+
+    Note:
+        This method utilizes the `put_bucket_logging` operation to enable
+        server access logging for the specified source bucket, directing
+        logs to the specified target bucket and prefix.
+
+    Example:
+        put_bucket_logging(s3_client, 'source_bucket', 'log_bucket', 'logs/')
+    """
+    logging_config = {
+        'LoggingEnabled': {
+            'TargetBucket': target_bucket,
+            'TargetPrefix': target_prefix,
+        },
+    }
+
+    # Enable bucket logging
+    s3.put_bucket_logging(
+        Bucket=bucket_name,
+        BucketLoggingStatus=logging_config
+    )
+
+
 def send_files_to_s3(folder_path, s3_prefix, bucket_name):
     """
     Upload files from a local folder to an AWS S3 bucket with
@@ -454,34 +405,6 @@ def get_aws_accounts_for_dataset(dataset_name):
     return aws_accounts
 
 
-def get_initial_bucket_policy(bucket_name):
-    """
-    Create an initial bucket policy for an AWS S3 bucket.
-    """
-    bucket_policy = {
-        "Version": "2012-10-17",
-        "Statement": [
-            {
-                "Sid": "DenyListAndGetObject",
-                "Effect": "Deny",
-                "Principal": "*",
-                "Action": ["s3:Get*", "s3:List*"],
-                "Resource": [
-                    f"arn:aws:s3:::{bucket_name}",
-                    f"arn:aws:s3:::{bucket_name}/*",
-                ],
-                "Condition": {
-                    "StringNotLike": {"aws:PrincipalArn": "arn:aws:iam::724665945834:*"}
-                },
-            }
-        ],
-    }
-    # Convert the policy from JSON dict to string
-    bucket_policy_str = json.dumps(bucket_policy)
-
-    return bucket_policy_str
-
-
 def create_bucket_policy(bucket_name, aws_ids, public):
     """
     Generate an initial AWS S3 bucket policy that restricts
@@ -495,6 +418,10 @@ def create_bucket_policy(bucket_name, aws_ids, public):
     Args:
         bucket_name (str): The name of the AWS S3 bucket for
         which to generate the initial policy.
+        aws_ids (list): A list of AWS account IDs and user names
+        allowed to access the bucket.
+        public (bool): True if the bucket should be made public,
+        False otherwise.
 
     Returns:
         str: A JSON-formatted string representing the initial
@@ -503,6 +430,9 @@ def create_bucket_policy(bucket_name, aws_ids, public):
     Note:
     - This initial policy serves as a baseline and can be customized
     further to meet specific access requirements for the bucket.
+    - If 'public' is True, the policy allows public access to the
+    bucket. If 'public' is False, it restricts access to the specified
+    AWS accounts and users.
     """
     user = None
     principal_value = (
@@ -529,19 +459,9 @@ def create_bucket_policy(bucket_name, aws_ids, public):
                 "Resource": [
                     f"arn:aws:s3:::{bucket_name}",
                     f"arn:aws:s3:::{bucket_name}/*",
-                ],
-            },
-            {
-                "Sid": "RestrictDeleteActions",
-                "Effect": "Deny",
-                "Principal": "*",
-                "Action": "s3:Delete*",
-                "Resource": f"arn:aws:s3:::{bucket_name}/*",
-                "Condition": {
-                    "StringNotLike": {"aws:PrincipalArn": "arn:aws:iam::724665945834:*"}
-                },
-            },
-        ],
+                ]
+            }
+        ]
     }
 
     # Convert the policy from JSON dict to string
@@ -653,6 +573,7 @@ def update_bucket_policy(project, bucket_name):
     - Ensure that AWS credentials (Access Key and Secret Key)
     are properly configured for the S3 client used in this function.
     """
+    bucket_policy = ""
     project_name = project.slug + "-" + project.version
     aws_ids = get_aws_accounts_for_dataset(project_name)
     if project.access_policy == AccessPolicy.OPEN:
@@ -666,12 +587,11 @@ def update_bucket_policy(project, bucket_name):
         or project.access_policy == AccessPolicy.RESTRICTED
         or project.access_policy == AccessPolicy.CONTRIBUTOR_REVIEW
     ):
-        if aws_ids == []:
-            bucket_policy = get_initial_bucket_policy(bucket_name)
-        else:
+        if aws_ids != []:
             bucket_policy = create_bucket_policy(bucket_name, aws_ids, False)
 
-    set_bucket_policy(bucket_name, bucket_policy)
+    if bucket_policy not in [None, ""]:
+        set_bucket_policy(bucket_name, bucket_policy)
 
 
 def upload_project_to_S3(project):
@@ -723,6 +643,10 @@ def upload_project_to_S3(project):
     if project.access_policy == AccessPolicy.OPEN:
         s3_prefix = f"{project.slug}/"
     send_files_to_s3(folder_path, s3_prefix, bucket_name)
+    
+    put_bucket_logging(
+        s3, bucket_name, settings.S3_SERVER_ACCESS_LOGS, bucket_name + "/logs/"
+    )
 
 
 def upload_list_of_projects(projects):
@@ -774,224 +698,3 @@ def upload_all_projects():
     published_projects = PublishedProject.objects.all()
     for project in published_projects:
         upload_project_to_S3(project)
-
-
-def empty_s3_bucket(bucket):
-    """
-    Delete all objects (files) within an AWS S3 bucket.
-
-    This function iterates through all objects within the specified
-    'bucket' and deletes each object, effectively emptying
-    the bucket of its contents.
-
-    Args:
-        bucket (boto3.resources.factory.s3.Bucket): The AWS S3
-        bucket to empty.
-
-    Returns:
-        None
-
-    Note:
-    - Ensure that AWS credentials (Access Key and Secret Key) with
-    sufficient permissions are properly configured for the S3
-    client used in this function.
-    """
-    for key in bucket.objects.all():
-        key.delete()
-
-
-def empty_project_from_S3(project):
-    """
-    Empty an AWS S3 bucket associated with a project if
-    AWS_PROFILE is not None.
-
-    This function removes all files within the AWS S3 bucket
-    associated with the specified 'project', effectively emptying
-    the bucket. It leverages the 'empty_s3_bucket' function
-    for this purpose.
-
-    Args:
-        project (project.models.Project): The project for
-        which to empty the associated AWS S3 bucket.
-
-    Returns:
-        None
-
-    Note:
-    - The function ensures that the AWS S3 bucket remains intact
-    but contains no files after execution.
-    - Ensure that AWS credentials (Access Key and Secret Key)
-    with sufficient permissions are properly configured for the
-    S3 client used in this function.
-    """
-    if not check_s3_bucket_exists(project):
-        return
-    s3 = create_s3_resource()
-    bucket_name = get_bucket_name(project)
-    bucket = s3.Bucket(bucket_name)
-    empty_s3_bucket(bucket)
-
-
-def empty_list_of_projects(projects):
-    """
-    Bulk empty AWS S3 buckets associated with a list of projects.
-
-    This function takes a list of 'projects' and empties the
-    AWS S3 buckets associated with each project, effectively
-    removing all files within them. It delegates the emptying
-    process to the 'empty_project_from_S3' function.
-
-    Args:
-        projects (list of project.models.Project): A list of
-        projects for which to empty the associated AWS S3 buckets.
-
-    Returns:
-        None
-
-    Note:
-    - The function ensures that AWS S3 buckets remain intact
-    but contain no files after execution.
-    - Ensure that AWS credentials (Access Key and Secret Key)
-    with sufficient permissions are properly configured for
-    the S3 client used in this function.
-    """
-    for project in projects:
-        empty_project_from_S3(project)
-
-
-def empty_all_projects_from_S3():
-    """
-    Bulk empty all AWS S3 buckets associated with published
-    projects.
-
-    This function iterates through all published projects in the
-    database and empties the AWS S3 buckets associated with each
-    project, effectively removing all files within them. It
-    leverages the 'empty_project_from_S3' function for this purpose.
-
-    Args:
-        None
-
-    Returns:
-        None
-
-    Note:
-    - The function ensures that AWS S3 buckets remain intact
-    but contain no files after execution.
-    - Ensure that AWS credentials (Access Key and Secret Key)
-    with sufficient permissions are properly configured for the
-    S3 client used in this function.
-    """
-
-    # Retrieve all published projects from the database
-    published_projects = PublishedProject.objects.all()
-    # Empty the AWS S3 buckets associated with each published project
-    for project in published_projects:
-        empty_project_from_S3(project)
-
-
-def delete_project_from_S3(project):
-    """
-    Delete a project's files and associated AWS S3 bucket
-    if AWS_PROFILE is not None.
-
-    This function cleans up and removes all files associated
-    with the specified 'project' from an AWS S3 bucket.
-    It then deletes the empty S3 bucket itself.
-
-    Args:
-        project (project.models.Project): The project for which
-        to delete files and the associated bucket.
-
-    Returns:
-        None
-
-    Note:
-    - The function leverages the 'delete_s3_bucket' method
-    to perform the deletion.
-    - Ensure that AWS credentials (Access Key and Secret Key)
-    with sufficient permissions are properly configured for
-    the S3 client used in this function.
-    """
-    if not check_s3_bucket_exists(project):
-        return
-    s3 = create_s3_resource()
-    bucket_name = get_bucket_name(project)
-    bucket = s3.Bucket(bucket_name)
-    # Empty the specified AWS S3 'bucket'
-    # by deleting all objects (files) within it
-    empty_s3_bucket(bucket)
-    # Delete the empty bucket itself
-    bucket.delete()
-
-
-def delete_list_of_projects_from_s3(projects):
-    """ "
-    Bulk delete a list of projects from AWS S3.
-
-    This function deletes a list of projects, including
-    their files and associated AWS S3 buckets.
-    If the list includes open projects, the
-    'delete_project_from_S3' function will be called only
-    once for the first open project encountered.
-
-    Args:
-        projects (list of project.models.Project): A list of
-        projects to delete from AWS S3.
-
-    Returns:
-        None
-
-    Note:
-    - The function leverages the 'delete_project_from_S3' method
-    to perform project deletions.
-    - Ensure that AWS credentials (Access Key and Secret Key)
-    with sufficient permissions are properly configured for
-    the S3 client used in this function.
-    """
-    for project in projects:
-        delete_project_from_S3(project)
-        if project.access_policy == AccessPolicy.OPEN:
-            break
-
-
-def delete_all_projects_from_S3():
-    """
-    Bulk delete all published projects from AWS S3.
-
-    This function deletes all published projects, including
-    their files and associated AWS S3 buckets.
-    It distinguishes between open and non-open projects,
-    calling the 'delete_project_from_S3' function
-    accordingly. Non-open projects, such as 'RESTRICTED',
-    'CREDENTIALED', or 'CONTRIBUTOR_REVIEW', are
-    deleted individually. Open projects are stored in the same
-    S3 bucket, so only one delete operation is needed, which is
-    performed by passing the first open project.
-
-    Args:
-        None
-
-    Returns:
-        None
-
-    Note:
-    - The function leverages the 'delete_project_from_S3' method
-    to perform project deletions.
-    - Ensure that AWS credentials (Access Key and Secret Key)
-    with sufficient permissions are properly configured for the
-    S3 client used in this function.
-    """
-    not_open_published_projects = (
-        PublishedProject.objects.filter(access_policy=AccessPolicy.RESTRICTED)
-        | PublishedProject.objects.filter(access_policy=AccessPolicy.CREDENTIALED)
-        | PublishedProject.objects.filter(access_policy=AccessPolicy.CONTRIBUTOR_REVIEW)
-    )
-    for project in not_open_published_projects:
-        delete_project_from_S3(project)
-    # since all open projects are stored in the same bucket, we only
-    # need to delete the bucket once by passing
-    # the first open project
-    delete_project_from_S3(
-        PublishedProject.objects.filter(access_policy=AccessPolicy.OPEN)[0]
-    )
diff --git a/physionet-django/project/views.py b/physionet-django/project/views.py
index 1e554bc2de..6e01d18d15 100644
--- a/physionet-django/project/views.py
+++ b/physionet-django/project/views.py
@@ -57,8 +57,6 @@
 from user.forms import AssociatedEmailChoiceForm
 from user.models import CloudInformation, CredentialApplication, LegacyCredential, User, Training
 from project.cloud.s3 import (
-    get_bucket_name_and_prefix,
-    check_s3_bucket_with_prefix_exists,
     has_s3_credentials,
 )
 from django.db.models import F, DateTimeField, ExpressionWrapper