Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

command suite: add new list-projects command #496

Merged
merged 1 commit into from
Oct 8, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions src/bindings/python/fluxacct/accounting/project_subcommands.py
Original file line number Diff line number Diff line change
Expand Up @@ -115,3 +115,32 @@ def delete_project(conn, project):
return warning_stmt

return 0


def list_projects(conn):
Copy link
Member

@wihobbs wihobbs Oct 7, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, sorry for another newbie question but... In my mental map projects were subsidiaries of banks. So does this list only the projects for one bank, or can projects cut across banks altogether?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Projects are created and managed separately from banks. My understanding of projects is that they can be thought of as annotations on a job that multiple users (who might belong to different banks) can make. So, this command will list all registered projects that the flux-accounting database knows about, i.e ones that are added via flux account add-project my_project.

Copy link
Member

@wihobbs wihobbs Oct 8, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you so much @cmoussa1 for all the clarifications. I created more work for you by trying to use these reviews to learn about the interfaces we might use to solve some issues raised by our users at Sandia. I really appreciate the explanations!

"""
List all of the available projects registered in the project_table.
"""
cur = conn.cursor()

cur.execute("SELECT * FROM project_table")
rows = cur.fetchall()

# fetch column names and determine width of each column
col_names = [description[0] for description in cur.description]
col_widths = [
max(len(str(value)) for value in [col] + [row[i] for row in rows])
for i, col in enumerate(col_names)
]

def format_row(row):
return " | ".join(
[f"{str(value).ljust(col_widths[i])}" for i, value in enumerate(row)]
)

header = format_row(col_names)
separator = "-+-".join(["-" * width for width in col_widths])
data_rows = "\n".join([format_row(row) for row in rows])
table = f"{header}\n{separator}\n{data_rows}"

return table
15 changes: 15 additions & 0 deletions src/cmd/flux-account-service.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ def __init__(self, flux_handle, conn):
"view_job_records",
"view_queue",
"view_project",
"list_projects",
]

privileged_endpoints = [
Expand Down Expand Up @@ -513,6 +514,20 @@ def delete_project(self, handle, watcher, msg, arg):
msg, 0, f"a non-OSError exception was caught: {str(exc)}"
)

def list_projects(self, handle, watcher, msg, arg):
try:
val = p.list_projects(self.conn)

payload = {"list_projects": val}

handle.respond(msg, payload)
except KeyError as exc:
handle.respond_error(msg, 0, f"missing key in payload: {exc}")
except Exception as exc:
handle.respond_error(
msg, 0, f"a non-OSError exception was caught: {str(exc)}"
)

def scrub_old_jobs(self, handle, watcher, msg, arg):
try:
val = jobs.scrub_old_jobs(self.conn, msg.payload["num_weeks"])
Expand Down
16 changes: 16 additions & 0 deletions src/cmd/flux-account.py
Original file line number Diff line number Diff line change
Expand Up @@ -519,6 +519,16 @@ def add_delete_project_arg(subparsers):
)


def add_list_projects_arg(subparsers):
subparser_list_projects = subparsers.add_parser(
"list-projects",
help="list all registered projects",
formatter_class=flux.util.help_formatter(),
)

subparser_list_projects.set_defaults(func="list_projects")


def add_scrub_job_records_arg(subparsers):
subparser = subparsers.add_parser(
"scrub-old-jobs",
Expand Down Expand Up @@ -628,6 +638,7 @@ def add_arguments_to_parser(parser, subparsers):
add_add_project_arg(subparsers)
add_view_project_arg(subparsers)
add_delete_project_arg(subparsers)
add_list_projects_arg(subparsers)
add_scrub_job_records_arg(subparsers)
add_export_db_arg(subparsers)
add_pop_db_arg(subparsers)
Expand Down Expand Up @@ -805,6 +816,11 @@ def select_accounting_function(args, output_file, parser):
"project": args.project,
}
return_val = flux.Flux().rpc("accounting.delete_project", data).get()
elif args.func == "list_projects":
data = {
"path": args.path,
}
return_val = flux.Flux().rpc("accounting.list_projects", data).get()
elif args.func == "scrub_old_jobs":
data = {
"path": args.path,
Expand Down
22 changes: 22 additions & 0 deletions t/t1025-flux-account-projects.t
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,16 @@ test_expect_success 'add some queues to the DB' '
flux account add-queue special --priority=99999
'

test_expect_success 'list contents of project_table before adding projects' '
flux account list-projects > project_table.test &&
cat <<-EOF >project_table.expected
project_id | project | usage
-----------+---------+------
1 | * | 0.0
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Usage is reported in what unit?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Usage in accounting is calculated as a product of nnodes and t_elapsed (which is defined as t_inactive - t_run). I have some more detailed explanations and examples here.

EOF
grep -f project_table.expected project_table.test
'

test_expect_success 'add some projects to the project_table' '
flux account add-project project_1 &&
flux account add-project project_2 &&
Expand Down Expand Up @@ -131,6 +141,18 @@ test_expect_success 'reset the projects list for an association' '
grep "\"projects\": \"*\"" user5018.json
'

test_expect_success 'list all of the projects currently registered in project_table' '
flux account list-projects > project_table.test &&
cat <<-EOF >project_table.expected
project_id | project | usage
-----------+-----------+------
1 | * | 0.0
3 | project_2 | 0.0
4 | project_3 | 0.0
EOF
grep -f project_table.expected project_table.test
'

test_expect_success 'remove flux-accounting DB' '
rm $(pwd)/FluxAccountingTest.db
'
Expand Down
Loading