diff --git a/captn/captn_agents/backend/teams/_gbb_google_sheets_team.py b/captn/captn_agents/backend/teams/_gbb_google_sheets_team.py index 14f02c66..b65a93f7 100644 --- a/captn/captn_agents/backend/teams/_gbb_google_sheets_team.py +++ b/captn/captn_agents/backend/teams/_gbb_google_sheets_team.py @@ -184,10 +184,8 @@ def _commands(self) -> str: @classmethod def get_capabilities(cls) -> str: - return "Able to read and edit Google Sheets." + return "Able to read Google Sheets and create NEW Google Ads campaigns." @classmethod def get_brief_template(cls) -> str: - return ( - "The client wants to create new campaigns using a Google Sheets template." - ) + return "The client wants to create new Google Ads campaigns from the Google Sheets template." diff --git a/captn/captn_agents/backend/teams/_gbb_initial_team.py b/captn/captn_agents/backend/teams/_gbb_initial_team.py index 3b79022c..828b922b 100644 --- a/captn/captn_agents/backend/teams/_gbb_initial_team.py +++ b/captn/captn_agents/backend/teams/_gbb_initial_team.py @@ -60,7 +60,7 @@ def _guidelines(self) -> str: If you fail to choose the appropriate team, you will be penalized! 3. Here is a list of teams you can choose from after you determine which one is the most appropriate for the task: -{self.construct_team_names_and_descriptions_message(use_only_team_names={"gbb_google_sheets_team"})} +{self.construct_team_names_and_descriptions_message(use_only_team_names={"gbb_google_sheets_team", "gbb_page_feed_team"})} Guidelines SUMMARY: - Write a detailed step-by-step plan diff --git a/captn/captn_agents/backend/teams/_gbb_page_feed_team.py b/captn/captn_agents/backend/teams/_gbb_page_feed_team.py index 08c64c97..61c4256d 100644 --- a/captn/captn_agents/backend/teams/_gbb_page_feed_team.py +++ b/captn/captn_agents/backend/teams/_gbb_page_feed_team.py @@ -4,9 +4,10 @@ from ..tools._gbb_page_feed_team_tools import create_page_feed_team_toolbox from ._gbb_google_sheets_team import GOOGLE_SHEETS_OPENAPI_URL, GBBGoogleSheetsTeam from ._shared_prompts import REPLY_TO_CLIENT_COMMAND +from ._team import Team -# @Team.register_team("gbb_page_feed_team") +@Team.register_team("gbb_page_feed_team") class GBBPageFeedTeam(GBBGoogleSheetsTeam): def __init__( self, @@ -71,8 +72,12 @@ def _guidelines(self) -> str: - mandatory input parameters: user_id, spreadsheet_id 6. In the spreadsheet with page feeds, you must find the title of the sheet with page feeds (by using 'get_all_sheet_titles_get_all_sheet_titles_get'). - If there are multiple sheets within the spreadsheet, ask the client to choose the correct sheet. -7. Once you have the correct sheet title, you must validate the data in the page feed sheet by using 'validate_page_feed_data' function. +7. Once you have the correct sheet title, you must validate the data in the page feed sheet by using 'get_and_validate_page_feed_data' function. 8. If the data is correct, you must update the page feeds in Google Ads by using 'update_page_feeds' function. +9. Before EACH update_page_feeds call also repeat the get_and_validate_page_feed_data call to ensure the data is still correct. +- If the user asks you to repeat the process, it might be because the data has changed, so ALWAYS validate the data before updating. +10. Once you have updated the page feeds, you must inform the client which page feeds have been updated and you must list ALL the changes which the client needs to do manually. +- Do not write few changes and "etc." or "[Additional URLs]". You MUST list ALL the changes (both complete and the ones that need to be done manually). ALL ENDPOINT PARAMETERS ARE MANDATORY (even if the documentation says they are optional). @@ -115,7 +120,7 @@ def _commands(self) -> str: If you want to refresh google sheets token or change google sheets use 'get_login_url_login_get' with 'force_new_login' parameter set to True. 4. Only Google_ads_expert has access to the following commands: -- 'validate_page_feed_data': +- 'get_and_validate_page_feed_data': parameters: template_spreadsheet_id, page_feed_spreadsheet_id, page_feed_sheet_title - 'update_page_feeds': parameters: customer_id, login_customer_id @@ -124,7 +129,7 @@ def _commands(self) -> str: @classmethod def get_capabilities(cls) -> str: - return "Able update Google Ads Page Feeds." + return "Able to update Google Ads Page Feeds." @classmethod def get_brief_template(cls) -> str: diff --git a/captn/captn_agents/backend/tools/_gbb_google_sheets_team_tools.py b/captn/captn_agents/backend/tools/_gbb_google_sheets_team_tools.py index 0e4c79b1..f68cf763 100644 --- a/captn/captn_agents/backend/tools/_gbb_google_sheets_team_tools.py +++ b/captn/captn_agents/backend/tools/_gbb_google_sheets_team_tools.py @@ -142,7 +142,7 @@ class GoogleSheetsTeamContext(Context): ] -def _check_mandatory_columns( +def check_mandatory_columns( df: pd.DataFrame, mandatory_columns: List[str], table_title: str ) -> str: missing_columns = [col for col in mandatory_columns if col not in df.columns] @@ -169,7 +169,7 @@ def _validat_and_convert_to_df( values.values[1:], columns=values.values[0], ) - mandatory_columns_error_msg = _check_mandatory_columns(df, mandatory_columns, title) + mandatory_columns_error_msg = check_mandatory_columns(df, mandatory_columns, title) return df, mandatory_columns_error_msg @@ -778,6 +778,10 @@ def _update_campaign_with_additional_settings( ) +def get_time() -> str: + return datetime.now(ZAGREB_TIMEZONE).strftime("%Y-%m-%d %H:%M:%S") + + def _setup_campaign( customer_id: str, login_customer_id: str, @@ -895,13 +899,15 @@ def _setup_campaign( ) campaign["ad_groups"][row["ad group name"]] = ad_group_ad.ad_group_id - message = f"[{datetime.now(ZAGREB_TIMEZONE).strftime('%Y-%m-%d %H:%M:%S')}] Created campaign: {campaign_name}" + message = f"[{get_time()}] Created campaign: {campaign_name}" iostream.print(colored(message, "green"), flush=True) return True, None except Exception as e: error_msg = str(e) - message = f"[{datetime.now(ZAGREB_TIMEZONE).strftime('%Y-%m-%d %H:%M:%S')}] Failed to create campaign: {campaign_name}: {error_msg}" + message = ( + f"[{get_time()}] Failed to create campaign: {campaign_name}: {error_msg}" + ) iostream.print(colored(message, "red"), flush=True) try: diff --git a/captn/captn_agents/backend/tools/_gbb_page_feed_team_tools.py b/captn/captn_agents/backend/tools/_gbb_page_feed_team_tools.py index c053dbd2..dfd31142 100644 --- a/captn/captn_agents/backend/tools/_gbb_page_feed_team_tools.py +++ b/captn/captn_agents/backend/tools/_gbb_page_feed_team_tools.py @@ -1,8 +1,21 @@ +import json +import traceback from dataclasses import dataclass from typing import Annotated, Any, Dict, Optional import pandas as pd +from autogen.formatting_utils import colored +from autogen.io.base import IOStream +from google_ads.model import AddPageFeedItems # , RemoveResource + +from ....google_ads.client import ( + check_for_client_approval, + execute_query, + google_ads_api_call, + google_ads_create_update, # noqa + google_ads_post_or_get, +) from ..toolboxes import Toolbox from ._functions import ( REPLY_TO_CLIENT_DESCRIPTION, @@ -14,13 +27,17 @@ CHANGE_GOOGLE_ADS_ACCOUNT_DESCRIPTION, GoogleSheetsTeamContext, GoogleSheetValues, + check_mandatory_columns, get_sheet_data, + get_time, ) from ._google_ads_team_tools import ( change_google_account, ) -VALIDATE_PAGE_FEED_DATA_DESCRIPTION = "Validate page feed data." +GET_AND_VALIDATE_PAGE_FEED_DATA_DESCRIPTION = """Get and validate page feed data. +Use this function before EACH update_page_feeds call to validate the page feed data. +It is mandatory because the data can change between calls. So it is important to validate the data before each update.""" def _get_sheet_data_and_return_df( @@ -78,7 +95,26 @@ class PageFeedTeamContext(GoogleSheetsTeamContext): page_feeds_df: Optional[pd.DataFrame] = None -def validate_page_feed_data( +ACCOUNTS_TITLE = "Accounts" +ACCOUNTS_TEMPLATE_MANDATORY_COLUMNS = [ + "Customer Id", + "Name", + "Manager Customer Id", +] + +PAGE_FEEDS_TEMPLATE_TITLE = "Page Feeds" +PAGE_FEEDS_TEMPLATE_MANDATORY_COLUMNS = [ + "Customer Id", + "Name", +] + +PAGE_FEEDS_MANDATORY_COLUMNS = [ + "Page URL", + "Custom Label", +] + + +def get_and_validate_page_feed_data( template_spreadsheet_id: Annotated[str, "Template spreadsheet id"], page_feed_spreadsheet_id: Annotated[str, "Page feed spreadsheet id"], page_feed_sheet_title: Annotated[ @@ -90,17 +126,41 @@ def validate_page_feed_data( user_id=context.user_id, base_url=context.google_sheets_api_url, spreadsheet_id=template_spreadsheet_id, - title="Accounts", + title=ACCOUNTS_TITLE, ) - for col in ["Manager Customer Id", "Customer Id"]: - account_templ_df[col] = account_templ_df[col].str.replace("-", "") - page_feeds_template_df = _get_sheet_data_and_return_df( user_id=context.user_id, base_url=context.google_sheets_api_url, spreadsheet_id=template_spreadsheet_id, - title="Page Feeds", + title=PAGE_FEEDS_TEMPLATE_TITLE, + ) + page_feeds_df = _get_sheet_data_and_return_df( + user_id=context.user_id, + base_url=context.google_sheets_api_url, + spreadsheet_id=page_feed_spreadsheet_id, + title=page_feed_sheet_title, ) + + error_msg = "" + for df, mandatory_columns, table_title in [ + (account_templ_df, ACCOUNTS_TEMPLATE_MANDATORY_COLUMNS, ACCOUNTS_TITLE), + ( + page_feeds_template_df, + PAGE_FEEDS_TEMPLATE_MANDATORY_COLUMNS, + PAGE_FEEDS_TEMPLATE_TITLE, + ), + (page_feeds_df, PAGE_FEEDS_MANDATORY_COLUMNS, page_feed_sheet_title), + ]: + error_msg += check_mandatory_columns( + df=df, mandatory_columns=mandatory_columns, table_title=table_title + ) + + if error_msg: + return error_msg + + for col in ["Manager Customer Id", "Customer Id"]: + account_templ_df[col] = account_templ_df[col].str.replace("-", "") + page_feeds_template_df["Customer Id"] = page_feeds_template_df[ "Customer Id" ].str.replace("-", "") @@ -113,13 +173,6 @@ def validate_page_feed_data( suffixes=(" Page Feed", " Account"), ) - page_feeds_df = _get_sheet_data_and_return_df( - user_id=context.user_id, - base_url=context.google_sheets_api_url, - spreadsheet_id=page_feed_spreadsheet_id, - title=page_feed_sheet_title, - ) - page_feeds_and_accounts_templ_df = _get_relevant_page_feeds_and_accounts( page_feeds_and_accounts_templ_df, page_feeds_df ) @@ -149,14 +202,377 @@ def validate_page_feed_data( UPDATE_PAGE_FEED_DESCRIPTION = "Update Google Ads Page Feeds." +def _get_page_feed_asset_sets( + user_id: int, + conv_id: int, + customer_id: str, + login_customer_id: str, +) -> Dict[str, Dict[str, str]]: + page_feed_asset_sets: Dict[str, Dict[str, str]] = {} + query = """SELECT +asset_set.id, +asset_set.name, +asset_set.resource_name +FROM campaign_asset_set +WHERE +campaign.advertising_channel_type = 'PERFORMANCE_MAX' +AND asset_set.type = 'PAGE_FEED' +AND asset_set.status = 'ENABLED' +AND campaign_asset_set.status = 'ENABLED' +AND campaign.status != 'REMOVED' +""" + response = execute_query( + user_id=user_id, + conv_id=conv_id, + customer_ids=[customer_id], + login_customer_id=login_customer_id, + query=query, + ) + if isinstance(response, dict): + raise ValueError(response) + + response_json = json.loads(response.replace("'", '"')) + + if customer_id not in response_json: + return page_feed_asset_sets + + for row in response_json[customer_id]: + asset_set = row["assetSet"] + page_feed_asset_sets[asset_set["name"]] = { + "id": asset_set["id"], + "resourceName": asset_set["resourceName"], + } + + return page_feed_asset_sets + + +def _get_page_feed_items( + user_id: int, + conv_id: int, + customer_id: str, + login_customer_id: str, + asset_set_resource_name: str, +) -> pd.DataFrame: + query = f""" +SELECT + asset.id, + asset.name, + asset.type, + asset.page_feed_asset.page_url, + asset.page_feed_asset.labels +FROM + asset_set_asset +WHERE + asset.type = 'PAGE_FEED' + AND asset_set_asset.asset_set = '{asset_set_resource_name}' + AND asset_set_asset.status != 'REMOVED' +""" # nosec: [B608] + + response = execute_query( + user_id=user_id, + conv_id=conv_id, + customer_ids=[customer_id], + login_customer_id=login_customer_id, + query=query, + ) + + if isinstance(response, dict): + raise ValueError(response) + + response_json = json.loads(response.replace("'", '"')) + + page_urls_and_labels_df = pd.DataFrame(columns=["Id", "Page URL", "Custom Label"]) + for asset in response_json[customer_id]: + id = asset["asset"]["id"] + url = asset["asset"]["pageFeedAsset"]["pageUrl"].strip() + + if "labels" in asset["asset"]["pageFeedAsset"]: + labels_list = asset["asset"]["pageFeedAsset"]["labels"] + labels = "; ".join(labels_list) + else: + labels = None + page_urls_and_labels_df = pd.concat( + [ + page_urls_and_labels_df, + pd.DataFrame([{"Id": id, "Page URL": url, "Custom Label": labels}]), + ], + ignore_index=True, + ) + + return page_urls_and_labels_df + + +def _add_missing_page_urls( + user_id: int, + conv_id: int, + customer_id: str, + login_customer_id: str, + page_feed_asset_set: Dict[str, str], + missing_page_urls: pd.DataFrame, +) -> str: + url_and_labels = missing_page_urls.set_index("Page URL")["Custom Label"].to_dict() + for key, value in url_and_labels.items(): + if value: + url_and_labels[key] = [ + label.strip() for label in value.split(";") if label.strip() + ] + else: + url_and_labels[key] = None + add_model = AddPageFeedItems( + login_customer_id=login_customer_id, + customer_id=customer_id, + asset_set_resource_name=page_feed_asset_set["resourceName"], + urls_and_labels=url_and_labels, + ) + + try: + response = google_ads_api_call( + function=google_ads_post_or_get, # type: ignore[arg-type] + kwargs={ + "user_id": user_id, + "conv_id": conv_id, + "model": add_model, + "recommended_modifications_and_answer_list": [], + "already_checked_clients_approval": True, + "endpoint": "/add-items-to-page-feed", + }, + ) + except Exception as e: + return f"Failed to add page feed items:\n{url_and_labels}\n\n{str(e)}\n\n" + if isinstance(response, dict): + raise ValueError(response) + urls_to_string = "\n".join(url_and_labels.keys()) + return f"Added page feed items:\n{urls_to_string}\n\n" + + +def _remove_extra_page_urls( + user_id: int, + conv_id: int, + customer_id: str, + login_customer_id: str, + page_feed_asset_set: Dict[str, str], + extra_page_urls: pd.DataFrame, + iostream: IOStream, +) -> str: + return_value = "The following page feed items should be removed by you manually:\n" + iostream.print(colored(f"[{get_time()}] " + return_value, "green"), flush=True) + for row in extra_page_urls.iterrows(): + id = row[1]["Id"] + # remove_model = RemoveResource( + # customer_id=customer_id, + # parent_id=page_feed_asset_set["id"], + # resource_id=id, + # resource_type="asset_set_asset", + # ) + # try: + # response = google_ads_api_call( + # function=google_ads_create_update, # type: ignore[arg-type] + # kwargs={ + # "user_id": user_id, + # "conv_id": conv_id, + # "recommended_modifications_and_answer_list": [], + # "already_checked_clients_approval": True, + # "ad": remove_model, + # "login_customer_id": login_customer_id, + # "endpoint": "/remove-google-ads-resource", + # }, + # ) + # except Exception as e: + # return f"Failed to remove page feed item with id {id} - {row[1]['Page URL']}:\n{str(e)}\n\n" + + response = "REMOVE THIS LINE ONCE THE ABOVE CODE IS UNCOMMENTED" + if isinstance(response, dict): + msg = f"Failed to remove page feed item with id {id} - {row[1]['Page URL']}:\n" + return_value += msg + str(response) + "\n\n" + iostream.print( + colored(f"[{get_time()}] " + msg + str(response), "red"), flush=True + ) + else: + msg = f"- {row[1]['Page URL']}" + return_value += msg + "\n" + iostream.print(colored(f"[{get_time()}] " + msg, "green"), flush=True) + + return_value += "\n" + return return_value + + +def _sync_page_feed_asset_set( + user_id: int, + conv_id: int, + customer_id: str, + login_customer_id: str, + page_feeds_and_accounts_templ_df: pd.DataFrame, + page_feeds_df: pd.DataFrame, + page_feed_asset_set_name: str, + page_feed_asset_set: Dict[str, str], + iostream: IOStream, +) -> str: + page_feed_rows = page_feeds_and_accounts_templ_df[ + page_feeds_and_accounts_templ_df["Name Page Feed"] == page_feed_asset_set_name + ] + if page_feed_rows.empty: + msg = f"Skipping page feed '**{page_feed_asset_set_name}**' (not found in the page feed template).\n\n" + iostream.print(colored(f"[{get_time()}] " + msg, "yellow"), flush=True) + return msg + + elif page_feed_rows["Customer Id"].nunique() > 1: + msg = f"Page feed template has multiple values for the same page feed '**{page_feed_asset_set_name}**'!\n\n" + iostream.print(colored(f"[{get_time()}] " + msg, "red"), flush=True) + return msg + + page_feed_template_row = page_feed_rows.iloc[0] + custom_labels_values = [ + page_feed_template_row[col] + for col in page_feed_rows.columns + if col.startswith("Custom Label") + ] + page_feed_rows = page_feeds_df[ + page_feeds_df["Custom Label"].isin(custom_labels_values) + ] + + if page_feed_rows.empty: + msg = f"No page feed data found for page feed '**{page_feed_asset_set_name}**'\n\n" + iostream.print(colored(f"[{get_time()}] " + msg, "yellow"), flush=True) + return msg + + page_feed_url_and_label_df = page_feed_rows[["Page URL", "Custom Label"]] + + gads_page_urls_and_labels_df = _get_page_feed_items( + user_id=user_id, + conv_id=conv_id, + customer_id=customer_id, + login_customer_id=login_customer_id, + asset_set_resource_name=page_feed_asset_set["resourceName"], + ) + + for df in [page_feed_url_and_label_df, gads_page_urls_and_labels_df]: + df.loc[:, "Page URL"] = df["Page URL"].str.rstrip("/") + + missing_page_urls = page_feed_url_and_label_df[ + ~page_feed_url_and_label_df["Page URL"].isin( + gads_page_urls_and_labels_df["Page URL"] + ) + ] + extra_page_urls = gads_page_urls_and_labels_df[ + ~gads_page_urls_and_labels_df["Page URL"].isin( + page_feed_url_and_label_df["Page URL"] + ) + ] + + if missing_page_urls.empty and extra_page_urls.empty: + msg = f"No changes needed for page feed '**{page_feed_asset_set_name}**'\n\n" + iostream.print(colored(f"[{get_time()}] " + msg, "green"), flush=True) + return msg + + return_value = f"Page feed '**{page_feed_asset_set_name}**' changes:\n" + iostream.print( + colored(f"[{get_time()}] " + return_value.strip(), "green"), flush=True + ) + if not missing_page_urls.empty: + msg = _add_missing_page_urls( + user_id=user_id, + conv_id=conv_id, + customer_id=customer_id, + login_customer_id=login_customer_id, + page_feed_asset_set=page_feed_asset_set, + missing_page_urls=missing_page_urls, + ) + iostream.print(colored(f"[{get_time()}] " + msg, "green"), flush=True) + return_value += msg + + if not extra_page_urls.empty: + return_value += _remove_extra_page_urls( + user_id=user_id, + conv_id=conv_id, + customer_id=customer_id, + login_customer_id=login_customer_id, + page_feed_asset_set=page_feed_asset_set, + extra_page_urls=extra_page_urls, + iostream=iostream, + ) + + return return_value + + +def _sync_page_feed_asset_sets( + user_id: int, + conv_id: int, + customer_id: str, + login_customer_id: str, + page_feeds_and_accounts_templ_df: pd.DataFrame, + page_feeds_df: pd.DataFrame, + page_feed_asset_sets: Dict[str, Dict[str, str]], + context: PageFeedTeamContext, +) -> str: + iostream = IOStream.get_default() + return_value = "" + for page_feed_asset_set_name, page_feed_asset_set in page_feed_asset_sets.items(): + return_value += _sync_page_feed_asset_set( + user_id=user_id, + conv_id=conv_id, + customer_id=customer_id, + login_customer_id=login_customer_id, + page_feeds_and_accounts_templ_df=page_feeds_and_accounts_templ_df, + page_feeds_df=page_feeds_df, + page_feed_asset_set_name=page_feed_asset_set_name, + page_feed_asset_set=page_feed_asset_set, + iostream=iostream, + ) + + return_value = reply_to_client( + message=return_value, + completed=False, + context=context, + ) + + return return_value + + def update_page_feeds( customer_id: Annotated[str, "Customer Id to update"], login_customer_id: Annotated[str, "Login Customer Id (Manager Account)"], context: PageFeedTeamContext, ) -> str: + error_msg = check_for_client_approval( + modification_function_parameters={ + "customer_id": customer_id, + "login_customer_id": login_customer_id, + }, + recommended_modifications_and_answer_list=context.recommended_modifications_and_answer_list, + ) + if error_msg: + raise ValueError(error_msg) + if context.page_feeds_and_accounts_templ_df is None: - return f"Please validate the page feed data first by running the '{validate_page_feed_data.__name__}' function." - return "All page feeds have been updated." + return f"Please (re)validate the page feed data first by running the '{get_and_validate_page_feed_data.__name__}' function." + + try: + page_feed_asset_sets = _get_page_feed_asset_sets( + user_id=context.user_id, + conv_id=context.conv_id, + customer_id=customer_id, + login_customer_id=login_customer_id, + ) + if len(page_feed_asset_sets) == 0: + return f"No page feeds found for customer id {customer_id}." + + return _sync_page_feed_asset_sets( + user_id=context.user_id, + conv_id=context.conv_id, + customer_id=customer_id, + login_customer_id=login_customer_id, + page_feeds_and_accounts_templ_df=context.page_feeds_and_accounts_templ_df, + page_feeds_df=context.page_feeds_df, + page_feed_asset_sets=page_feed_asset_sets, + context=context, + ) + except Exception as e: + traceback.print_stack() + traceback.print_exc() + raise e + finally: + context.page_feeds_and_accounts_templ_df = None + context.page_feeds_df = None def create_page_feed_team_toolbox( @@ -181,7 +597,9 @@ def create_page_feed_team_toolbox( toolbox.add_function( description=ask_client_for_permission_description, )(ask_client_for_permission) - toolbox.add_function(VALIDATE_PAGE_FEED_DATA_DESCRIPTION)(validate_page_feed_data) + toolbox.add_function(GET_AND_VALIDATE_PAGE_FEED_DATA_DESCRIPTION)( + get_and_validate_page_feed_data + ) toolbox.add_function(UPDATE_PAGE_FEED_DESCRIPTION)(update_page_feeds) toolbox.add_function( diff --git a/captn/google_ads/client.py b/captn/google_ads/client.py index 8b131786..8f6cfeed 100644 --- a/captn/google_ads/client.py +++ b/captn/google_ads/client.py @@ -1,10 +1,11 @@ import json from os import environ -from typing import Any, Dict, List, Literal, Optional, Tuple, Union +from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Union from pydantic import BaseModel from requests import get as requests_get from requests import post as requests_post +from tenacity import retry, stop_after_attempt, wait_exponential BASE_URL = environ.get("CAPTN_BACKEND_URL", "http://localhost:9000") ALREADY_AUTHENTICATED = "User is already authenticated" @@ -388,3 +389,11 @@ def google_ads_post_or_get( response_dict: Union[Dict[str, Any], str] = response.json() return response_dict + + +@retry(stop=stop_after_attempt(3), wait=wait_exponential(min=4, max=15)) +def google_ads_api_call( + function: Callable[[Any], Union[Dict[str, Any], str]], + **kwargs: Any, +) -> Union[Dict[str, Any], str]: + return function(**kwargs) # type: ignore[call-arg] diff --git a/google_ads/application.py b/google_ads/application.py index 511b2e7b..d698bdba 100644 --- a/google_ads/application.py +++ b/google_ads/application.py @@ -34,7 +34,6 @@ ExistingCampaignSitelinks, GeoTargetCriterion, NewCampaignSitelinks, - PageFeedItems, RemoveResource, ) @@ -2012,73 +2011,40 @@ async def add_items_to_page_feed( ) operations = [] - # Creates one asset per URL. - for url, label in model.urls_and_labels.items(): - # Creates an asset operation and adds it to the list of operations. - operation = client.get_type("AssetOperation") - asset = operation.create - page_feed_asset = asset.page_feed_asset - page_feed_asset.page_url = url - # Recommended: adds labels to the asset. These labels can be used later - # in ad group targeting to restrict the set of pages that can serve. - if label: - page_feed_asset.labels.append(label) - operations.append(operation) - - # Issues a mutate request to add the assets and prints its information. - asset_service = client.get_service("AssetService") - response = asset_service.mutate_assets( - customer_id=model.customer_id, operations=operations - ) - - resource_names = [] - return_text = "" - for result in response.results: - resource_name = result.resource_name - return_text += f"Created an asset with resource name: '{resource_name}'\n" - resource_names.append(resource_name) - - return _add_assets_to_asset_set( - client=client, - customer_id=model.customer_id, - asset_resource_names=resource_names, - asset_set_resource_name=model.asset_set_resource_name, - ) - - -@router.get("/list-page-feed-items") -async def list_page_feed_items( - user_id: int, - model: PageFeedItems, -) -> Dict[str, Any]: - # client = await _get_client( - # user_id=user_id, login_customer_id=model.login_customer_id - # ) - query = f""" -SELECT -# asset_set_asset.asset, -# asset_set_asset.asset_set, - asset.id, - asset.name, - asset.type, - asset.page_feed_asset.page_url -FROM - asset_set_asset -WHERE - asset.type = 'PAGE_FEED' - AND asset_set_asset.asset_set = '{model.asset_set_resource_name}' - AND asset_set_asset.status != 'REMOVED' -""" # nosec: [B608] - - page_feed_assets_response = await search( - user_id=user_id, - customer_ids=[model.customer_id], - query=query, - login_customer_id=model.login_customer_id, - ) + try: + # Creates one asset per URL. + for url, labels in model.urls_and_labels.items(): + # Creates an asset operation and adds it to the list of operations. + operation = client.get_type("AssetOperation") + asset = operation.create + page_feed_asset = asset.page_feed_asset + page_feed_asset.page_url = url + # Recommended: adds labels to the asset. These labels can be used later + # in ad group targeting to restrict the set of pages that can serve. + if labels: + page_feed_asset.labels.extend(labels) + operations.append(operation) + + # Issues a mutate request to add the assets and prints its information. + asset_service = client.get_service("AssetService") + response = asset_service.mutate_assets( + customer_id=model.customer_id, operations=operations + ) - # page_urls = [] - # for asset in page_feed_assets_response[model.customer_id]: - # page_urls.append(asset["asset"]["pageFeedAsset"]["pageUrl"]) + resource_names = [] + return_text = "" + for result in response.results: + resource_name = result.resource_name + return_text += f"Created an asset with resource name: '{resource_name}'\n" + resource_names.append(resource_name) - return page_feed_assets_response + return _add_assets_to_asset_set( + client=client, + customer_id=model.customer_id, + asset_resource_names=resource_names, + asset_set_resource_name=model.asset_set_resource_name, + ) + except Exception as e: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e) + ) from e diff --git a/google_ads/model.py b/google_ads/model.py index 44eedb71..3c2149d4 100644 --- a/google_ads/model.py +++ b/google_ads/model.py @@ -225,4 +225,4 @@ class PageFeedItems(BaseModel): class AddPageFeedItems(PageFeedItems): - urls_and_labels: Dict[str, Optional[str]] + urls_and_labels: Dict[str, Optional[List[str]]] diff --git a/migrations/20240909212345_sync_page_feed_smart_suggestion/migration.sql b/migrations/20240909212345_sync_page_feed_smart_suggestion/migration.sql new file mode 100644 index 00000000..8d5aef8d --- /dev/null +++ b/migrations/20240909212345_sync_page_feed_smart_suggestion/migration.sql @@ -0,0 +1,4 @@ +-- Update the smart_suggestions array for the gbb_initial_team +UPDATE "InitialTeam" +SET smart_suggestions = array_append(smart_suggestions, 'Sync PMax page feed data') +WHERE name = 'gbb_initial_team'; diff --git a/pyproject.toml b/pyproject.toml index 742f46e3..84f9a164 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,16 +58,16 @@ lint = [ "mypy==1.11.2", "black==24.8.0", "isort>=5", - "ruff==0.6.4", + "ruff==0.6.9", "pyupgrade-directories", "bandit==1.7.10", "semgrep==1.78.0", - "pre-commit==3.8.0", + "pre-commit==4.0.1", "detect-secrets==1.5.0", ] test-core = [ - "coverage[toml]==7.6.1", + "coverage[toml]==7.6.3", "pytest==8.3.3", "pytest-asyncio>=0.23.6", "dirty-equals==0.8.0", @@ -94,11 +94,11 @@ agents = [ "prisma==0.13.1", "google-ads==25.0.0", "httpx==0.27.0", - "uvicorn==0.31.0", + "uvicorn==0.31.1", "python-dotenv==1.0.1", "pyautogen[websurfer,websockets,anthropic,together]==0.2.35", "pandas>=2.1", - "fastcore==1.7.4", + "fastcore==1.7.11", "pydantic==2.9.2", "tenacity==9.0.0", "prometheus-client==0.21.0", @@ -108,9 +108,9 @@ agents = [ "opentelemetry-exporter-otlp==1.27.0", "openpyxl==3.1.5", "aiofiles==24.1.0", - "fastapi==0.114.2", + "fastapi==0.115.2", "asyncer==0.0.8", - "fastagency[openapi]==0.2.1", + "fastagency[openapi]==0.2.5", "markdownify==0.13.1", "python-multipart==0.0.12", "flaml==2.2.0", diff --git a/tests/ci/captn/captn_agents/backend/teams/test_gbb_page_feed_team.py b/tests/ci/captn/captn_agents/backend/teams/test_gbb_page_feed_team.py index a0cb2851..fc4788f8 100644 --- a/tests/ci/captn/captn_agents/backend/teams/test_gbb_page_feed_team.py +++ b/tests/ci/captn/captn_agents/backend/teams/test_gbb_page_feed_team.py @@ -1,3 +1,4 @@ +import unittest from typing import Any, Iterator import pytest @@ -7,6 +8,10 @@ Team, ) +from ..tools.test_gbb_page_feed_team_tools import ( + _get_asset_sets_execute_query_return_value, + _get_assets_execute_query_return_value, +) from .helpers import helper_test_init, start_converstaion from .test_gbb_google_sheets_team import mock_list_accessible_customers_f @@ -50,10 +55,6 @@ def test_init(self, mock_get_conv_uuid: Iterator[Any]) -> None: def test_page_feed_real_fastapi_team_end2end( self, mock_get_conv_uuid: Iterator[Any], - # mock_get_login_url: Iterator[Any], - # mock_requests_get: Iterator[Any], - # mock_requests_post: Iterator[Any], - # mock_execute_query_f: Iterator[Any], ) -> None: user_id = 13 openapi_url = "https://google-sheets.tools.staging.fastagency.ai/openapi.json" @@ -67,10 +68,42 @@ def test_page_feed_real_fastapi_team_end2end( ) expected_messages = [ - "All page feeds have been updated.", + "Page feed '**fastagency-reference**' changes:", + "Page feed '**fastagency-tutorial**' changes:", + "Added page feed items:", + "The following page feed items should be removed by you manually:", + ] + + customer_id = "1111" + page_urls = [ + "https://fastagency.ai/latest/api/fastagency/FunctionCallExecution", + "https://fastagency.ai/latest/api/fastagency/FastAgency", + ] + execuse_query_side_effect = [ + _get_asset_sets_execute_query_return_value(customer_id), + _get_assets_execute_query_return_value( + customer_id=customer_id, page_urls=page_urls + ), + _get_assets_execute_query_return_value( + customer_id=customer_id, page_urls=[] + ), ] - with mock_list_accessible_customers_f(): + with ( + mock_list_accessible_customers_f(), + unittest.mock.patch( + "captn.captn_agents.backend.tools._gbb_page_feed_team_tools.execute_query", + side_effect=execuse_query_side_effect, + ), + unittest.mock.patch( + "captn.captn_agents.backend.tools._gbb_page_feed_team_tools.google_ads_post_or_get", + return_value="Created an asset set asset link", + ), + unittest.mock.patch( + "captn.captn_agents.backend.tools._gbb_page_feed_team_tools.google_ads_create_update", + return_value="Removed an asset set asset link", + ), + ): start_converstaion( user_id=user_id, team=team, expected_messages=expected_messages ) diff --git a/tests/ci/captn/captn_agents/backend/tools/test_gbb_google_sheets_team_tools.py b/tests/ci/captn/captn_agents/backend/tools/test_gbb_google_sheets_team_tools.py index e369d47f..321edcb7 100644 --- a/tests/ci/captn/captn_agents/backend/tools/test_gbb_google_sheets_team_tools.py +++ b/tests/ci/captn/captn_agents/backend/tools/test_gbb_google_sheets_team_tools.py @@ -16,7 +16,6 @@ _add_negative_campaign_keywords_lists, _add_new_sitelinks, _check_if_both_include_and_exclude_language_values_exist, - _check_mandatory_columns, _get_alredy_existing_campaigns, _get_language_codes, _setup_campaign, @@ -25,6 +24,7 @@ _update_callouts, _update_geo_targeting, _update_language_targeting, + check_mandatory_columns, create_google_ads_resources, create_google_sheets_team_toolbox, ) @@ -143,7 +143,7 @@ def test_check_mandatory_columns( "Negative": ["abcd"], } ) - result = _check_mandatory_columns( + result = check_mandatory_columns( df=df, mandatory_columns=mandatory_columns, table_title="table_title", diff --git a/tests/ci/captn/captn_agents/backend/tools/test_gbb_page_feed_team_tools.py b/tests/ci/captn/captn_agents/backend/tools/test_gbb_page_feed_team_tools.py index 04f7cb86..0f1a192b 100644 --- a/tests/ci/captn/captn_agents/backend/tools/test_gbb_page_feed_team_tools.py +++ b/tests/ci/captn/captn_agents/backend/tools/test_gbb_page_feed_team_tools.py @@ -1,22 +1,87 @@ import unittest -from typing import Iterator +import unittest.mock +from typing import Any, Iterator, List import pandas as pd import pytest from autogen.agentchat import AssistantAgent, UserProxyAgent +from autogen.io.base import IOStream from captn.captn_agents.backend.config import Config from captn.captn_agents.backend.tools._gbb_page_feed_team_tools import ( PageFeedTeamContext, + _add_missing_page_urls, + _get_page_feed_asset_sets, + _get_page_feed_items, _get_relevant_page_feeds_and_accounts, _get_sheet_data_and_return_df, + _sync_page_feed_asset_set, create_page_feed_team_toolbox, - validate_page_feed_data, + get_and_validate_page_feed_data, ) from .helpers import check_llm_config_descriptions, check_llm_config_total_tools +def _get_assets_execute_query_return_value( + customer_id: str, page_urls: List[str] +) -> str: + request_json = {customer_id: []} + for i in range(len(page_urls)): + asset_id = f"17311100649{i}" + asset = { + "asset": { + "resourceName": f"customers/{customer_id}/assets/{asset_id}", + "type": "PAGE_FEED", + "id": asset_id, + "pageFeedAsset": { + "pageUrl": page_urls[i], + }, + }, + "assetSetAsset": { + "resourceName": f"customers/{customer_id}/assetSetAssets/8783430659~{asset_id}" + }, + } + + request_json[customer_id].append(asset) + + mock_execute_query_return_value = str(request_json) + return mock_execute_query_return_value + + +def _get_asset_sets_execute_query_return_value(customer_id: str) -> str: + response_json = { + customer_id: [ + { + "assetSet": { + "resourceName": f"customers/{customer_id}/assetSets/8783430659", + "name": "fastagency-reference", + "id": "8783430659", + }, + }, + { + "assetSet": { + "resourceName": f"customers/{customer_id}/assetSets/8841207092", + "name": "fastagency-tutorial", + "id": "8841207092", + }, + }, + ] + } + return str(response_json) + + +@pytest.fixture() +def mock_execute_query_f(request: pytest.FixtureRequest) -> Iterator[Any]: + response_str = _get_asset_sets_execute_query_return_value(request.param) + + with unittest.mock.patch( + "captn.captn_agents.backend.tools._gbb_page_feed_team_tools.execute_query", + return_value=response_str, + ) as mock_execute_query: + yield mock_execute_query + + class TestPageFeedTeamTools: @pytest.fixture(autouse=True) def setup(self) -> Iterator[None]: @@ -55,7 +120,7 @@ def test_llm_config(self) -> None: { "reply_to_client": r"Respond to the client \(answer to his task or question for additional information\)", "ask_client_for_permission": "Ask the client for permission to make the changes.", - "validate_page_feed_data": "Validate page feed data", + "get_and_validate_page_feed_data": "Get and validate page feed data", "update_page_feeds": "Update Google Ads Page Feeds", "change_google_ads_account_or_refresh_token": "Change Google Ads account or refresh access token", }, @@ -157,7 +222,7 @@ def test_get_relevant_page_feeds_and_accounts( else: assert return_df.empty - def test_validate_page_feed_data(self) -> None: + def test_get_and_validate_page_feed_data(self) -> None: side_effect_get_sheet_data = [ { "values": [ @@ -272,7 +337,7 @@ def test_validate_page_feed_data(self) -> None: "captn.captn_agents.backend.tools._gbb_page_feed_team_tools.get_sheet_data", side_effect=side_effect_get_sheet_data, ): - return_value = validate_page_feed_data( + return_value = get_and_validate_page_feed_data( template_spreadsheet_id="abc123", page_feed_spreadsheet_id="def456", page_feed_sheet_title="Sheet1", @@ -283,3 +348,212 @@ def test_validate_page_feed_data(self) -> None: "{'7119828439': {'Login Customer Id': '7587037554', 'Name Account': 'airt technologies d.o.o.'}" in return_value ) + + @pytest.mark.parametrize("mock_execute_query_f", ["1111"], indirect=True) + def test_get_page_feed_asset_sets( + self, mock_execute_query_f: Iterator[Any] + ) -> None: + customer_id = "1111" + with mock_execute_query_f: + page_feed_asset_sets = _get_page_feed_asset_sets( + user_id=1, + conv_id=2, + customer_id=customer_id, + login_customer_id=customer_id, + ) + + assert len(page_feed_asset_sets) == 2 + + @pytest.mark.parametrize( + ("gads_page_urls", "page_feeds_df", "expected"), + [ + ( + [ + "https://getbybus.com/en/bus-zagreb-to-split", + "https://getbybus.com/hr/bus-zagreb-to-split", + ], + pd.DataFrame( + { + "Page URL": [ + "https://getbybus.com/en/bus-zagreb-to-split", + "https://getbybus.com/hr/bus-zagreb-to-split/", + "https://getbybus.com/it/bus-zagreb-to-split", + ], + "Custom Label": [ + "StS; en; Croatia", + "StS; hr; Croatia", + "StS; it; Croatia", + ], + } + ), + "No changes needed for page feed '**fastagency-reference**'\n\n", + ), + ( + [ + "https://getbybus.com/en/bus-zagreb-to-split", + "https://getbybus.com/hr/bus-zagreb-to-split/", + "https://getbybus.com/it/bus-zagreb-to-split", + ], + pd.DataFrame( + { + "Page URL": [ + "https://getbybus.com/en/bus-zagreb-to-split", + "https://getbybus.com/hr/bus-zagreb-to-split/", + "https://getbybus.com/it/bus-zagreb-to-split", + ], + "Custom Label": [ + "StS; en; Croatia", + "StS; hr; Croatia", + "StS; it; Croatia", + ], + } + ), + """Page feed '**fastagency-reference**' changes: +The following page feed items should be removed by you manually: +- https://getbybus.com/it/bus-zagreb-to-split\n\n""", + ), + ( + [ + "https://getbybus.com/en/bus-zagreb-to-split", + "https://getbybus.com/hr/bus-zagreb-to-split/", + ], + pd.DataFrame( + { + "Page URL": [ + "https://getbybus.com/en/bus-zagreb-to-split", + "https://getbybus.com/hr/bus-zagreb-to-split/", + "https://getbybus.com/hr/bus-zagreb-to-karlovac", + ], + "Custom Label": [ + "StS; en; Croatia", + "StS; hr; Croatia", + "StS; hr; Croatia", + ], + } + ), + """Page feed '**fastagency-reference**' changes: +Added page feed items: +https://getbybus.com/hr/bus-zagreb-to-karlovac\n\n""", + ), + ], + ) + def test_sync_page_feed_asset_set( + self, gads_page_urls: List[str], page_feeds_df: pd.DataFrame, expected: str + ) -> None: + customer_id = "1111" + page_feeds_and_accounts_templ_df = pd.DataFrame( + { + "Customer Id": [customer_id], + "Name Page Feed": ["fastagency-reference"], + "Custom Label 1": ["StS; hr; Croatia"], + "Custom Label 2": ["StS; en; Croatia"], + } + ) + + page_feed_asset_set_name = "fastagency-reference" + page_feed_asset_set = { + "resourceName": f"customers/{customer_id}/assetSets/8783430659", + "id": "8783430659", + } + + mock_execute_query_return_value = _get_assets_execute_query_return_value( + customer_id, gads_page_urls + ) + + with ( + unittest.mock.patch( + "captn.captn_agents.backend.tools._gbb_page_feed_team_tools.execute_query", + side_effect=[mock_execute_query_return_value], + ), + unittest.mock.patch( + "captn.captn_agents.backend.tools._gbb_page_feed_team_tools.google_ads_post_or_get", + return_value="Created an asset set asset link", + ), + unittest.mock.patch( + "captn.captn_agents.backend.tools._gbb_page_feed_team_tools.google_ads_create_update", + return_value="Removed an asset set asset link", + ), + ): + result = _sync_page_feed_asset_set( + user_id=-1, + conv_id=-1, + customer_id=customer_id, + login_customer_id=customer_id, + page_feeds_and_accounts_templ_df=page_feeds_and_accounts_templ_df, + page_feeds_df=page_feeds_df, + page_feed_asset_set_name=page_feed_asset_set_name, + page_feed_asset_set=page_feed_asset_set, + iostream=IOStream.get_default(), + ) + assert result == expected + + def test_get_page_feed_items(self) -> None: + expected_page_urls = [ + "https://fastagency.ai/latest/api/fastagency/FunctionCallExecution", + "https://fastagency.ai/latest/api/fastagency/FastAgency", + ] + expected_page_urls_and_labels_df = pd.DataFrame( + { + "Page URL": expected_page_urls, + "Custom Label": [None, None], + } + ) + customer_id = "1111" + + mock_execute_query_return_value = _get_assets_execute_query_return_value( + customer_id, expected_page_urls + ) + asset_set_resource_name = f"customers/{customer_id}/assetSets/8783430659" + + with unittest.mock.patch( + "captn.captn_agents.backend.tools._gbb_page_feed_team_tools.execute_query", + return_value=mock_execute_query_return_value, + ): + page_urls_and_labels_df = _get_page_feed_items( + user_id=-1, + conv_id=-1, + customer_id=customer_id, + login_customer_id=customer_id, + asset_set_resource_name=asset_set_resource_name, + ) + + page_urls_and_labels_df = page_urls_and_labels_df.drop(columns=["Id"]) + assert page_urls_and_labels_df.sort_values("Page URL").equals( + expected_page_urls_and_labels_df.sort_values("Page URL") + ) + + def test_add_missing_page_urls(self) -> None: + with unittest.mock.patch( + "captn.captn_agents.backend.tools._gbb_page_feed_team_tools.google_ads_post_or_get", + ) as mock_google_ads_post_or_get: + mock_google_ads_post_or_get.side_effect = [ + ValueError("Error1"), + ValueError("Error2"), + "Created an asset set asset link", + ] + + result = _add_missing_page_urls( + user_id=-1, + conv_id=-1, + customer_id="1111", + login_customer_id="1111", + page_feed_asset_set={ + "resourceName": "customers/1111/assetSets/8783430659", + "id": "8783430659", + }, + missing_page_urls=pd.DataFrame( + { + "Page URL": [ + "https://fastagency.ai/latest/api/fastagency/FunctionCallExecution", + "https://fastagency.ai/latest/api/fastagency/FastAgency", + ], + "Custom Label": [None, None], + } + ), + ) + expected = """Added page feed items: +https://fastagency.ai/latest/api/fastagency/FunctionCallExecution +https://fastagency.ai/latest/api/fastagency/FastAgency\n\n""" + assert result == expected, result + + assert mock_google_ads_post_or_get.call_count == 3