From dfc17a6133bf6a0cfee29b19740098d924a9a231 Mon Sep 17 00:00:00 2001 From: marketcalls Date: Sat, 30 Nov 2024 22:35:28 +0530 Subject: [PATCH 01/41] initial commit for shoonya --- blueprints/brlogin.py | 14 + broker/shoonya/__init__.py | 0 broker/shoonya/api/__init__.py | 0 broker/shoonya/api/auth_api.py | 57 ++ broker/shoonya/api/funds.py | 71 +++ broker/shoonya/api/order_api.py | 312 ++++++++++ broker/shoonya/database/master_contract_db.py | 555 ++++++++++++++++++ broker/shoonya/mapping/order_data.py | 392 +++++++++++++ broker/shoonya/mapping/transform_data.py | 88 +++ broker/shoonya/plugin.json | 8 + templates/shoonya.html | 134 +++++ 11 files changed, 1631 insertions(+) create mode 100644 broker/shoonya/__init__.py create mode 100644 broker/shoonya/api/__init__.py create mode 100644 broker/shoonya/api/auth_api.py create mode 100644 broker/shoonya/api/funds.py create mode 100644 broker/shoonya/api/order_api.py create mode 100644 broker/shoonya/database/master_contract_db.py create mode 100644 broker/shoonya/mapping/order_data.py create mode 100644 broker/shoonya/mapping/transform_data.py create mode 100644 broker/shoonya/plugin.json create mode 100644 templates/shoonya.html diff --git a/blueprints/brlogin.py b/blueprints/brlogin.py index e1cf26a0..12e2a2d1 100644 --- a/blueprints/brlogin.py +++ b/blueprints/brlogin.py @@ -123,6 +123,20 @@ def broker_callback(broker,para=None): auth_token, error_message = auth_function(userid, password, totp_code) forward_url = 'zebu.html' + elif broker == 'shoonya': + if request.method == 'GET': + if 'user' not in session: + return redirect(url_for('auth.login')) + return render_template('shoonya.html') + + elif request.method == 'POST': + userid = request.form.get('userid') + password = request.form.get('password') + totp_code = request.form.get('totp') + + auth_token, error_message = auth_function(userid, password, totp_code) + forward_url = 'shoonya.html' + elif broker=='kotak': print(f"The Broker is {broker}") if request.method == 'GET': diff --git a/broker/shoonya/__init__.py b/broker/shoonya/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/broker/shoonya/api/__init__.py b/broker/shoonya/api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/broker/shoonya/api/auth_api.py b/broker/shoonya/api/auth_api.py new file mode 100644 index 00000000..ddf3b521 --- /dev/null +++ b/broker/shoonya/api/auth_api.py @@ -0,0 +1,57 @@ +import requests +import hashlib +import json +import os + +def sha256_hash(text): + """Generate SHA256 hash.""" + return hashlib.sha256(text.encode('utf-8')).hexdigest() + +def authenticate_broker(userid, password, totp_code): + """ + Authenticate with Shoonya and return the auth token. + """ + # Get the Shoonya API key and other credentials from environment variables + api_secretkey = os.getenv('BROKER_API_SECRET') + vendor_code = os.getenv('BROKER_API_KEY') + imei = '1234567890abcdef' # Default IMEI if not provided + + try: + # Shoonya API login URL + url = "https://api.shoonya.com/NorenWClientTP/QuickAuth" + + # Prepare login payload + payload = { + "uid": userid, # User ID + "pwd": sha256_hash(password), # SHA256 hashed password + "factor2": totp_code, # PAN or TOTP or DOB (second factor) + "apkversion": "1.0.0", # API version (as per Shoonya's requirement) + "appkey": sha256_hash(f"{userid}|{api_secretkey}"), # SHA256 of uid and API key + "imei": imei, # IMEI or MAC address + "vc": vendor_code, # Vendor code + "source": "API" # Source of login request + } + + # Convert payload to string with 'jData=' prefix + payload_str = "jData=" + json.dumps(payload) + + # Set headers for the API request + headers = { + 'Content-Type': 'application/x-www-form-urlencoded' + } + + # Send the POST request to Shoonya's API + response = requests.post(url, data=payload_str, headers=headers) + + # Handle the response + if response.status_code == 200: + data = response.json() + if data['stat'] == "Ok": + return data['susertoken'], None # Return the token on success + else: + return None, data.get('emsg', 'Authentication failed. Please try again.') + else: + return None, f"Error: {response.status_code}, {response.text}" + + except Exception as e: + return None, str(e) \ No newline at end of file diff --git a/broker/shoonya/api/funds.py b/broker/shoonya/api/funds.py new file mode 100644 index 00000000..fdaa1847 --- /dev/null +++ b/broker/shoonya/api/funds.py @@ -0,0 +1,71 @@ +import os +import http.client +import json + +def get_margin_data(auth_token): + """Fetch margin data from Shoonya's API using the provided auth token.""" + + # Shoonya API endpoint for fetching margin data + url = "api.shoonya.com" + + # Fetch UserID and AccountID from environment variables + userid = os.getenv('BROKER_API_KEY') + actid = userid # Assuming AccountID is the same as UserID + + # Prepare the payload for the request + data = { + "uid": userid, # User ID + "actid": actid # Account ID + } + + # Prepare the jData payload with the authentication token (jKey) + payload = "jData=" + json.dumps(data) + "&jKey=" + auth_token + + # Initialize HTTP connection + conn = http.client.HTTPSConnection(url) + + # Set headers + headers = { + 'Content-Type': 'application/json' + } + + # Send the POST request to Shoonya's API + conn.request("POST", "/NorenWClientTP/Limits", payload, headers) + + # Get the response + res = conn.getresponse() + data = res.read() + + # Parse the response + margin_data = json.loads(data.decode("utf-8")) + + print(margin_data) + + # Check if the request was successful + if margin_data.get('stat') != 'Ok': + # Log the error or return an empty dictionary to indicate failure + print(f"Error fetching margin data: {margin_data.get('emsg')}") + return {} + + try: + # Calculate total_available_margin as the sum of 'cash' and 'payin' + total_available_margin = float(margin_data.get('cash',0)) + float(margin_data.get('payin',0)) - float(margin_data.get('marginused',0)) + total_collateral = float(margin_data.get('brkcollamt',0)) + total_used_margin = float(margin_data.get('marginused',0)) + total_realised = float(margin_data.get('rpnl',0)) + total_unrealised = float(margin_data.get('urmtom',0)) + + # Construct and return the processed margin data + processed_margin_data = { + "availablecash": "{:.2f}".format(total_available_margin), + "collateral": "{:.2f}".format(total_collateral), + "m2munrealized": "{:.2f}".format(total_unrealised), + "m2mrealized": "{:.2f}".format(total_realised), + "utiliseddebits": "{:.2f}".format(total_used_margin), + } + return processed_margin_data + except KeyError as e: + # Log the exception and return an empty dictionary if there's an unexpected error + print(f"Error processing margin data: {str(e)}") + return {} + diff --git a/broker/shoonya/api/order_api.py b/broker/shoonya/api/order_api.py new file mode 100644 index 00000000..c439be59 --- /dev/null +++ b/broker/shoonya/api/order_api.py @@ -0,0 +1,312 @@ +import http.client +import json +import os +from database.auth_db import get_auth_token +from database.token_db import get_token , get_br_symbol, get_symbol +from broker.shoonya.mapping.transform_data import transform_data , map_product_type, reverse_map_product_type, transform_modify_order_data + + +def get_api_response(endpoint, auth, method="GET", payload=''): + + AUTH_TOKEN = auth + + api_key = os.getenv('BROKER_API_KEY') + + data = f'{{"uid": "{api_key}", "actid": "{api_key}"}}' + + if(endpoint == "/NorenWClientTP/Holdings"): + data = f'{{"uid": "{api_key}", "actid": "{api_key}", "prd": "C"}}' + + payload = "jData=" + data + "&jKey=" + AUTH_TOKEN + + conn = http.client.HTTPSConnection("api.shoonya.com") + headers = {'Content-Type': 'application/json'} + + conn.request(method, endpoint, payload, headers) + res = conn.getresponse() + data = res.read() + + return json.loads(data.decode("utf-8")) + +def get_order_book(auth): + return get_api_response("/NorenWClientTP/OrderBook",auth,method="POST") + +def get_trade_book(auth): + return get_api_response("/NorenWClientTP/TradeBook",auth,method="POST") + +def get_positions(auth): + return get_api_response("/NorenWClientTP/PositionBook",auth,method="POST") + +def get_holdings(auth): + return get_api_response("/NorenWClientTP/Holdings",auth,method="POST") + +def get_open_position(tradingsymbol, exchange, producttype,auth): + #Convert Trading Symbol from OpenAlgo Format to Broker Format Before Search in OpenPosition + tradingsymbol = get_br_symbol(tradingsymbol,exchange) + positions_data = get_positions(auth) + + print(positions_data) + + net_qty = '0' + + if positions_data is None or (isinstance(positions_data, dict) and (positions_data['stat'] == "Not_Ok")): + # Handle the case where there is no data + print("No data available.") + net_qty = '0' + + if positions_data and isinstance(positions_data, list): + for position in positions_data: + if position.get('tsym') == tradingsymbol and position.get('exch') == exchange and position.get('prd') == producttype: + net_qty = position.get('netqty', '0') + break # Assuming you need the first match + + return net_qty + +def place_order_api(data,auth): + AUTH_TOKEN = auth + BROKER_API_KEY = os.getenv('BROKER_API_KEY') + data['apikey'] = BROKER_API_KEY + token = get_token(data['symbol'], data['exchange']) + newdata = transform_data(data, token) + headers = {'Content-Type': 'application/json'} + + payload = "jData=" + json.dumps(newdata) + "&jKey=" + AUTH_TOKEN + + print(payload) + conn = http.client.HTTPSConnection("api.shoonya.com") + conn.request("POST", "/NorenWClientTP/PlaceOrder", payload, headers) + res = conn.getresponse() + response_data = json.loads(res.read().decode("utf-8")) + if response_data['stat'] == "Ok": + orderid = response_data['norenordno'] + else: + orderid = None + return res, response_data, orderid + +def place_smartorder_api(data,auth): + + AUTH_TOKEN = auth + + #If no API call is made in this function then res will return None + res = None + + # Extract necessary info from data + symbol = data.get("symbol") + exchange = data.get("exchange") + product = data.get("product") + position_size = int(data.get("position_size", "0")) + + + + # Get current open position for the symbol + current_position = int(get_open_position(symbol, exchange, map_product_type(product),AUTH_TOKEN)) + + + print(f"position_size : {position_size}") + print(f"Open Position : {current_position}") + + # Determine action based on position_size and current_position + action = None + quantity = 0 + + + # If both position_size and current_position are 0, do nothing + if position_size == 0 and current_position == 0: + action = data['action'] + quantity = data['quantity'] + #print(f"action : {action}") + #print(f"Quantity : {quantity}") + res, response, orderid = place_order_api(data,auth) + # print(res) + # print(response) + # print(orderid) + return res , response, orderid + + elif position_size == current_position: + response = {"status": "success", "message": "No action needed. Position size matches current position."} + orderid = None + return res, response, orderid # res remains None as no API call was mad + + + + if position_size == 0 and current_position>0 : + action = "SELL" + quantity = abs(current_position) + elif position_size == 0 and current_position<0 : + action = "BUY" + quantity = abs(current_position) + elif current_position == 0: + action = "BUY" if position_size > 0 else "SELL" + quantity = abs(position_size) + else: + if position_size > current_position: + action = "BUY" + quantity = position_size - current_position + #print(f"smart buy quantity : {quantity}") + elif position_size < current_position: + action = "SELL" + quantity = current_position - position_size + #print(f"smart sell quantity : {quantity}") + + + + + if action: + # Prepare data for placing the order + order_data = data.copy() + order_data["action"] = action + order_data["quantity"] = str(quantity) + + #print(order_data) + # Place the order + res, response, orderid = place_order_api(order_data,auth) + #print(res) + print(response) + print(orderid) + + return res , response, orderid + + + + +def close_all_positions(current_api_key,auth): + # Fetch the current open positions + AUTH_TOKEN = auth + + positions_response = get_positions(AUTH_TOKEN) + + # Check if the positions data is null or empty + if positions_response is None or positions_response[0]['stat'] == "Not_Ok": + return {"message": "No Open Positions Found"}, 200 + + if positions_response: + # Loop through each position to close + for position in positions_response: + # Skip if net quantity is zero + if int(position['netqty']) == 0: + continue + + # Determine action based on net quantity + action = 'SELL' if int(position['netqty']) > 0 else 'BUY' + quantity = abs(int(position['netqty'])) + + + #get openalgo symbol to send to placeorder function + symbol = get_symbol(position['token'],position['exch']) + print(f'The Symbol is {symbol}') + + # Prepare the order payload + place_order_payload = { + "apikey": current_api_key, + "strategy": "Squareoff", + "symbol": symbol, + "action": action, + "exchange": position['exch'], + "pricetype": "MARKET", + "product": reverse_map_product_type(position['prd']), + "quantity": str(quantity) + } + + print(place_order_payload) + + # Place the order to close the position + res, response, orderid = place_order_api(place_order_payload,auth) + + # print(res) + # print(response) + # print(orderid) + + + + # Note: Ensure place_order_api handles any errors and logs accordingly + + return {'status': 'success', "message": "All Open Positions SquaredOff"}, 200 + + +def cancel_order(orderid,auth): + # Assuming you have a function to get the authentication token + AUTH_TOKEN = auth + api_key = os.getenv('BROKER_API_KEY') + data = {"uid": api_key, "norenordno": orderid} + + + payload = "jData=" + json.dumps(data) + "&jKey=" + AUTH_TOKEN + # Set up the request headers + headers = {'Content-Type': 'application/json'} + + + + + # Establish the connection and send the request + conn = http.client.HTTPSConnection("api.shoonya.com") # Adjust the URL as necessary + conn.request("POST", "/NorenWClientTP/CancelOrder", payload, headers) + res = conn.getresponse() + data = json.loads(res.read().decode("utf-8")) + print(data) + + # Check if the request was successful + if data.get("stat")=='Ok': + # Return a success response + return {"status": "success", "orderid": orderid}, 200 + else: + # Return an error response + return {"status": "error", "message": data.get("message", "Failed to cancel order")}, res.status + + +def modify_order(data,auth): + + # Assuming you have a function to get the authentication token + AUTH_TOKEN = auth + api_key = os.getenv('BROKER_API_KEY') + + token = get_token(data['symbol'], data['exchange']) + data['symbol'] = get_br_symbol(data['symbol'],data['exchange']) + data["apikey"] = api_key + + transformed_data = transform_modify_order_data(data, token) # You need to implement this function + # Set up the request headers + headers = {'Content-Type': 'application/json'} + payload = "jData=" + json.dumps(transformed_data) + "&jKey=" + AUTH_TOKEN + + + conn = http.client.HTTPSConnection("api.shoonya.com") + conn.request("POST", "/NorenWClientTP/ModifyOrder", payload, headers) + res = conn.getresponse() + response = json.loads(res.read().decode("utf-8")) + + if response.get("stat")=='Ok': + return {"status": "success", "orderid": data["orderid"]}, 200 + else: + return {"status": "error", "message": response.get("emsg", "Failed to modify order")}, res.status + + + +def cancel_all_orders_api(data,auth): + # Get the order book + + AUTH_TOKEN = auth + + + order_book_response = get_order_book(AUTH_TOKEN) + #print(order_book_response) + if order_book_response is None: + return [], [] # Return empty lists indicating failure to retrieve the order book + + # Filter orders that are in 'open' or 'trigger_pending' state + orders_to_cancel = [order for order in order_book_response + if order['status'] in ['OPEN', 'TRIGGER PENDING']] + #print(orders_to_cancel) + canceled_orders = [] + failed_cancellations = [] + + # Cancel the filtered orders + for order in orders_to_cancel: + orderid = order['norenordno'] + cancel_response, status_code = cancel_order(orderid,auth) + if status_code == 200: + canceled_orders.append(orderid) + else: + failed_cancellations.append(orderid) + + return canceled_orders, failed_cancellations + diff --git a/broker/shoonya/database/master_contract_db.py b/broker/shoonya/database/master_contract_db.py new file mode 100644 index 00000000..1cccb952 --- /dev/null +++ b/broker/shoonya/database/master_contract_db.py @@ -0,0 +1,555 @@ +import os +import requests +import zipfile +import io +import pandas as pd +from datetime import datetime +from sqlalchemy import create_engine, Column, Integer, String, Float, Sequence, Index +from sqlalchemy.orm import scoped_session, sessionmaker +from sqlalchemy.ext.declarative import declarative_base +from dotenv import load_dotenv +from extensions import socketio # Import SocketIO + + + +# Load environment variables +load_dotenv() + +# Database setup +DATABASE_URL = os.getenv('DATABASE_URL') # Replace with your database path +engine = create_engine(DATABASE_URL) +db_session = scoped_session(sessionmaker(autocommit=False, autoflush=False, bind=engine)) +Base = declarative_base() +Base.query = db_session.query_property() + +# Define SymToken table +class SymToken(Base): + __tablename__ = 'symtoken' + id = Column(Integer, Sequence('symtoken_id_seq'), primary_key=True) + symbol = Column(String, nullable=False, index=True) # Single column index + brsymbol = Column(String, nullable=False, index=True) # Single column index + name = Column(String) + exchange = Column(String, index=True) # Include this column in a composite index + brexchange = Column(String, index=True) + token = Column(String, index=True) # Indexed for performance + expiry = Column(String) + strike = Column(Float) + lotsize = Column(Integer) + instrumenttype = Column(String) + tick_size = Column(Float) + + # Define a composite index on symbol and exchange columns + __table_args__ = (Index('idx_symbol_exchange', 'symbol', 'exchange'),) + +def init_db(): + print("Initializing Master Contract DB") + Base.metadata.create_all(bind=engine) + +def delete_symtoken_table(): + print("Deleting Symtoken Table") + SymToken.query.delete() + db_session.commit() + +def copy_from_dataframe(df): + print("Performing Bulk Insert") + # Convert DataFrame to a list of dictionaries + data_dict = df.to_dict(orient='records') + + # Retrieve existing tokens to filter them out from the insert + existing_tokens = {result.token for result in db_session.query(SymToken.token).all()} + + # Filter out data_dict entries with tokens that already exist + filtered_data_dict = [row for row in data_dict if row['token'] not in existing_tokens] + + # Insert in bulk the filtered records + try: + if filtered_data_dict: # Proceed only if there's anything to insert + db_session.bulk_insert_mappings(SymToken, filtered_data_dict) + db_session.commit() + print(f"Bulk insert completed successfully with {len(filtered_data_dict)} new records.") + else: + print("No new records to insert.") + except Exception as e: + print(f"Error during bulk insert: {e}") + db_session.rollback() + +# Define the shoonya URLs for downloading the symbol files +shoonya_urls = { + "NSE": "https://api.shoonya.com/NSE_symbols.txt.zip", + "NFO": "https://api.shoonya.com/NFO_symbols.txt.zip", + "CDS": "https://api.shoonya.com/CDS_symbols.txt.zip", + "MCX": "https://api.shoonya.com/MCX_symbols.txt.zip", + "BSE": "https://api.shoonya.com/BSE_symbols.txt.zip", + "BFO": "https://api.shoonya.com/BFO_symbols.txt.zip" +} + +def download_and_unzip_shoonya_data(output_path): + """ + Downloads and unzips the shoonya text files to the tmp folder. + """ + print("Downloading and Unzipping shoonya Data") + + # Create the tmp directory if it doesn't exist + if not os.path.exists(output_path): + os.makedirs(output_path) + + downloaded_files = [] + + # Iterate through the shoonya URLs and download/unzip files + for key, url in shoonya_urls.items(): + try: + # Send GET request to download the zip file + response = requests.get(url, timeout=10) + + if response.status_code == 200: + print(f"Successfully downloaded {key} from {url}") + + # Use in-memory file to handle the downloaded zip file + z = zipfile.ZipFile(io.BytesIO(response.content)) + z.extractall(output_path) + downloaded_files.append(f"{key}.txt") + else: + print(f"Failed to download {key} from {url}. Status code: {response.status_code}") + except Exception as e: + print(f"Error downloading {key} from {url}: {e}") + + return downloaded_files + +# Placeholder functions for processing data + +def process_shoonya_nse_data(output_path): + """ + Processes the shoonya NSE data (NSE_symbols.txt) to generate OpenAlgo symbols. + Separates EQ, BE symbols, and Index symbols. + """ + print("Processing shoonya NSE Data") + file_path = f'{output_path}/NSE_symbols.txt' + + # Read the NSE symbols file, specifying the exact columns to use and ignoring extra columns + df = pd.read_csv(file_path, usecols=['Exchange', 'Token', 'LotSize', 'Symbol', 'TradingSymbol', 'Instrument', 'TickSize']) + + # Rename columns to match your schema + df.columns = ['exchange', 'token', 'lotsize', 'name', 'brsymbol', 'instrumenttype', 'tick_size'] + + # Add missing columns to ensure DataFrame matches the database structure + df['symbol'] = df['brsymbol'] # Initialize 'symbol' with 'brsymbol' + + # Apply transformation for OpenAlgo symbols + def get_openalgo_symbol(broker_symbol): + # Separate by hyphen and apply logic for EQ and BE + if '-EQ' in broker_symbol: + return broker_symbol.replace('-EQ', '') + elif '-BE' in broker_symbol: + return broker_symbol.replace('-BE', '') + else: + # For other symbols (including index), OpenAlgo symbol remains the same as broker symbol + return broker_symbol + + # Update the 'symbol' column + df['symbol'] = df['brsymbol'].apply(get_openalgo_symbol) + + # Define Exchange: 'NSE' for EQ and BE, 'NSE_INDEX' for indexes + df['exchange'] = df.apply(lambda row: 'NSE_INDEX' if row['instrumenttype'] == 'INDEX' else 'NSE', axis=1) + df['brexchange'] = df['exchange'] # Broker exchange is the same as exchange + + # Set empty columns for 'expiry' and fill -1 for 'strike' where the data is missing + df['expiry'] = '' # No expiry for these instruments + df['strike'] = -1 # Set default value -1 for strike price where missing + + # Ensure the instrument type is consistent + df['instrumenttype'] = df['instrumenttype'].apply(lambda x: 'EQ' if x in ['EQ', 'BE'] else x) + + # Handle missing or invalid numeric values in 'lotsize' and 'tick_size' + df['lotsize'] = pd.to_numeric(df['lotsize'], errors='coerce').fillna(0).astype(int) # Convert to int, default to 0 + df['tick_size'] = pd.to_numeric(df['tick_size'], errors='coerce').fillna(0).astype(float) # Convert to float, default to 0.0 + + # Reorder the columns to match the database structure + columns_to_keep = ['symbol', 'brsymbol', 'name', 'exchange', 'brexchange', 'token', 'expiry', 'strike', 'lotsize', 'instrumenttype', 'tick_size'] + df_filtered = df[columns_to_keep] + + # Return the processed DataFrame + return df_filtered + + + +def process_shoonya_nfo_data(output_path): + """ + Processes the shoonya NFO data (NFO_symbols.txt) to generate OpenAlgo symbols. + Handles both futures and options formatting. + """ + print("Processing shoonya NFO Data") + file_path = f'{output_path}/NFO_symbols.txt' + + # Read the NFO symbols file, specifying the exact columns to use + df = pd.read_csv(file_path, usecols=['Exchange', 'Token', 'LotSize', 'Symbol', 'TradingSymbol', 'Expiry', 'Instrument', 'OptionType', 'StrikePrice', 'TickSize']) + + # Rename columns to match your schema + df.columns = ['exchange', 'token', 'lotsize', 'name', 'brsymbol', 'expiry', 'instrumenttype', 'optiontype', 'strike', 'tick_size'] + + # Add missing columns to ensure DataFrame matches the database structure + df['expiry'] = df['expiry'].fillna('') # Fill expiry with empty strings if missing + df['strike'] = df['strike'].fillna('-1') # Fill strike with -1 if missing + + # Define a function to format the expiry date as DDMMMYY + def format_expiry_date(date_str): + try: + return datetime.strptime(date_str, '%d-%b-%Y').strftime('%d%b%y').upper() + except ValueError: + print(f"Invalid expiry date format: {date_str}") + return None + + # Apply the expiry date format + df['expiry'] = df['expiry'].apply(format_expiry_date) + + # Replace the 'XX' option type with 'FUT' for futures + df['instrumenttype'] = df.apply(lambda row: 'FUT' if row['optiontype'] == 'XX' else row['optiontype'], axis=1) + + # Format the symbol column based on the instrument type + def format_symbol(row): + if row['instrumenttype'] == 'FUT': + return f"{row['name']}{row['expiry']}FUT" + else: + # Ensure strike prices are either integers or floats + formatted_strike = int(row['strike']) if float(row['strike']).is_integer() else row['strike'] + return f"{row['name']}{row['expiry']}{formatted_strike}{row['instrumenttype']}" + + df['symbol'] = df.apply(format_symbol, axis=1) + + # Define Exchange + df['exchange'] = 'NFO' + df['brexchange'] = df['exchange'] + + # Ensure strike prices are handled as either float or int + def handle_strike_price(strike): + try: + if float(strike).is_integer(): + return int(float(strike)) # Return as integer if no decimal + else: + return float(strike) # Return as float if decimal exists + except (ValueError, TypeError): + return -1 # If there's an error or it's empty, return -1 + + # Apply the function to strike column + df['strike'] = df['strike'].apply(handle_strike_price) + + # Reorder the columns to match the database structure + columns_to_keep = ['symbol', 'brsymbol', 'name', 'exchange', 'brexchange', 'token', 'expiry', 'strike', 'lotsize', 'instrumenttype', 'tick_size'] + df_filtered = df[columns_to_keep] + + # Return the processed DataFrame + return df_filtered + +def process_shoonya_cds_data(output_path): + """ + Processes the shoonya CDS data (CDS_symbols.txt) to generate OpenAlgo symbols. + Handles both futures and options formatting. + """ + print("Processing shoonya CDS Data") + file_path = f'{output_path}/CDS_symbols.txt' + + # Read the CDS symbols file, specifying the exact columns to use + df = pd.read_csv(file_path, usecols=['Exchange', 'Token', 'LotSize', 'Precision', 'Multiplier', 'Symbol', 'TradingSymbol', 'Expiry', 'Instrument', 'OptionType', 'StrikePrice', 'TickSize']) + + # Rename columns to match your schema + df.columns = ['exchange', 'token', 'lotsize', 'precision', 'multiplier', 'name', 'brsymbol', 'expiry', 'instrumenttype', 'optiontype', 'strike', 'tick_size'] + + # Add missing columns to ensure DataFrame matches the database structure + df['expiry'] = df['expiry'].fillna('') # Fill expiry with empty strings if missing + df['strike'] = df['strike'].fillna('-1') # Fill strike with -1 if missing + + # Define a function to format the expiry date as DDMMMYY + def format_expiry_date(date_str): + try: + return datetime.strptime(date_str, '%d-%b-%Y').strftime('%d%b%y').upper() + except ValueError: + print(f"Invalid expiry date format: {date_str}") + return None + + # Apply the expiry date format + df['expiry'] = df['expiry'].apply(format_expiry_date) + + # Replace the 'XX' option type with 'FUT' for futures + df['instrumenttype'] = df.apply(lambda row: 'FUT' if row['optiontype'] == 'XX' else row['instrumenttype'], axis=1) + + # Update instrumenttype to 'CE' or 'PE' based on the option type + df['instrumenttype'] = df.apply(lambda row: row['optiontype'] if row['instrumenttype'] == 'OPTCUR' else row['instrumenttype'], axis=1) + + # Format the symbol column based on the instrument type + def format_symbol(row): + if row['instrumenttype'] == 'FUT': + return f"{row['name']}{row['expiry']}FUT" + else: + return f"{row['name']}{row['expiry']}{row['strike']}{row['instrumenttype']}" + + df['symbol'] = df.apply(format_symbol, axis=1) + + # Define Exchange + df['exchange'] = 'CDS' + df['brexchange'] = df['exchange'] + + # Ensure strike prices are handled as either float or int + def handle_strike_price(strike): + try: + if float(strike).is_integer(): + return int(float(strike)) # Return as integer if no decimal + else: + return float(strike) # Return as float if decimal exists + except (ValueError, TypeError): + return -1 # If there's an error or it's empty, return -1 + + # Apply the function to strike column + df['strike'] = df['strike'].apply(handle_strike_price) + + # Reorder the columns to match the database structure + columns_to_keep = ['symbol', 'brsymbol', 'name', 'exchange', 'brexchange', 'token', 'expiry', 'strike', 'lotsize', 'instrumenttype', 'tick_size'] + df_filtered = df[columns_to_keep] + + # Return the processed DataFrame + return df_filtered + +def process_shoonya_mcx_data(output_path): + """ + Processes the shoonya MCX data (MCX_symbols.txt) to generate OpenAlgo symbols. + Handles both futures and options formatting. + """ + print("Processing shoonya MCX Data") + file_path = f'{output_path}/MCX_symbols.txt' + + # Read the MCX symbols file, specifying the exact columns to use + df = pd.read_csv(file_path, usecols=['Exchange', 'Token', 'LotSize', 'GNGD', 'Symbol', 'TradingSymbol', 'Expiry', 'Instrument', 'OptionType', 'StrikePrice', 'TickSize']) + + # Rename columns to match your schema + df.columns = ['exchange', 'token', 'lotsize', 'gngd', 'name', 'brsymbol', 'expiry', 'instrumenttype', 'optiontype', 'strike', 'tick_size'] + + # Add missing columns to ensure DataFrame matches the database structure + df['expiry'] = df['expiry'].fillna('') # Fill expiry with empty strings if missing + df['strike'] = df['strike'].fillna('-1') # Fill strike with -1 if missing + + # Define a function to format the expiry date as DDMMMYY + def format_expiry_date(date_str): + try: + return datetime.strptime(date_str, '%d-%b-%Y').strftime('%d%b%y').upper() + except ValueError: + print(f"Invalid expiry date format: {date_str}") + return None + + # Apply the expiry date format + df['expiry'] = df['expiry'].apply(format_expiry_date) + + # Replace the 'XX' option type with 'FUT' for futures + df['instrumenttype'] = df.apply(lambda row: 'FUT' if row['optiontype'] == 'XX' else row['instrumenttype'], axis=1) + + # Update instrumenttype to 'CE' or 'PE' based on the option type + df['instrumenttype'] = df.apply(lambda row: row['optiontype'] if row['instrumenttype'] == 'OPTFUT' else row['instrumenttype'], axis=1) + + # Format the symbol column based on the instrument type + def format_symbol(row): + if row['instrumenttype'] == 'FUT': + return f"{row['name']}{row['expiry']}FUT" + else: + return f"{row['name']}{row['expiry']}{row['strike']}{row['instrumenttype']}" + + df['symbol'] = df.apply(format_symbol, axis=1) + + # Define Exchange + df['exchange'] = 'MCX' + df['brexchange'] = df['exchange'] + + # Ensure strike prices are handled as either float or int + def handle_strike_price(strike): + try: + if float(strike).is_integer(): + return int(float(strike)) # Return as integer if no decimal + else: + return float(strike) # Return as float if decimal exists + except (ValueError, TypeError): + return -1 # If there's an error or it's empty, return -1 + + # Apply the function to strike column + df['strike'] = df['strike'].apply(handle_strike_price) + + # Reorder the columns to match the database structure + columns_to_keep = ['symbol', 'brsymbol', 'name', 'exchange', 'brexchange', 'token', 'expiry', 'strike', 'lotsize', 'instrumenttype', 'tick_size'] + df_filtered = df[columns_to_keep] + + # Return the processed DataFrame + return df_filtered + +def process_shoonya_bse_data(output_path): + """ + Processes the shoonya BSE data (BSE_symbols.txt) to generate OpenAlgo symbols. + Ensures that the instrument type is always 'EQ'. + """ + print("Processing shoonya BSE Data") + file_path = f'{output_path}/BSE_symbols.txt' + + # Read the BSE symbols file + df = pd.read_csv(file_path) + + # Read the BSE symbols file, specifying the exact columns to use and ignoring extra columns + df = pd.read_csv(file_path, usecols=['Exchange', 'Token', 'LotSize', 'Symbol', 'TradingSymbol', 'Instrument', 'TickSize']) + + # Rename columns to match your schema + df.columns = ['exchange', 'token', 'lotsize', 'name', 'brsymbol', 'instrumenttype', 'tick_size'] + + + # Add missing columns to ensure DataFrame matches the database structure + df['symbol'] = df['brsymbol'] # Initialize 'symbol' with 'brsymbol' + + # Apply transformation for OpenAlgo symbols (no special logic needed here) + def get_openalgo_symbol(broker_symbol): + return broker_symbol + + # Update the 'symbol' column + df['symbol'] = df['brsymbol'].apply(get_openalgo_symbol) + + # Set Exchange: 'BSE' for all rows + df['exchange'] = 'BSE' + df['brexchange'] = df['exchange'] # Broker exchange is the same as exchange + + # Set expiry and strike, fill -1 for missing strike prices + df['expiry'] = '' # No expiry for these instruments + df['strike'] = -1 # Default to -1 for strike price + + # Ensure the instrument type is always 'EQ' + df['instrumenttype'] = 'EQ' + + # Handle missing or invalid numeric values in 'lotsize' and 'tick_size' + df['lotsize'] = pd.to_numeric(df['lotsize'], errors='coerce').fillna(0).astype(int) # Convert to int, default to 0 + df['tick_size'] = pd.to_numeric(df['tick_size'], errors='coerce').fillna(0).astype(float) # Convert to float, default to 0.0 + + # Reorder the columns to match the database structure + columns_to_keep = ['symbol', 'brsymbol', 'name', 'exchange', 'brexchange', 'token', 'expiry', 'strike', 'lotsize', 'instrumenttype', 'tick_size'] + df_filtered = df[columns_to_keep] + + # Return the processed DataFrame + return df_filtered + +def process_shoonya_bfo_data(output_path): + """ + Processes the shoonya BFO data (BFO_symbols.txt) to generate OpenAlgo symbols and correctly extract the name column. + Handles both futures and options formatting, ensuring strike prices are handled as either float or integer. + """ + print("Processing shoonya BFO Data") + file_path = f'{output_path}/BFO_symbols.txt' + + # Read the BFO symbols file, specifying the exact columns to use + df = pd.read_csv(file_path, usecols=['Exchange', 'Token', 'LotSize', 'Symbol', 'TradingSymbol', 'Expiry', 'Instrument', 'Strike', 'TickSize']) + + # Rename columns to match your schema + df.columns = ['exchange', 'token', 'lotsize', 'name', 'brsymbol', 'expiry', 'instrumenttype', 'strike', 'tick_size'] + + # Add missing columns to ensure DataFrame matches the database structure + df['expiry'] = df['expiry'].fillna('') # Fill expiry with empty strings if missing + df['strike'] = df['strike'].fillna('-1') # Fill strike with -1 if missing + + # Define a function to format the expiry date as DDMMMYY + def format_expiry_date(date_str): + try: + return datetime.strptime(date_str, '%d-%b-%Y').strftime('%d%b%y').upper() + except ValueError: + print(f"Invalid expiry date format: {date_str}") + return None + + # Apply the expiry date format + df['expiry'] = df['expiry'].apply(format_expiry_date) + + # Extract the 'name' from the 'TradingSymbol' + def extract_name(tradingsymbol): + import re + match = re.match(r'([A-Za-z]+)', tradingsymbol) + return match.group(1) if match else tradingsymbol + + # Apply name extraction + df['name'] = df['brsymbol'].apply(extract_name) + + # Extract the instrument type (CE, PE, FUT) from TradingSymbol + def extract_instrument_type(tradingsymbol): + if tradingsymbol.endswith('FUT'): + return 'FUT' + elif tradingsymbol.endswith('CE'): + return 'CE' + elif tradingsymbol.endswith('PE'): + return 'PE' + else: + return 'UNKNOWN' # Handle cases where the suffix is not FUT, CE, or PE + + # Apply instrument type extraction + df['instrumenttype'] = df['brsymbol'].apply(extract_instrument_type) + + # Ensure strike prices are handled as either float or int + def handle_strike_price(strike): + try: + if float(strike).is_integer(): + return int(float(strike)) # Return as integer if no decimal + else: + return float(strike) # Return as float if decimal exists + except (ValueError, TypeError): + return -1 # If there's an error or it's empty, return -1 + + df['strike'] = df['strike'].apply(handle_strike_price) + + # Format the symbol column based on the instrument type and correctly handle the strike price + def format_symbol(row): + if row['instrumenttype'] == 'FUT': + return f"{row['name']}{row['expiry']}FUT" + else: + # Correctly format the strike price based on whether it's an integer or a float + formatted_strike = f"{int(row['strike'])}" if isinstance(row['strike'], int) else f"{row['strike']:.2f}".rstrip('0').rstrip('.') + return f"{row['name']}{row['expiry']}{formatted_strike}{row['instrumenttype']}" + + # Apply the symbol format + df['symbol'] = df.apply(format_symbol, axis=1) + + # Define Exchange and Broker Exchange + df['exchange'] = 'BFO' + df['brexchange'] = df['exchange'] + + # Reorder the columns to match the database structure + columns_to_keep = ['symbol', 'brsymbol', 'name', 'exchange', 'brexchange', 'token', 'expiry', 'strike', 'lotsize', 'instrumenttype', 'tick_size'] + df_filtered = df[columns_to_keep] + + # Return the processed DataFrame + return df_filtered + +def delete_shoonya_temp_data(output_path): + """ + Deletes the shoonya symbol files from the tmp folder after processing. + """ + for filename in os.listdir(output_path): + file_path = os.path.join(output_path, filename) + if filename.endswith(".txt") and os.path.isfile(file_path): + os.remove(file_path) + print(f"Deleted {file_path}") + +def master_contract_download(): + """ + Downloads, processes, and deletes shoonya data. + """ + print("Downloading shoonya Master Contract") + + output_path = 'tmp' + try: + download_and_unzip_shoonya_data(output_path) + delete_symtoken_table() + + # Placeholders for processing different exchanges + token_df = process_shoonya_nse_data(output_path) + copy_from_dataframe(token_df) + token_df = process_shoonya_bse_data(output_path) + copy_from_dataframe(token_df) + token_df = process_shoonya_nfo_data(output_path) + copy_from_dataframe(token_df) + token_df = process_shoonya_cds_data(output_path) + copy_from_dataframe(token_df) + token_df = process_shoonya_mcx_data(output_path) + copy_from_dataframe(token_df) + token_df = process_shoonya_bfo_data(output_path) + copy_from_dataframe(token_df) + + delete_shoonya_temp_data(output_path) + + return socketio.emit('master_contract_download', {'status': 'success', 'message': 'Successfully Downloaded'}) + except Exception as e: + print(str(e)) + return socketio.emit('master_contract_download', {'status': 'error', 'message': str(e)}) \ No newline at end of file diff --git a/broker/shoonya/mapping/order_data.py b/broker/shoonya/mapping/order_data.py new file mode 100644 index 00000000..0c9b963f --- /dev/null +++ b/broker/shoonya/mapping/order_data.py @@ -0,0 +1,392 @@ +import json +from database.token_db import get_symbol, get_oa_symbol + +def map_order_data(order_data): + """ + Processes and modifies a list of order dictionaries based on specific conditions. + + Parameters: + - order_data: A list of dictionaries, where each dictionary represents an order. + + Returns: + - The modified order_data with updated 'tradingsymbol' and 'product' fields. + """ + # Check if 'data' is None + if order_data is None or (isinstance(order_data, dict) and (order_data['stat'] == "Not_Ok")): + # Handle the case where there is no data + # For example, you might want to display a message to the user + # or pass an empty list or dictionary to the template. + print("No data available.") + order_data = {} # or set it to an empty list if it's supposed to be a list + else: + order_data = order_data + + + + if order_data: + for order in order_data: + # Extract the instrument_token and exchange for the current order + symboltoken = order['token'] + exchange = order['exch'] + + # Use the get_symbol function to fetch the symbol from the database + symbol_from_db = get_symbol(symboltoken, exchange) + + # Check if a symbol was found; if so, update the trading_symbol in the current order + if symbol_from_db: + order['tsym'] = symbol_from_db + if (order['exch'] == 'NSE' or order['exch'] == 'BSE') and order['prd'] == 'C': + order['prd'] = 'CNC' + + elif order['prd'] == 'I': + order['prd'] = 'MIS' + + elif order['exch'] in ['NFO', 'MCX', 'BFO', 'CDS'] and order['prd'] == 'M': + order['prd'] = 'NRML' + + if(order['prctyp']=="MKT"): + order['prctyp']="MARKET" + elif(order['prctyp']=="LMT"): + order['prctyp']="LIMIT" + elif(order['prctyp']=="SL-MKT"): + order['prctyp']="SL-M" + elif(order['prctyp']=="SL-LMT"): + order['prctyp']="SL" + + else: + print(f"Symbol not found for token {symboltoken} and exchange {exchange}. Keeping original trading symbol.") + + return order_data + + +def calculate_order_statistics(order_data): + """ + Calculates statistics from order data, including totals for buy orders, sell orders, + completed orders, open orders, and rejected orders. + + Parameters: + - order_data: A list of dictionaries, where each dictionary represents an order. + + Returns: + - A dictionary containing counts of different types of orders. + """ + # Initialize counters + total_buy_orders = total_sell_orders = 0 + total_completed_orders = total_open_orders = total_rejected_orders = 0 + + if order_data: + for order in order_data: + # Count buy and sell orders + if order['trantype'] == 'B': + order['trantype'] = 'BUY' + total_buy_orders += 1 + elif order['trantype'] == 'S': + order['trantype'] = 'SELL' + total_sell_orders += 1 + + # Count orders based on their status + if order['status'] == 'COMPLETE': + total_completed_orders += 1 + elif order['status'] == 'OPEN': + total_open_orders += 1 + elif order['status'] == 'REJECTED': + total_rejected_orders += 1 + + # Compile and return the statistics + return { + 'total_buy_orders': total_buy_orders, + 'total_sell_orders': total_sell_orders, + 'total_completed_orders': total_completed_orders, + 'total_open_orders': total_open_orders, + 'total_rejected_orders': total_rejected_orders + } + + +def transform_order_data(orders): + + + transformed_orders = [] + + for order in orders: + # Make sure each item is indeed a dictionary + if not isinstance(order, dict): + print(f"Warning: Expected a dict, but found a {type(order)}. Skipping this item.") + continue + + transformed_order = { + "symbol": order.get("tsym", ""), + "exchange": order.get("exch", ""), + "action": order.get("trantype", ""), + "quantity": order.get("qty", 0), + "price": order.get("prc", 0.0), + "trigger_price": order.get("trgprc", 0.0), + "pricetype": order.get("prctyp", ""), + "product": order.get("prd", ""), + "orderid": order.get("norenordno", ""), + "order_status": order.get("status", "").lower(), + "timestamp": order.get("norentm", "") + } + + transformed_orders.append(transformed_order) + + return transformed_orders + + + +def map_trade_data(trade_data): + """ + Processes and modifies a list of order dictionaries based on specific conditions. + + Parameters: + - order_data: A list of dictionaries, where each dictionary represents an order. + + Returns: + - The modified order_data with updated 'tradingsymbol' and 'product' fields. + """ + # Check if 'data' is None + if trade_data is None or (isinstance(trade_data, dict) and (trade_data['stat'] == "Not_Ok")): + # Handle the case where there is no data + # For example, you might want to display a message to the user + # or pass an empty list or dictionary to the template. + print("No data available.") + trade_data = {} # or set it to an empty list if it's supposed to be a list + else: + trade_data = trade_data + + + + if trade_data: + for order in trade_data: + # Extract the instrument_token and exchange for the current order + symbol = order['tsym'] + exchange = order['exch'] + + # Use the get_symbol function to fetch the symbol from the database + symbol_from_db = get_oa_symbol(symbol, exchange) + + # Check if a symbol was found; if so, update the trading_symbol in the current order + if symbol_from_db: + order['tsym'] = symbol_from_db + if (order['exch'] == 'NSE' or order['exch'] == 'BSE') and order['prd'] == 'C': + order['prd'] = 'CNC' + + elif order['prd'] == 'I': + order['prd'] = 'MIS' + + elif order['exch'] in ['NFO', 'MCX', 'BFO', 'CDS'] and order['prd'] == 'M': + order['prd'] = 'NRML' + + if(order['trantype']=="B"): + order['trantype']="BUY" + elif(order['trantype']=="S"): + order['trantype']="SELL" + + + else: + print(f"Unable to find the symbol {symbol} and exchange {exchange}. Keeping original trading symbol.") + + return trade_data + + + + +def transform_tradebook_data(tradebook_data): + transformed_data = [] + for trade in tradebook_data: + transformed_trade = { + "symbol": trade.get('tsym', ''), + "exchange": trade.get('exch', ''), + "product": trade.get('prd', ''), + "action": trade.get('trantype', ''), + "quantity": trade.get('qty', 0), + "average_price": trade.get('avgprc', 0.0), + "trade_value": float(trade.get('avgprc', 0)) * int(trade.get('qty', 0)), + "orderid": trade.get('norenordno', ''), + "timestamp": trade.get('norentm', '') + } + transformed_data.append(transformed_trade) + return transformed_data + + +def map_position_data(position_data): + + if position_data is None or (isinstance(position_data, dict) and (position_data['stat'] == "Not_Ok")): + # Handle the case where there is no data + # For example, you might want to display a message to the user + # or pass an empty list or dictionary to the template. + print("No data available.") + position_data = {} # or set it to an empty list if it's supposed to be a list + else: + position_data = position_data + + + + if position_data: + for order in position_data: + # Extract the instrument_token and exchange for the current order + symbol = order['tsym'] + exchange = order['exch'] + + # Use the get_symbol function to fetch the symbol from the database + symbol_from_db = get_oa_symbol(symbol, exchange) + + # Check if a symbol was found; if so, update the trading_symbol in the current order + if symbol_from_db: + order['tsym'] = symbol_from_db + if (order['exch'] == 'NSE' or order['exch'] == 'BSE') and order['prd'] == 'C': + order['prd'] = 'CNC' + + elif order['prd'] == 'I': + order['prd'] = 'MIS' + + elif order['exch'] in ['NFO', 'MCX', 'BFO', 'CDS'] and order['prd'] == 'M': + order['prd'] = 'NRML' + + + + + else: + print(f"Unable to find the symbol {symbol} and exchange {exchange}. Keeping original trading symbol.") + + return position_data + + +def transform_positions_data(positions_data): + transformed_data = [] + for position in positions_data: + transformed_position = { + "symbol": position.get('tsym', ''), + "exchange": position.get('exch', ''), + "product": position.get('prd', ''), + "quantity": position.get('netqty', 0), + "average_price": position.get('netavgprc', 0.0), + } + transformed_data.append(transformed_position) + return transformed_data + +def map_portfolio_data(portfolio_data): + """ + Processes and modifies a list of Portfolio dictionaries based on specific conditions and + ensures both holdings and totalholding parts are transmitted in a single response. + + Parameters: + - portfolio_data: A list of dictionaries, where each dictionary represents portfolio information. + + Returns: + - The modified portfolio_data with 'product' fields changed for 'holdings' and 'totalholding' included. + """ + # Check if 'portfolio_data' is a list + if not portfolio_data or not isinstance(portfolio_data, list): + print("No data available or incorrect data format.") + return [] + + # Iterate over the portfolio_data list and process each entry + for portfolio in portfolio_data: + # Ensure 'stat' is 'Ok' before proceeding + if portfolio.get('stat') != 'Ok': + print(f"Error: {portfolio.get('emsg', 'Unknown error occurred.')}") + continue + + # Process the 'exch_tsym' list inside each portfolio entry + for exch_tsym in portfolio.get('exch_tsym', []): + symbol = exch_tsym.get('tsym', '') + exchange = exch_tsym.get('exch', '') + + # Replace 'get_oa_symbol' function with your actual symbol fetching logic + symbol_from_db = get_oa_symbol(symbol, exchange) + + if symbol_from_db: + exch_tsym['tsym'] = symbol_from_db + else: + print(f"Shoonya Portfolio - Product Value for {symbol} Not Found or Changed.") + + return portfolio_data + +def calculate_portfolio_statistics(holdings_data): + totalholdingvalue = 0 + totalinvvalue = 0 + totalprofitandloss = 0 + totalpnlpercentage = 0 + + # Check if the data is valid or contains an error + if not holdings_data or not isinstance(holdings_data, list): + print("Error: Invalid or missing holdings data.") + return { + 'totalholdingvalue': totalholdingvalue, + 'totalinvvalue': totalinvvalue, + 'totalprofitandloss': totalprofitandloss, + 'totalpnlpercentage': totalpnlpercentage + } + + # Iterate over the list of holdings + for holding in holdings_data: + # Ensure 'stat' is 'Ok' before proceeding + if holding.get('stat') != 'Ok': + print(f"Error: {holding.get('emsg', 'Unknown error occurred.')}") + continue + + # Filter out the NSE entry and ignore BSE for the same symbol + nse_entry = next((exch for exch in holding.get('exch_tsym', []) if exch.get('exch') == 'NSE'), None) + if not nse_entry: + continue # Skip if no NSE entry is found + + # Process only the NSE entry + quantity = float(holding.get('holdqty', 0)) + max(float(holding.get('npoadt1qty', 0)) , float(holding.get('dpqty', 0))) + upload_price = float(holding.get('upldprc', 0)) + market_price = float(nse_entry.get('upldprc', 0)) # Assuming 'pp' is the market price for NSE + + # Calculate investment value and holding value for NSE + inv_value = quantity * upload_price + holding_value = quantity * upload_price + profit_and_loss = holding_value - inv_value + pnl_percentage = (profit_and_loss / inv_value) * 100 if inv_value != 0 else 0 + + # Accumulate the totals + #totalholdingvalue += holding_value + totalinvvalue += inv_value + totalprofitandloss += profit_and_loss + + # Valuation formula from API + holdqty = float(holding.get('holdqty', 0)) + btstqty = float(holding.get('btstqty', 0)) + brkcolqty = float(holding.get('brkcolqty', 0)) + unplgdqty = float(holding.get('unplgdqty', 0)) + benqty = float(holding.get('benqty', 0)) + npoadqty = float(holding.get('npoadt1qty', 0)) + dpqty = float(holding.get('dpqty', 0)) + usedqty = float(holding.get('usedqty', 0)) + + # Valuation formula from API + valuation = ((btstqty + holdqty + brkcolqty + unplgdqty + benqty + max(npoadqty, dpqty)) - usedqty)*upload_price + print("test valuation :"+str(npoadqty)) + print("test valuation :"+str(upload_price)) + # Accumulate total valuation + totalholdingvalue += valuation + + # Calculate overall P&L percentage + totalpnlpercentage = (totalprofitandloss / totalinvvalue) * 100 if totalinvvalue != 0 else 0 + + return { + 'totalholdingvalue': totalholdingvalue, + 'totalinvvalue': totalinvvalue, + 'totalprofitandloss': totalprofitandloss, + 'totalpnlpercentage': totalpnlpercentage + } + +def transform_holdings_data(holdings_data): + transformed_data = [] + if isinstance(holdings_data, list): + for holding in holdings_data: + # Filter out only NSE exchange + nse_entries = [exch for exch in holding.get('exch_tsym', []) if exch.get('exch') == 'NSE'] + for exch_tsym in nse_entries: + transformed_position = { + "symbol": exch_tsym.get('tsym', ''), + "exchange": exch_tsym.get('exch', ''), + "quantity": int(holding.get('holdqty', 0)) + max(int(holding.get('npoadt1qty', 0)) , int(holding.get('dpqty', 0))), + "product": exch_tsym.get('product', 'CNC'), + "pnl": holding.get('profitandloss', 0.0), + "pnlpercent": holding.get('pnlpercentage', 0.0) + } + transformed_data.append(transformed_position) + return transformed_data \ No newline at end of file diff --git a/broker/shoonya/mapping/transform_data.py b/broker/shoonya/mapping/transform_data.py new file mode 100644 index 00000000..c34d0f6a --- /dev/null +++ b/broker/shoonya/mapping/transform_data.py @@ -0,0 +1,88 @@ +#Mapping OpenAlgo API Request https://openalgo.in/docs +#Mapping Shoonya Broking Parameters https://shoonya.com/api-documentation + +from database.token_db import get_br_symbol + +def transform_data(data,token): + """ + Transforms the new API request structure to the current expected structure. + """ + symbol = get_br_symbol(data["symbol"],data["exchange"]) + # Basic mapping + transformed = { + "uid": data["apikey"], + "actid": data["apikey"], + "exch": data["exchange"], + "tsym": symbol, + "qty": data["quantity"], + "prc": data.get("price", "0"), + "trgprc": data.get("trigger_price", "0"), + "dscqty": data.get("disclosed_quantity", "0"), + "prd": map_product_type(data["product"]), + "trantype": 'B' if data["action"] == "BUY" else 'S', + "prctyp": map_order_type(data["pricetype"]), + "mkt_protection": "0", + "ret": "DAY", + "ordersource": "API" + + } + + + + + return transformed + + +def transform_modify_order_data(data, token): + return { + "exch": data["exchange"], + "norenordno": data["orderid"], + "prctyp": map_order_type(data["pricetype"]), + "prc": data["price"], + "qty": data["quantity"], + "tsym": data["symbol"], + "ret": "DAY", + "mkt_protection": "0", + "trdprc": data.get("trigger_price", "0"), + "dscqty": data.get("disclosed_quantity", "0"), + "uid": data["apikey"] + } + + + +def map_order_type(pricetype): + """ + Maps the new pricetype to the existing order type. + """ + order_type_mapping = { + "MARKET": "MKT", + "LIMIT": "LMT", + "SL": "SL-LMT", + "SL-M": "SL-MKT" + } + return order_type_mapping.get(pricetype, "MARKET") # Default to MARKET if not found + +def map_product_type(product): + """ + Maps the new product type to the existing product type. + """ + product_type_mapping = { + "CNC": "C", + "NRML": "M", + "MIS": "I", + } + return product_type_mapping.get(product, "I") # Default to DELIVERY if not found + + + +def reverse_map_product_type(product): + """ + Maps the new product type to the existing product type. + """ + reverse_product_type_mapping = { + "C": "CNC", + "M": "NRML", + "I": "MIS", + } + return reverse_product_type_mapping.get(product) + diff --git a/broker/shoonya/plugin.json b/broker/shoonya/plugin.json new file mode 100644 index 00000000..a66fb559 --- /dev/null +++ b/broker/shoonya/plugin.json @@ -0,0 +1,8 @@ +{ + "Plugin Name": "Shoonya", + "Plugin URI": "https://openalgo.in", + "Description": "Shoonya OpenAlgo Plugin", + "Version": "1.0", + "Author": "Rajandran R", + "Author URI": "https://openalgo.in" +} \ No newline at end of file diff --git a/templates/shoonya.html b/templates/shoonya.html new file mode 100644 index 00000000..7b54fc31 --- /dev/null +++ b/templates/shoonya.html @@ -0,0 +1,134 @@ +{% extends "layout.html" %} + +{% block title %}Shoonya Authentication - OpenAlgo{% endblock %} + +{% block content %} +
+
+
+ +
+
+
+ OpenAlgo +
+ +
+
+ + + +
+ +
+ + + +
+ +
+ + + +
+ + {% if error_message %} +
+ + + + {{ error_message }} +
+ {% endif %} + +
+ +
+
+ +
OR
+ + + + + + Back to Broker Selection + +
+
+ + +
+

Connect Shoonya

+

+ Enter your Shoonya credentials to connect your trading account with OpenAlgo. You can use TOTP, Date of Birth, or PAN for verification. +

+
+
+ + + +
+

Verification Options

+
You can use TOTP, Date of Birth (DOB), or PAN for verification
+
+
+ +
+
+
+
+
+{% endblock %} From 1954fa4ad0653ea229fb35a8c15c31c5b0ab1a35 Mon Sep 17 00:00:00 2001 From: marketcalls Date: Sat, 30 Nov 2024 22:43:55 +0530 Subject: [PATCH 02/41] shoonya updated numpy and dummy imei value --- broker/shoonya/api/auth_api.py | 1 + requirements-nginx.txt | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/broker/shoonya/api/auth_api.py b/broker/shoonya/api/auth_api.py index ddf3b521..77d2deaa 100644 --- a/broker/shoonya/api/auth_api.py +++ b/broker/shoonya/api/auth_api.py @@ -15,6 +15,7 @@ def authenticate_broker(userid, password, totp_code): api_secretkey = os.getenv('BROKER_API_SECRET') vendor_code = os.getenv('BROKER_API_KEY') imei = '1234567890abcdef' # Default IMEI if not provided + #imei = 'abc1234' # Default IMEI if not provided try: # Shoonya API login URL diff --git a/requirements-nginx.txt b/requirements-nginx.txt index b700af8c..5c17798a 100644 --- a/requirements-nginx.txt +++ b/requirements-nginx.txt @@ -32,7 +32,7 @@ markdown-it-py==3.0.0 MarkupSafe==2.1.5 marshmallow==3.22.0 mdurl==0.1.2 -numpy==1.26.1 +numpy==1.26.2 openalgo==1.0.2 ordered-set==4.1.0 packaging==24.1 diff --git a/requirements.txt b/requirements.txt index a1f8729f..8b1e492b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,7 +32,7 @@ markdown-it-py==3.0.0 MarkupSafe==2.1.5 marshmallow==3.22.0 mdurl==0.1.2 -numpy==1.26.1 +numpy==1.26.2 openalgo==1.0.2 ordered-set==4.1.0 packaging==24.1 From 767ed827df62e5a6b5b91f71ba5c1ef85fcce3a9 Mon Sep 17 00:00:00 2001 From: marketcalls Date: Sat, 30 Nov 2024 23:07:27 +0530 Subject: [PATCH 03/41] fixing shoonya callback, imei and adding shoonya broker to the auth list --- blueprints/auth.py | 2 +- broker/shoonya/api/auth_api.py | 4 ++-- templates/broker.html | 4 ++++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/blueprints/auth.py b/blueprints/auth.py index 771edbd6..32f940d1 100644 --- a/blueprints/auth.py +++ b/blueprints/auth.py @@ -86,7 +86,7 @@ def broker_login(): # Validate broker name is one of the supported brokers valid_brokers = {'fivepaisa', 'aliceblue', 'angel', 'dhan', 'fyers', - 'icici', 'kotak', 'upstox', 'zebu', 'zerodha'} + 'icici', 'kotak', 'shoonya','upstox', 'zebu', 'zerodha'} if broker_name not in valid_brokers: raise ValueError(f"Invalid broker name: {broker_name}") diff --git a/broker/shoonya/api/auth_api.py b/broker/shoonya/api/auth_api.py index 77d2deaa..f03b5b47 100644 --- a/broker/shoonya/api/auth_api.py +++ b/broker/shoonya/api/auth_api.py @@ -14,8 +14,8 @@ def authenticate_broker(userid, password, totp_code): # Get the Shoonya API key and other credentials from environment variables api_secretkey = os.getenv('BROKER_API_SECRET') vendor_code = os.getenv('BROKER_API_KEY') - imei = '1234567890abcdef' # Default IMEI if not provided - #imei = 'abc1234' # Default IMEI if not provided + #imei = '1234567890abcdef' # Default IMEI if not provided + imei = 'abc1234' # Default IMEI if not provided try: # Shoonya API login URL diff --git a/templates/broker.html b/templates/broker.html index ba68e0a8..8ce6d1a2 100644 --- a/templates/broker.html +++ b/templates/broker.html @@ -40,6 +40,9 @@ case 'kotak': loginUrl = '/kotak/callback'; break; + case 'shoonya': + loginUrl = '/shoonya/callback'; + break; case 'upstox': loginUrl = 'https://api.upstox.com/v2/login/authorization/dialog?response_type=code&client_id={{broker_api_key}}&redirect_uri={{ redirect_url }}'; break; @@ -86,6 +89,7 @@

Connect Your Trading + From dce42aebf913f0a77e03852bbeadae2da3574973 Mon Sep 17 00:00:00 2001 From: marketcalls Date: Sat, 30 Nov 2024 23:14:15 +0530 Subject: [PATCH 04/41] updating .env file --- .sample.env | 5 ++++- blueprints/auth.py | 10 +++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.sample.env b/.sample.env index dc4b9654..dde22c31 100644 --- a/.sample.env +++ b/.sample.env @@ -4,6 +4,9 @@ BROKER_API_SECRET = 'YOUR_BROKER_API_SECRET' REDIRECT_URL = 'http://127.0.0.1:5000//callback' # Change if different +# Valid Brokers Configuration +VALID_BROKERS = 'fivepaisa,aliceblue,angel,dhan,fyers,icici,kotak,shoonya,upstox,zebu,zerodha' + # OpenAlgo Application Key - Change the Key to Some Random Values APP_KEY = 'dfsd98sdf98dsfjk34ghuu85df' @@ -27,7 +30,7 @@ FLASK_DEBUG='False' FLASK_ENV='development' # OpenAlgo Flask App Version Management -FLASK_APP_VERSION='1.0.0.13' +FLASK_APP_VERSION='1.0.0.14' # OpenAlgo Rate Limit Settings LOGIN_RATE_LIMIT_MIN = "5 per minute" diff --git a/blueprints/auth.py b/blueprints/auth.py index 32f940d1..b6687314 100644 --- a/blueprints/auth.py +++ b/blueprints/auth.py @@ -84,9 +84,13 @@ def broker_login(): broker_name = match.group(1) - # Validate broker name is one of the supported brokers - valid_brokers = {'fivepaisa', 'aliceblue', 'angel', 'dhan', 'fyers', - 'icici', 'kotak', 'shoonya','upstox', 'zebu', 'zerodha'} + # Get valid brokers from environment variable + valid_brokers_str = os.getenv('VALID_BROKERS', '') + valid_brokers = set(valid_brokers_str.split(',')) if valid_brokers_str else set() + + if not valid_brokers: + raise ValueError("VALID_BROKERS not configured in .env file") + if broker_name not in valid_brokers: raise ValueError(f"Invalid broker name: {broker_name}") From 7f43f1638a221b7a595c5aca48ef9222c051f827 Mon Sep 17 00:00:00 2001 From: marketcalls Date: Sat, 30 Nov 2024 23:14:46 +0530 Subject: [PATCH 05/41] update .sample.env --- .sample.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.sample.env b/.sample.env index dde22c31..5ca8d7be 100644 --- a/.sample.env +++ b/.sample.env @@ -8,7 +8,7 @@ REDIRECT_URL = 'http://127.0.0.1:5000//callback' # Change if different VALID_BROKERS = 'fivepaisa,aliceblue,angel,dhan,fyers,icici,kotak,shoonya,upstox,zebu,zerodha' # OpenAlgo Application Key - Change the Key to Some Random Values -APP_KEY = 'dfsd98sdf98dsfjk34ghuu85df' +APP_KEY = 'dfsd98tyhgfrtk34ghuu85df' # OpenAlgo Database Configuration From 99e4776868c572fede205acba12f6bf2efc679a9 Mon Sep 17 00:00:00 2001 From: marketcalls Date: Sat, 30 Nov 2024 23:31:35 +0530 Subject: [PATCH 06/41] update supported broker --- templates/faq.html | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/faq.html b/templates/faq.html index d708ee30..48c38505 100644 --- a/templates/faq.html +++ b/templates/faq.html @@ -65,6 +65,7 @@

Important Links

  • Aliceblue
  • Fyers
  • ICICI Direct
  • +
  • Shoonya
  • Upstox
  • Zebu
  • Zerodha
  • From f4eb050d12e6490c941213afba3171a73e450044 Mon Sep 17 00:00:00 2001 From: marketcalls Date: Mon, 2 Dec 2024 16:08:17 +0530 Subject: [PATCH 07/41] revert to 99e4776 --- blueprints/auth.py | 40 ++++++++++++++++------------------------ 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/blueprints/auth.py b/blueprints/auth.py index b6687314..9800dfe5 100644 --- a/blueprints/auth.py +++ b/blueprints/auth.py @@ -6,7 +6,7 @@ from database.auth_db import upsert_auth from database.user_db import authenticate_user, User, db_session, find_user_by_username # Import the function import re -from utils.session import check_session_validity +from utils.session import check_session_validity, set_session_login_time # Load environment variables load_dotenv() @@ -25,12 +25,12 @@ def ratelimit_handler(e): @limiter.limit(LOGIN_RATE_LIMIT_MIN) @limiter.limit(LOGIN_RATE_LIMIT_HOUR) def login(): - if find_user_by_username() is None: return redirect(url_for('core_bp.setup')) - if 'user' in session: - return redirect(url_for('auth.broker_login')) + # Only redirect to broker_login if both user and logged_in are in session + if 'user' in session and session.get('logged_in'): + return redirect(url_for('auth.broker_login')) if session.get('logged_in'): return redirect(url_for('dashboard_bp.dashboard')) @@ -40,12 +40,12 @@ def login(): elif request.method == 'POST': username = request.form['username'] password = request.form['password'] - if authenticate_user(username, password): session['user'] = username # Set the username in the session + session['logged_in'] = True # Set logged_in flag + set_session_login_time() # Set the login timestamp print("login success") - # Redirect to broker login without marking as fully logged in return jsonify({'status': 'success'}), 200 else: return jsonify({'status': 'error', 'message': 'Invalid credentials'}), 401 @@ -54,8 +54,15 @@ def login(): @limiter.limit(LOGIN_RATE_LIMIT_MIN) @limiter.limit(LOGIN_RATE_LIMIT_HOUR) def broker_login(): - if session.get('logged_in'): + # First check if user is fully logged in + if not session.get('logged_in'): + session.clear() # Clear any partial session data + return redirect(url_for('auth.login')) + + # Then check if already at dashboard + if session.get('broker'): return redirect(url_for('dashboard_bp.dashboard')) + if request.method == 'GET': if 'user' not in session: return redirect(url_for('auth.login')) @@ -68,16 +75,8 @@ def broker_login(): flash('REDIRECT_URL is not configured in .env file', 'error') return redirect(url_for('auth.login')) - # Extract broker name from REDIRECT_URL - # Handles all valid formats: - # - http://127.0.0.1:5000/broker_name/callback - # - http://yourdomain.com/broker_name/callback - # - https://yourdomain.com/broker_name/callback - # - https://sub.yourdomain.com/broker_name/callback - # - http://sub.yourdomain.com/broker_name/callback try: # This pattern looks for the broker name between the last two forward slashes - # It works regardless of the domain format or protocol match = re.search(r'/([^/]+)/callback$', REDIRECT_URL) if not match: raise ValueError("Invalid URL format") @@ -116,7 +115,6 @@ def broker_login(): @check_session_validity def change_password(): if 'user' not in session: - # If the user is not logged in, redirect to login page flash('You must be logged in to change your password.', 'warning') return redirect(url_for('auth.login')) @@ -130,18 +128,14 @@ def change_password(): if user and user.check_password(old_password): if new_password == confirm_password: - # Here, you should also ensure the new password meets your policy before updating user.set_password(new_password) db_session.commit() - # Use flash to notify the user of success flash('Your password has been changed successfully.', 'success') - # Redirect to a page where the user can see this confirmation, or stay on the same page return redirect(url_for('auth.change_password')) else: flash('New password and confirm password do not match.', 'error') else: flash('Old Password is incorrect.', 'error') - # Optionally, redirect to the same page to let the user try again return redirect(url_for('auth.change_password')) return render_template('profile.html', username=session['user']) @@ -161,10 +155,8 @@ def logout(): else: print("Failed to upsert auth token") - # Remove tokens and user information from session - session.pop('user', None) # Remove 'user' from session if exists - session.pop('broker', None) # Remove 'user' from session if exists - session.pop('logged_in', None) + # Clear all session data + session.clear() # Redirect to login page after logout return redirect(url_for('auth.login')) From a3248734cd2c3c1fe4323b35567652b1e8c6d3b1 Mon Sep 17 00:00:00 2001 From: marketcalls Date: Mon, 2 Dec 2024 16:11:32 +0530 Subject: [PATCH 08/41] revert auth.py --- blueprints/auth.py | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/blueprints/auth.py b/blueprints/auth.py index 9800dfe5..d17000e5 100644 --- a/blueprints/auth.py +++ b/blueprints/auth.py @@ -6,7 +6,7 @@ from database.auth_db import upsert_auth from database.user_db import authenticate_user, User, db_session, find_user_by_username # Import the function import re -from utils.session import check_session_validity, set_session_login_time +from utils.session import check_session_validity # Load environment variables load_dotenv() @@ -25,12 +25,12 @@ def ratelimit_handler(e): @limiter.limit(LOGIN_RATE_LIMIT_MIN) @limiter.limit(LOGIN_RATE_LIMIT_HOUR) def login(): + if find_user_by_username() is None: return redirect(url_for('core_bp.setup')) - # Only redirect to broker_login if both user and logged_in are in session - if 'user' in session and session.get('logged_in'): - return redirect(url_for('auth.broker_login')) + if 'user' in session: + return redirect(url_for('auth.broker_login')) if session.get('logged_in'): return redirect(url_for('dashboard_bp.dashboard')) @@ -40,12 +40,12 @@ def login(): elif request.method == 'POST': username = request.form['username'] password = request.form['password'] + if authenticate_user(username, password): session['user'] = username # Set the username in the session - session['logged_in'] = True # Set logged_in flag - set_session_login_time() # Set the login timestamp print("login success") + # Redirect to broker login without marking as fully logged in return jsonify({'status': 'success'}), 200 else: return jsonify({'status': 'error', 'message': 'Invalid credentials'}), 401 @@ -54,15 +54,8 @@ def login(): @limiter.limit(LOGIN_RATE_LIMIT_MIN) @limiter.limit(LOGIN_RATE_LIMIT_HOUR) def broker_login(): - # First check if user is fully logged in - if not session.get('logged_in'): - session.clear() # Clear any partial session data - return redirect(url_for('auth.login')) - - # Then check if already at dashboard - if session.get('broker'): + if session.get('logged_in'): return redirect(url_for('dashboard_bp.dashboard')) - if request.method == 'GET': if 'user' not in session: return redirect(url_for('auth.login')) @@ -75,8 +68,16 @@ def broker_login(): flash('REDIRECT_URL is not configured in .env file', 'error') return redirect(url_for('auth.login')) + # Extract broker name from REDIRECT_URL + # Handles all valid formats: + # - http://127.0.0.1:5000/broker_name/callback + # - http://yourdomain.com/broker_name/callback + # - https://yourdomain.com/broker_name/callback + # - https://sub.yourdomain.com/broker_name/callback + # - http://sub.yourdomain.com/broker_name/callback try: # This pattern looks for the broker name between the last two forward slashes + # It works regardless of the domain format or protocol match = re.search(r'/([^/]+)/callback$', REDIRECT_URL) if not match: raise ValueError("Invalid URL format") @@ -115,6 +116,7 @@ def broker_login(): @check_session_validity def change_password(): if 'user' not in session: + # If the user is not logged in, redirect to login page flash('You must be logged in to change your password.', 'warning') return redirect(url_for('auth.login')) @@ -128,14 +130,18 @@ def change_password(): if user and user.check_password(old_password): if new_password == confirm_password: + # Here, you should also ensure the new password meets your policy before updating user.set_password(new_password) db_session.commit() + # Use flash to notify the user of success flash('Your password has been changed successfully.', 'success') + # Redirect to a page where the user can see this confirmation, or stay on the same page return redirect(url_for('auth.change_password')) else: flash('New password and confirm password do not match.', 'error') else: flash('Old Password is incorrect.', 'error') + # Optionally, redirect to the same page to let the user try again return redirect(url_for('auth.change_password')) return render_template('profile.html', username=session['user']) @@ -155,8 +161,10 @@ def logout(): else: print("Failed to upsert auth token") - # Clear all session data - session.clear() + # Remove tokens and user information from session + session.pop('user', None) # Remove 'user' from session if exists + session.pop('broker', None) # Remove 'user' from session if exists + session.pop('logged_in', None) # Redirect to login page after logout - return redirect(url_for('auth.login')) + return redirect(url_for('auth.login')) \ No newline at end of file From 16002f7efee44c71992d3b3e90fa4ad6157be603 Mon Sep 17 00:00:00 2001 From: marketcalls Date: Tue, 3 Dec 2024 01:20:46 +0530 Subject: [PATCH 09/41] handling wrong REDIRECT URL --- blueprints/auth.py | 48 +++---------------------------------- utils/env_check.py | 60 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 62 insertions(+), 46 deletions(-) diff --git a/blueprints/auth.py b/blueprints/auth.py index d17000e5..de99f676 100644 --- a/blueprints/auth.py +++ b/blueprints/auth.py @@ -25,7 +25,6 @@ def ratelimit_handler(e): @limiter.limit(LOGIN_RATE_LIMIT_MIN) @limiter.limit(LOGIN_RATE_LIMIT_HOUR) def login(): - if find_user_by_username() is None: return redirect(url_for('core_bp.setup')) @@ -41,7 +40,6 @@ def login(): username = request.form['username'] password = request.form['password'] - if authenticate_user(username, password): session['user'] = username # Set the username in the session print("login success") @@ -60,51 +58,11 @@ def broker_login(): if 'user' not in session: return redirect(url_for('auth.login')) + # Get broker configuration (already validated at startup) BROKER_API_KEY = os.getenv('BROKER_API_KEY') BROKER_API_SECRET = os.getenv('BROKER_API_SECRET') REDIRECT_URL = os.getenv('REDIRECT_URL') - - if not REDIRECT_URL: - flash('REDIRECT_URL is not configured in .env file', 'error') - return redirect(url_for('auth.login')) - - # Extract broker name from REDIRECT_URL - # Handles all valid formats: - # - http://127.0.0.1:5000/broker_name/callback - # - http://yourdomain.com/broker_name/callback - # - https://yourdomain.com/broker_name/callback - # - https://sub.yourdomain.com/broker_name/callback - # - http://sub.yourdomain.com/broker_name/callback - try: - # This pattern looks for the broker name between the last two forward slashes - # It works regardless of the domain format or protocol - match = re.search(r'/([^/]+)/callback$', REDIRECT_URL) - if not match: - raise ValueError("Invalid URL format") - - broker_name = match.group(1) - - # Get valid brokers from environment variable - valid_brokers_str = os.getenv('VALID_BROKERS', '') - valid_brokers = set(valid_brokers_str.split(',')) if valid_brokers_str else set() - - if not valid_brokers: - raise ValueError("VALID_BROKERS not configured in .env file") - - if broker_name not in valid_brokers: - raise ValueError(f"Invalid broker name: {broker_name}") - - except ValueError as e: - flash(f'Invalid REDIRECT_URL format in .env file: {str(e)}. Expected format examples:\n' - '- http://127.0.0.1:5000/broker_name/callback\n' - '- http://yourdomain.com/broker_name/callback\n' - '- https://yourdomain.com/broker_name/callback\n' - '- https://sub.yourdomain.com/broker_name/callback\n' - '- http://sub.yourdomain.com/broker_name/callback', 'error') - return redirect(url_for('auth.login')) - except Exception as e: - flash(f'Error processing REDIRECT_URL: {str(e)}', 'error') - return redirect(url_for('auth.login')) + broker_name = re.search(r'/([^/]+)/callback$', REDIRECT_URL).group(1) return render_template('broker.html', broker_api_key=BROKER_API_KEY, @@ -167,4 +125,4 @@ def logout(): session.pop('logged_in', None) # Redirect to login page after logout - return redirect(url_for('auth.login')) \ No newline at end of file + return redirect(url_for('auth.login')) diff --git a/utils/env_check.py b/utils/env_check.py index ebc13209..aa3fa6c0 100644 --- a/utils/env_check.py +++ b/utils/env_check.py @@ -28,4 +28,62 @@ def load_and_check_env_variables(): if missing_vars: missing_list = ', '.join(missing_vars) print(f"Error: The following environment variables are missing: {missing_list}") - sys.exit(1) \ No newline at end of file + sys.exit(1) + + # Check REDIRECT_URL configuration + redirect_url = os.getenv('REDIRECT_URL') + default_value = 'http://127.0.0.1:5000//callback' + + if redirect_url == default_value: + print("\nError: Default REDIRECT_URL detected in .env file.") + print("The application cannot start with the default configuration.") + print("\nPlease:") + print("1. Open your .env file") + print("2. Change the REDIRECT_URL to use your specific broker") + print("3. Save the file") + print("\nExample: If using Zerodha, change:") + print(f" REDIRECT_URL = '{default_value}'") + print("to:") + print(" REDIRECT_URL = 'http://127.0.0.1:5000/zerodha/callback'") + sys.exit(1) + + if '' in redirect_url: + print("\nError: Invalid REDIRECT_URL configuration detected.") + print("The application cannot start with '' in REDIRECT_URL.") + print("\nPlease update your .env file to use your specific broker name.") + print("Example: http://127.0.0.1:5000/zerodha/callback") + sys.exit(1) + + # Validate broker name + valid_brokers_str = os.getenv('VALID_BROKERS', '') + if not valid_brokers_str: + print("\nError: VALID_BROKERS not configured in .env file.") + print("\nSoluton: Check the .sample.env file latest configuration file") + print("The application cannot start without valid broker configuration.") + sys.exit(1) + + valid_brokers = set(broker.strip().lower() for broker in valid_brokers_str.split(',')) + + try: + import re + match = re.search(r'/([^/]+)/callback$', redirect_url) + if not match: + print("\nError: Invalid REDIRECT_URL format.") + print("The URL must end with '/broker_name/callback'") + print("Example: http://127.0.0.1:5000/zerodha/callback") + sys.exit(1) + + broker_name = match.group(1).lower() + if broker_name not in valid_brokers: + print("\nError: Invalid broker name in REDIRECT_URL.") + print(f"Broker '{broker_name}' is not in the list of valid brokers.") + print(f"\nValid brokers are: {', '.join(sorted(valid_brokers))}") + print("\nPlease update your REDIRECT_URL with a valid broker name.") + sys.exit(1) + + except Exception as e: + print("\nError: Could not validate REDIRECT_URL format.") + print(f"Details: {str(e)}") + print("\nThe URL must follow the format: http://domain/broker_name/callback") + print("Example: http://127.0.0.1:5000/zerodha/callback") + sys.exit(1) From 65e634eeb3ee4960535038ae14de155ec2f5d298 Mon Sep 17 00:00:00 2001 From: marketcalls Date: Tue, 3 Dec 2024 17:41:09 +0530 Subject: [PATCH 10/41] placeorder analyzer --- .sample.env | 2 + app.py | 14 +- blueprints/analyzer.py | 150 ++++++++++++++++++ database/analyzer_db.py | 82 ++++++++++ restx_api/place_order.py | 132 ++++++++++++---- static/js/socket-events.js | 301 +++++++++++++++++++++++-------------- templates/analyzer.html | 190 +++++++++++++++++++++++ templates/base.html | 8 + templates/navbar.html | 14 ++ utils/api_analyzer.py | 219 +++++++++++++++++++++++++++ utils/constants.py | 73 +++++++++ 11 files changed, 1039 insertions(+), 146 deletions(-) create mode 100644 blueprints/analyzer.py create mode 100644 database/analyzer_db.py create mode 100644 templates/analyzer.html create mode 100644 utils/api_analyzer.py create mode 100644 utils/constants.py diff --git a/.sample.env b/.sample.env index 5ca8d7be..f69033f2 100644 --- a/.sample.env +++ b/.sample.env @@ -10,6 +10,8 @@ VALID_BROKERS = 'fivepaisa,aliceblue,angel,dhan,fyers,icici,kotak,shoonya,upstox # OpenAlgo Application Key - Change the Key to Some Random Values APP_KEY = 'dfsd98tyhgfrtk34ghuu85df' +# API Analyzer Mode - Set to true to analyze requests without placing orders +ANALYZE_MODE = 'true' # OpenAlgo Database Configuration DATABASE_URL = 'sqlite:///db/openalgo.db' diff --git a/app.py b/app.py index 1c98b548..2c2f7123 100644 --- a/app.py +++ b/app.py @@ -16,7 +16,8 @@ from blueprints.log import log_bp from blueprints.tv_json import tv_json_bp from blueprints.brlogin import brlogin_bp -from blueprints.core import core_bp +from blueprints.core import core_bp +from blueprints.analyzer import analyzer_bp # Import the analyzer blueprint from restx_api import api_v1_bp @@ -24,6 +25,7 @@ from database.user_db import init_db as ensure_user_tables_exists from database.symbol import init_db as ensure_master_contract_tables_exists from database.apilog_db import init_db as ensure_api_log_tables_exists +from database.analyzer_db import init_db as ensure_analyzer_tables_exists from utils.plugin_loader import load_broker_auth_functions @@ -46,15 +48,10 @@ def create_app(): load_dotenv() - - # Environment variables app.secret_key = os.getenv('APP_KEY') app.config['SQLALCHEMY_DATABASE_URI'] = os.getenv('DATABASE_URL') # Adjust the environment variable name as necessary - # Initialize SQLAlchemy - # db.init_app(app) - # Register the blueprints app.register_blueprint(auth_bp) app.register_blueprint(dashboard_bp) @@ -64,7 +61,8 @@ def create_app(): app.register_blueprint(log_bp) app.register_blueprint(tv_json_bp) app.register_blueprint(brlogin_bp) - app.register_blueprint(core_bp) + app.register_blueprint(core_bp) + app.register_blueprint(analyzer_bp) # Register the analyzer blueprint # Register RESTx API blueprint app.register_blueprint(api_v1_bp) @@ -82,7 +80,6 @@ def inject_version(): def setup_environment(app): with app.app_context(): - #load broker plugins app.broker_auth_functions = load_broker_auth_functions() # Ensure all the tables exist @@ -90,6 +87,7 @@ def setup_environment(app): ensure_user_tables_exists() ensure_master_contract_tables_exists() ensure_api_log_tables_exists() + ensure_analyzer_tables_exists() # Conditionally setup ngrok in development environment if os.getenv('NGROK_ALLOW') == 'TRUE': diff --git a/blueprints/analyzer.py b/blueprints/analyzer.py new file mode 100644 index 00000000..d2578c0b --- /dev/null +++ b/blueprints/analyzer.py @@ -0,0 +1,150 @@ +from flask import Blueprint, render_template, jsonify, request, session, flash, redirect, url_for +from database.analyzer_db import AnalyzerLog, db_session +from utils.session import check_session_validity +from sqlalchemy import func, desc +from utils.api_analyzer import get_analyzer_stats +import json +from datetime import datetime, timedelta +import pytz +import logging +import traceback + +logger = logging.getLogger(__name__) + +analyzer_bp = Blueprint('analyzer_bp', __name__, url_prefix='/analyzer') + +def format_request(req, ist): + """Format a single request entry""" + try: + request_data = json.loads(req.request_data) if isinstance(req.request_data, str) else req.request_data + response_data = json.loads(req.response_data) if isinstance(req.response_data, str) else req.response_data + + return { + 'timestamp': req.created_at.astimezone(ist).strftime('%Y-%m-%d %H:%M:%S'), + 'source': request_data.get('strategy', 'Unknown'), + 'symbol': request_data.get('symbol', 'Unknown'), + 'exchange': request_data.get('exchange', 'Unknown'), + 'action': request_data.get('action', 'Unknown'), + 'quantity': request_data.get('quantity', 0), + 'price_type': request_data.get('price_type', 'Unknown'), + 'product_type': request_data.get('product_type', 'Unknown'), + 'request_data': request_data, + 'analysis': { + 'issues': response_data.get('status') == 'error', + 'error': response_data.get('message'), + 'error_type': 'error' if response_data.get('status') == 'error' else 'success', + 'warnings': response_data.get('warnings', []) + } + } + except Exception as e: + logger.error(f"Error formatting request {req.id}: {str(e)}") + return None + +def get_recent_requests(): + """Get recent analyzer requests""" + try: + ist = pytz.timezone('Asia/Kolkata') + recent = AnalyzerLog.query.order_by(AnalyzerLog.created_at.desc()).limit(100).all() + requests = [] + + for req in recent: + formatted = format_request(req, ist) + if formatted: + requests.append(formatted) + + return requests + except Exception as e: + logger.error(f"Error getting recent requests: {str(e)}") + return [] + +@analyzer_bp.route('/') +@check_session_validity +def analyzer(): + """Render the analyzer dashboard""" + try: + # Get stats with proper structure + stats = get_analyzer_stats() + if not isinstance(stats, dict): + stats = { + 'total_requests': 0, + 'sources': {}, + 'symbols': [], + 'issues': { + 'total': 0, + 'by_type': { + 'rate_limit': 0, + 'invalid_symbol': 0, + 'missing_quantity': 0, + 'invalid_exchange': 0, + 'other': 0 + } + } + } + + # Get recent requests + requests = get_recent_requests() + + # If AJAX request, return JSON + if request.headers.get('X-Requested-With') == 'XMLHttpRequest': + return jsonify({ + 'stats': stats, + 'requests': requests + }) + + return render_template('analyzer.html', stats=stats, requests=requests) + except Exception as e: + logger.error(f"Error rendering analyzer: {str(e)}\n{traceback.format_exc()}") + flash('Error loading analyzer dashboard', 'error') + return redirect(url_for('core_bp.home')) + +@analyzer_bp.route('/stats') +@check_session_validity +def get_stats(): + """Get analyzer stats endpoint""" + try: + stats = get_analyzer_stats() + return jsonify(stats) + except Exception as e: + logger.error(f"Error getting analyzer stats: {str(e)}") + return jsonify({ + 'total_requests': 0, + 'sources': {}, + 'symbols': [], + 'issues': { + 'total': 0, + 'by_type': { + 'rate_limit': 0, + 'invalid_symbol': 0, + 'missing_quantity': 0, + 'invalid_exchange': 0, + 'other': 0 + } + } + }), 500 + +@analyzer_bp.route('/requests') +@check_session_validity +def get_requests(): + """Get analyzer requests endpoint""" + try: + requests = get_recent_requests() + return jsonify({'requests': requests}) + except Exception as e: + logger.error(f"Error getting analyzer requests: {str(e)}") + return jsonify({'requests': []}), 500 + +@analyzer_bp.route('/clear') +@check_session_validity +def clear_logs(): + """Clear analyzer logs""" + try: + # Delete all logs older than 24 hours + cutoff = datetime.now(pytz.UTC) - timedelta(hours=24) + AnalyzerLog.query.filter(AnalyzerLog.created_at < cutoff).delete() + db_session.commit() + flash('Analyzer logs cleared successfully', 'success') + except Exception as e: + logger.error(f"Error clearing analyzer logs: {str(e)}") + flash('Error clearing analyzer logs', 'error') + + return redirect(url_for('analyzer_bp.analyzer')) diff --git a/database/analyzer_db.py b/database/analyzer_db.py new file mode 100644 index 00000000..f4b6f162 --- /dev/null +++ b/database/analyzer_db.py @@ -0,0 +1,82 @@ +# database/analyzer_db.py + +import os +import json +from sqlalchemy import create_engine, Column, Integer, DateTime, Text +from sqlalchemy.orm import scoped_session, sessionmaker +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.sql import func +from concurrent.futures import ThreadPoolExecutor +from dotenv import load_dotenv +from datetime import datetime +import pytz + +load_dotenv() + +DATABASE_URL = os.getenv('DATABASE_URL') + +engine = create_engine( + DATABASE_URL, + pool_size=50, + max_overflow=100, + pool_timeout=10 +) + +db_session = scoped_session(sessionmaker(autocommit=False, autoflush=False, bind=engine)) +Base = declarative_base() +Base.query = db_session.query_property() + +class AnalyzerLog(Base): + __tablename__ = 'analyzer_logs' + id = Column(Integer, primary_key=True) + request_data = Column(Text, nullable=False) + response_data = Column(Text, nullable=False) + created_at = Column(DateTime(timezone=True), default=func.now()) + + def to_dict(self): + """Convert log entry to dictionary""" + try: + request_data = json.loads(self.request_data) if isinstance(self.request_data, str) else self.request_data + response_data = json.loads(self.response_data) if isinstance(self.response_data, str) else self.response_data + except json.JSONDecodeError: + request_data = self.request_data + response_data = self.response_data + + return { + 'id': self.id, + 'request_data': request_data, + 'response_data': response_data, + 'created_at': self.created_at.astimezone(pytz.UTC).isoformat() + } + +def init_db(): + """Initialize the analyzer table""" + print("Initializing Analyzer Table") + Base.metadata.create_all(bind=engine) + +# Executor for asynchronous tasks +executor = ThreadPoolExecutor(2) + +def async_log_analyzer(request_data, response_data): + """Asynchronously log analyzer request""" + try: + # Serialize JSON data for storage + request_json = json.dumps(request_data) + response_json = json.dumps(response_data) + + # Get current time in IST + ist = pytz.timezone('Asia/Kolkata') + now_ist = datetime.now(ist) + + analyzer_log = AnalyzerLog( + request_data=request_json, + response_data=response_json, + created_at=now_ist + ) + db_session.add(analyzer_log) + db_session.commit() + except Exception as e: + print(f"Error saving analyzer log: {e}") + db_session.rollback() + finally: + db_session.remove() diff --git a/restx_api/place_order.py b/restx_api/place_order.py index 2d8eccfc..66da6118 100644 --- a/restx_api/place_order.py +++ b/restx_api/place_order.py @@ -5,6 +5,14 @@ from database.apilog_db import async_log_order, executor from extensions import socketio from limiter import limiter +from utils.api_analyzer import analyze_request +from utils.constants import ( + VALID_EXCHANGES, + VALID_ACTIONS, + VALID_PRICE_TYPES, + VALID_PRODUCT_TYPES, + REQUIRED_ORDER_FIELDS +) import os from dotenv import load_dotenv import importlib @@ -14,6 +22,7 @@ load_dotenv() API_RATE_LIMIT = os.getenv("API_RATE_LIMIT", "10 per second") +ANALYZE_MODE = os.getenv("ANALYZE_MODE", "true").lower() == "true" api = Namespace('place_order', description='Place Order API') # Configure logging @@ -43,29 +52,87 @@ def post(self): order_data = order_schema.load(data) # Check for missing mandatory fields - mandatory_fields = ['apikey', 'strategy', 'exchange', 'symbol', 'action', 'quantity'] - missing_fields = [field for field in mandatory_fields if field not in data] - + missing_fields = [field for field in REQUIRED_ORDER_FIELDS if field not in data] if missing_fields: - return make_response(jsonify({ + error_response = { 'status': 'error', 'message': f'Missing mandatory field(s): {", ".join(missing_fields)}' - }), 400) + } + if not ANALYZE_MODE: + executor.submit(async_log_order, 'placeorder', data, error_response) + return make_response(jsonify(error_response), 400) + + # Validate exchange + if 'exchange' in data and data['exchange'] not in VALID_EXCHANGES: + error_response = { + 'status': 'error', + 'message': f'Invalid exchange. Must be one of: {", ".join(VALID_EXCHANGES)}' + } + if not ANALYZE_MODE: + executor.submit(async_log_order, 'placeorder', data, error_response) + return make_response(jsonify(error_response), 400) + + # Validate action + if 'action' in data and data['action'] not in VALID_ACTIONS: + error_response = { + 'status': 'error', + 'message': f'Invalid action. Must be one of: {", ".join(VALID_ACTIONS)}' + } + if not ANALYZE_MODE: + executor.submit(async_log_order, 'placeorder', data, error_response) + return make_response(jsonify(error_response), 400) + + # Validate price type if provided + if 'price_type' in data and data['price_type'] not in VALID_PRICE_TYPES: + error_response = { + 'status': 'error', + 'message': f'Invalid price type. Must be one of: {", ".join(VALID_PRICE_TYPES)}' + } + if not ANALYZE_MODE: + executor.submit(async_log_order, 'placeorder', data, error_response) + return make_response(jsonify(error_response), 400) + + # Validate product type if provided + if 'product_type' in data and data['product_type'] not in VALID_PRODUCT_TYPES: + error_response = { + 'status': 'error', + 'message': f'Invalid product type. Must be one of: {", ".join(VALID_PRODUCT_TYPES)}' + } + if not ANALYZE_MODE: + executor.submit(async_log_order, 'placeorder', data, error_response) + return make_response(jsonify(error_response), 400) api_key = order_data['apikey'] AUTH_TOKEN, broker = get_auth_token_broker(api_key) if AUTH_TOKEN is None: - return make_response(jsonify({ + error_response = { 'status': 'error', 'message': 'Invalid openalgo apikey' - }), 403) + } + if not ANALYZE_MODE: + executor.submit(async_log_order, 'placeorder', data, error_response) + return make_response(jsonify(error_response), 403) + + # If in analyze mode, analyze the request and store in analyzer_logs + if ANALYZE_MODE: + _, analysis = analyze_request(order_data) + response_data = { + 'status': analysis.get('status', 'error'), + 'message': analysis.get('message', 'Analysis failed'), + 'warnings': analysis.get('warnings', []), + 'broker': broker + } + return make_response(jsonify(response_data), 200) + # If not in analyze mode, proceed with actual order placement and store in order_logs broker_module = import_broker_module(broker) if broker_module is None: - return make_response(jsonify({ + error_response = { 'status': 'error', 'message': 'Broker-specific module not found' - }), 404) + } + executor.submit(async_log_order, 'placeorder', data, error_response) + return make_response(jsonify(error_response), 404) try: # Call the broker's place_order_api function @@ -73,46 +140,59 @@ def post(self): except Exception as e: logger.error(f"Error in broker_module.place_order_api: {e}") traceback.print_exc() - return make_response(jsonify({ + error_response = { 'status': 'error', 'message': 'Failed to place order due to internal error' - }), 500) + } + executor.submit(async_log_order, 'placeorder', data, error_response) + return make_response(jsonify(error_response), 500) if res.status == 200: socketio.emit('order_event', { 'symbol': order_data['symbol'], 'action': order_data['action'], - 'orderid': order_id + 'orderid': order_id, + 'exchange': order_data.get('exchange', 'Unknown'), + 'price_type': order_data.get('price_type', 'Unknown'), + 'product_type': order_data.get('product_type', 'Unknown') }) order_response_data = {'status': 'success', 'orderid': order_id} - - try: - executor.submit(async_log_order, 'placeorder', order_data, order_response_data) - except Exception as e: - logger.error(f"Error submitting async_log_order task: {e}") - traceback.print_exc() - + executor.submit(async_log_order, 'placeorder', order_data, order_response_data) return make_response(jsonify(order_response_data), 200) else: message = response_data.get('message', 'Failed to place order') if isinstance(response_data, dict) else 'Failed to place order' - return make_response(jsonify({ + error_response = { 'status': 'error', 'message': message - }), res.status if res.status != 200 else 500) + } + executor.submit(async_log_order, 'placeorder', data, error_response) + return make_response(jsonify(error_response), res.status if res.status != 200 else 500) + except ValidationError as err: logger.warning(f"Validation error: {err.messages}") - return make_response(jsonify({'status': 'error', 'message': err.messages}), 400) + error_response = {'status': 'error', 'message': err.messages} + if not ANALYZE_MODE: + executor.submit(async_log_order, 'placeorder', data, error_response) + return make_response(jsonify(error_response), 400) + except KeyError as e: missing_field = str(e) logger.error(f"KeyError: Missing field {missing_field}") - return make_response(jsonify({ + error_response = { 'status': 'error', 'message': f"A required field is missing: {missing_field}" - }), 400) + } + if not ANALYZE_MODE: + executor.submit(async_log_order, 'placeorder', data, error_response) + return make_response(jsonify(error_response), 400) + except Exception as e: logger.error("An unexpected error occurred in PlaceOrder endpoint.") traceback.print_exc() - return make_response(jsonify({ + error_response = { 'status': 'error', 'message': 'An unexpected error occurred' - }), 500) \ No newline at end of file + } + if not ANALYZE_MODE: + executor.submit(async_log_order, 'placeorder', data, error_response) + return make_response(jsonify(error_response), 500) diff --git a/static/js/socket-events.js b/static/js/socket-events.js index 4f6c7c3b..3aad353a 100644 --- a/static/js/socket-events.js +++ b/static/js/socket-events.js @@ -1,139 +1,204 @@ -document.addEventListener('DOMContentLoaded', function() { - var socket = io.connect(location.protocol + '//' + document.domain + ':' + location.port); - var alertSound = document.getElementById('alert-sound'); - - // Function to fetch and update logs - async function refreshLogs() { - try { - const response = await fetch('/logs'); - const html = await response.text(); - const tempDiv = document.createElement('div'); - tempDiv.innerHTML = html; - const newContent = tempDiv.querySelector('#logs-container'); - if (newContent) { - const currentContainer = document.getElementById('logs-container'); - if (currentContainer) { - currentContainer.innerHTML = newContent.innerHTML; - } +// Function to fetch and update logs +async function refreshLogs() { + try { + const response = await fetch('/logs'); + const html = await response.text(); + const tempDiv = document.createElement('div'); + tempDiv.innerHTML = html; + const newContent = tempDiv.querySelector('#logs-container'); + if (newContent) { + const currentContainer = document.getElementById('logs-container'); + if (currentContainer) { + currentContainer.innerHTML = newContent.innerHTML; } - } catch (error) { - console.error('Error refreshing logs:', error); } + } catch (error) { + console.error('Error refreshing logs:', error); } +} - // Function to fetch and update orderbook - async function refreshOrderbook() { - try { - const response = await fetch('/orderbook'); - const html = await response.text(); - const tempDiv = document.createElement('div'); - tempDiv.innerHTML = html; - - // Update stats grid - const newStatsGrid = tempDiv.querySelector('.grid-cols-1.sm\\:grid-cols-2.lg\\:grid-cols-5'); - if (newStatsGrid) { - const currentStatsGrid = document.querySelector('.grid-cols-1.sm\\:grid-cols-2.lg\\:grid-cols-5'); - if (currentStatsGrid) { - currentStatsGrid.innerHTML = newStatsGrid.innerHTML; - } +// Function to fetch and update orderbook +async function refreshOrderbook() { + try { + const response = await fetch('/orderbook'); + const html = await response.text(); + const tempDiv = document.createElement('div'); + tempDiv.innerHTML = html; + + // Update stats grid + const newStatsGrid = tempDiv.querySelector('.grid-cols-1.sm\\:grid-cols-2.lg\\:grid-cols-5'); + if (newStatsGrid) { + const currentStatsGrid = document.querySelector('.grid-cols-1.sm\\:grid-cols-2.lg\\:grid-cols-5'); + if (currentStatsGrid) { + currentStatsGrid.innerHTML = newStatsGrid.innerHTML; } - - // Update table - const newContent = tempDiv.querySelector('.table-container'); - if (newContent) { - const currentContainer = document.querySelector('.table-container'); - if (currentContainer) { - currentContainer.innerHTML = newContent.innerHTML; - } + } + + // Update table + const newContent = tempDiv.querySelector('.table-container'); + if (newContent) { + const currentContainer = document.querySelector('.table-container'); + if (currentContainer) { + currentContainer.innerHTML = newContent.innerHTML; } - } catch (error) { - console.error('Error refreshing orderbook:', error); } + } catch (error) { + console.error('Error refreshing orderbook:', error); } +} - // Function to fetch and update tradebook - async function refreshTradebook() { - try { - const response = await fetch('/tradebook'); - const html = await response.text(); - const tempDiv = document.createElement('div'); - tempDiv.innerHTML = html; - - // Update stats - const newStats = tempDiv.querySelector('.stats'); - if (newStats) { - const currentStats = document.querySelector('.stats'); - if (currentStats) { - currentStats.innerHTML = newStats.innerHTML; - } +// Function to fetch and update tradebook +async function refreshTradebook() { + try { + const response = await fetch('/tradebook'); + const html = await response.text(); + const tempDiv = document.createElement('div'); + tempDiv.innerHTML = html; + + // Update stats + const newStats = tempDiv.querySelector('.stats'); + if (newStats) { + const currentStats = document.querySelector('.stats'); + if (currentStats) { + currentStats.innerHTML = newStats.innerHTML; } - - // Update table - const newContent = tempDiv.querySelector('.table-container'); - if (newContent) { - const currentContainer = document.querySelector('.table-container'); - if (currentContainer) { - currentContainer.innerHTML = newContent.innerHTML; - } + } + + // Update table + const newContent = tempDiv.querySelector('.table-container'); + if (newContent) { + const currentContainer = document.querySelector('.table-container'); + if (currentContainer) { + currentContainer.innerHTML = newContent.innerHTML; } - } catch (error) { - console.error('Error refreshing tradebook:', error); } + } catch (error) { + console.error('Error refreshing tradebook:', error); } +} - // Function to fetch and update positions - async function refreshPositions() { - try { - const response = await fetch('/positions'); - const html = await response.text(); - const tempDiv = document.createElement('div'); - tempDiv.innerHTML = html; - const newContent = tempDiv.querySelector('.table-container'); - if (newContent) { - const currentContainer = document.querySelector('.table-container'); - if (currentContainer) { - currentContainer.innerHTML = newContent.innerHTML; - } +// Function to fetch and update positions +async function refreshPositions() { + try { + const response = await fetch('/positions'); + const html = await response.text(); + const tempDiv = document.createElement('div'); + tempDiv.innerHTML = html; + const newContent = tempDiv.querySelector('.table-container'); + if (newContent) { + const currentContainer = document.querySelector('.table-container'); + if (currentContainer) { + currentContainer.innerHTML = newContent.innerHTML; } - } catch (error) { - console.error('Error refreshing positions:', error); } + } catch (error) { + console.error('Error refreshing positions:', error); } +} - // Function to fetch and update dashboard funds - async function refreshDashboard() { - try { - const response = await fetch('/dashboard'); - const html = await response.text(); - const tempDiv = document.createElement('div'); - tempDiv.innerHTML = html; - const newContent = tempDiv.querySelector('.grid-cols-1.sm\\:grid-cols-2.lg\\:grid-cols-4'); - if (newContent) { - const currentContainer = document.querySelector('.grid-cols-1.sm\\:grid-cols-2.lg\\:grid-cols-4'); - if (currentContainer) { - currentContainer.innerHTML = newContent.innerHTML; - } +// Function to fetch and update dashboard funds +async function refreshDashboard() { + try { + const response = await fetch('/dashboard'); + const html = await response.text(); + const tempDiv = document.createElement('div'); + tempDiv.innerHTML = html; + const newContent = tempDiv.querySelector('.grid-cols-1.sm\\:grid-cols-2.lg\\:grid-cols-4'); + if (newContent) { + const currentContainer = document.querySelector('.grid-cols-1.sm\\:grid-cols-2.lg\\:grid-cols-4'); + if (currentContainer) { + currentContainer.innerHTML = newContent.innerHTML; } - } catch (error) { - console.error('Error refreshing dashboard:', error); } + } catch (error) { + console.error('Error refreshing dashboard:', error); } +} + +// Function to fetch and update analyzer +async function refreshAnalyzer() { + try { + // Update stats + const statsResponse = await fetch('/analyzer/stats'); + const statsData = await statsResponse.json(); + + document.getElementById('total-requests').textContent = statsData.total_requests; + document.getElementById('total-issues').textContent = statsData.issues.total; + document.getElementById('unique-symbols').textContent = statsData.symbols.length; + document.getElementById('active-sources').textContent = Object.keys(statsData.sources).length; - // Function to refresh content based on current page - function refreshCurrentPageContent() { - const path = window.location.pathname; - if (path.includes('/logs')) { - refreshLogs(); - } else if (path.includes('/orderbook')) { - refreshOrderbook(); - } else if (path.includes('/tradebook')) { - refreshTradebook(); - } else if (path.includes('/positions')) { - refreshPositions(); - } else if (path === '/dashboard' || path === '/') { - refreshDashboard(); + // Update requests table + const requestsResponse = await fetch('/analyzer/requests'); + const requestsData = await requestsResponse.json(); + const tbody = document.getElementById('requests-table'); + if (tbody && requestsData.requests) { + tbody.innerHTML = requestsData.requests.map(request => ` + + ${request.timestamp} +
    ${request.source}
    + ${request.symbol} + +
    + ${request.exchange} +
    + + +
    + ${request.action} +
    + + ${request.quantity} + +
    + ${request.analysis.issues ? 'Issues Found' : 'Valid'} +
    + + + + + + `).join(''); } + } catch (error) { + console.error('Error refreshing analyzer:', error); + } +} + +// Helper function to get exchange badge color +function getExchangeBadgeColor(exchange) { + const colors = { + 'NSE': 'badge-accent', + 'BSE': 'badge-neutral', + 'NFO': 'badge-secondary', + 'MCX': 'badge-primary' + }; + return colors[exchange] || 'badge-ghost'; +} + +// Make refreshCurrentPageContent available globally +window.refreshCurrentPageContent = function() { + const path = window.location.pathname; + if (path.includes('/logs')) { + refreshLogs(); + } else if (path.includes('/orderbook')) { + refreshOrderbook(); + } else if (path.includes('/tradebook')) { + refreshTradebook(); + } else if (path.includes('/positions')) { + refreshPositions(); + } else if (path === '/dashboard' || path === '/') { + refreshDashboard(); + } else if (path.includes('/analyzer')) { + refreshAnalyzer(); } +} + +document.addEventListener('DOMContentLoaded', function() { + var socket = io.connect(location.protocol + '//' + document.domain + ':' + location.port); + var alertSound = document.getElementById('alert-sound'); socket.on('connect', function() { console.log('Connected to WebSocket server'); @@ -218,6 +283,18 @@ document.addEventListener('DOMContentLoaded', function() { refreshCurrentPageContent(); }); + // Analyzer update notification + socket.on('analyzer_update', function(data) { + playAlertSound(); + showToast(`API Request: ${data.request.symbol || 'Unknown'} - ${data.response.status}`, + data.response.status === 'success' ? 'success' : 'warning'); + + // Refresh analyzer content if on analyzer page + if (window.location.pathname.includes('/analyzer')) { + refreshAnalyzer(); + } + }); + // Helper function to play alert sound function playAlertSound() { if (alertSound) { diff --git a/templates/analyzer.html b/templates/analyzer.html new file mode 100644 index 00000000..561ddb7d --- /dev/null +++ b/templates/analyzer.html @@ -0,0 +1,190 @@ +{% extends "base.html" %} + +{% block head %} + +{% endblock %} + +{% block content %} + + + +
    + +
    +

    API Request Analyzer

    +

    Monitor and analyze your API requests

    +
    + + +
    + +
    +
    +
    Total Requests
    +
    {{ stats.total_requests }}
    +
    +
    Last 24 hours
    +
    +
    +
    + + +
    +
    +
    Issues Found
    +
    {{ stats.issues.total }}
    +
    +
    Needs Attention
    +
    +
    +
    + + +
    +
    +
    Unique Symbols
    +
    {{ stats.symbols|length }}
    +
    +
    Tracked
    +
    +
    +
    + + +
    +
    +
    Active Sources
    +
    {{ stats.sources|length }}
    +
    +
    Connected
    +
    +
    +
    +
    + + +
    + + + + + + + + + + + + + + + {% for request in requests %} + + + + + + + + + + + {% endfor %} + +
    + Timestamp + + + + SourceSymbolExchangeActionQuantityStatusDetails
    {{ request.timestamp }} +
    {{ request.source }}
    +
    {{ request.symbol }} + {% set exchange_colors = { + 'NSE': 'badge-accent', + 'BSE': 'badge-neutral', + 'NFO': 'badge-secondary', + 'MCX': 'badge-primary' + } %} +
    + {{ request.exchange }} +
    +
    +
    + {{ request.action }} +
    +
    {{ request.quantity }} +
    + {% if request.analysis.issues %}Issues Found{% else %}Valid{% endif %} +
    +
    + +
    +
    +
    + + + + + +{% endblock %} + +{% block scripts %} + +{% endblock %} diff --git a/templates/base.html b/templates/base.html index 1c85e7df..6e75b7aa 100644 --- a/templates/base.html +++ b/templates/base.html @@ -208,6 +208,12 @@ Tradingview +
  • + + API Analyzer + +
  • + + {% block scripts %}{% endblock %} diff --git a/templates/navbar.html b/templates/navbar.html index 1c2233a2..a6e3d80e 100644 --- a/templates/navbar.html +++ b/templates/navbar.html @@ -51,6 +51,12 @@ Tradingview
  • +
  • + + API Analyzer + +
  • @@ -109,6 +115,14 @@ Search +
  • + + + + + API Analyzer + +
  • diff --git a/utils/api_analyzer.py b/utils/api_analyzer.py new file mode 100644 index 00000000..8b234557 --- /dev/null +++ b/utils/api_analyzer.py @@ -0,0 +1,219 @@ +import logging +from datetime import datetime, timedelta +import pytz +from database.analyzer_db import AnalyzerLog, db_session, async_log_analyzer +from sqlalchemy import func +import json +from extensions import socketio +from utils.constants import ( + VALID_EXCHANGES, + VALID_ACTIONS, + VALID_PRICE_TYPES, + VALID_PRODUCT_TYPES, + REQUIRED_ORDER_FIELDS +) + +logger = logging.getLogger(__name__) + +def check_rate_limits(user_id): + """Check if user has hit rate limits recently""" + try: + cutoff = datetime.now(pytz.UTC) - timedelta(minutes=5) + rate_limited = AnalyzerLog.query.filter( + AnalyzerLog.created_at >= cutoff, + AnalyzerLog.response_data.like('%rate limit%') + ).count() + return rate_limited > 0 + except Exception as e: + logger.error(f"Error checking rate limits: {str(e)}") + return False + +def analyze_api_request(order_data): + """Analyze an API request before processing""" + try: + issues = [] + warnings = [] + + # Check required fields + missing_fields = [field for field in REQUIRED_ORDER_FIELDS if field not in order_data] + if missing_fields: + issues.append(f"Missing required fields: {', '.join(missing_fields)}") + + # Validate quantity + if 'quantity' in order_data: + try: + quantity = float(order_data['quantity']) + if quantity <= 0: + issues.append("Quantity must be greater than 0") + except (ValueError, TypeError): + issues.append("Invalid quantity value") + + # Validate exchange + if 'exchange' in order_data: + if order_data['exchange'] not in VALID_EXCHANGES: + issues.append(f"Invalid exchange. Must be one of: {', '.join(VALID_EXCHANGES)}") + + # Validate action + if 'action' in order_data: + if order_data['action'] not in VALID_ACTIONS: + issues.append(f"Invalid action. Must be one of: {', '.join(VALID_ACTIONS)}") + + # Validate price type if provided + if 'price_type' in order_data: + if order_data['price_type'] not in VALID_PRICE_TYPES: + issues.append(f"Invalid price type. Must be one of: {', '.join(VALID_PRICE_TYPES)}") + + # Validate product type if provided + if 'product_type' in order_data: + if order_data['product_type'] not in VALID_PRODUCT_TYPES: + issues.append(f"Invalid product type. Must be one of: {', '.join(VALID_PRODUCT_TYPES)}") + + # Check for potential rate limit issues + try: + if AnalyzerLog.query.filter( + AnalyzerLog.created_at >= datetime.now(pytz.UTC) - timedelta(minutes=1) + ).count() > 50: + warnings.append("High request frequency detected. Consider reducing request rate.") + except Exception as e: + logger.error(f"Error checking rate limits: {str(e)}") + warnings.append("Unable to check rate limits") + + # Prepare response + response = { + 'status': 'success' if len(issues) == 0 else 'error', + 'message': ', '.join(issues) if issues else 'Request valid', + 'warnings': warnings + } + + return response + + except Exception as e: + logger.error(f"Error analyzing API request: {str(e)}") + return { + 'status': 'error', + 'message': "Internal error analyzing request", + 'warnings': [] + } + +def analyze_request(request_data): + """Analyze and log a request""" + try: + # Analyze request first + analysis = analyze_api_request(request_data) + + # Log to analyzer database + try: + async_log_analyzer(request_data, analysis) + except Exception as e: + logger.error(f"Error logging to analyzer database: {str(e)}") + + # Emit socket event for real-time updates + try: + socketio.emit('analyzer_update', { + 'request': { + 'symbol': request_data.get('symbol', 'Unknown'), + 'action': request_data.get('action', 'Unknown'), + 'exchange': request_data.get('exchange', 'Unknown'), + 'quantity': request_data.get('quantity', 0), + 'price_type': request_data.get('price_type', 'Unknown'), + 'product_type': request_data.get('product_type', 'Unknown') + }, + 'response': analysis + }) + except Exception as e: + logger.error(f"Error emitting socket event: {str(e)}") + + # Return analysis results + return True, analysis + + except Exception as e: + logger.error(f"Error analyzing request: {str(e)}") + error_response = { + 'status': 'error', + 'message': "Internal error analyzing request", + 'warnings': [] + } + return False, error_response + +def get_analyzer_stats(): + """Get analyzer statistics""" + try: + cutoff = datetime.now(pytz.UTC) - timedelta(hours=24) + + # Get recent requests + recent_requests = AnalyzerLog.query.filter( + AnalyzerLog.created_at >= cutoff + ).all() + + # Initialize stats + stats = { + 'total_requests': len(recent_requests), + 'sources': {}, + 'symbols': set(), + 'issues': { + 'total': 0, + 'by_type': { + 'rate_limit': 0, + 'invalid_symbol': 0, + 'missing_quantity': 0, + 'invalid_exchange': 0, + 'other': 0 + } + } + } + + # Process requests + for req in recent_requests: + try: + request_data = json.loads(req.request_data) + response_data = json.loads(req.response_data) + + # Update sources + source = request_data.get('strategy', 'Unknown') + stats['sources'][source] = stats['sources'].get(source, 0) + 1 + + # Update symbols + if 'symbol' in request_data: + stats['symbols'].add(request_data['symbol']) + + # Update issues + if response_data.get('status') == 'error': + stats['issues']['total'] += 1 + error_msg = response_data.get('message', '').lower() + + if 'rate limit' in error_msg: + stats['issues']['by_type']['rate_limit'] += 1 + elif 'invalid symbol' in error_msg: + stats['issues']['by_type']['invalid_symbol'] += 1 + elif 'quantity' in error_msg: + stats['issues']['by_type']['missing_quantity'] += 1 + elif 'exchange' in error_msg: + stats['issues']['by_type']['invalid_exchange'] += 1 + else: + stats['issues']['by_type']['other'] += 1 + + except Exception as e: + logger.error(f"Error processing request: {str(e)}") + continue + + # Convert set to list for JSON serialization + stats['symbols'] = list(stats['symbols']) + return stats + + except Exception as e: + logger.error(f"Error getting analyzer stats: {str(e)}") + return { + 'total_requests': 0, + 'sources': {}, + 'symbols': [], + 'issues': { + 'total': 0, + 'by_type': { + 'rate_limit': 0, + 'invalid_symbol': 0, + 'missing_quantity': 0, + 'invalid_exchange': 0, + 'other': 0 + } + } + } diff --git a/utils/constants.py b/utils/constants.py new file mode 100644 index 00000000..a73b23e3 --- /dev/null +++ b/utils/constants.py @@ -0,0 +1,73 @@ +""" +Constants used throughout the application. +Reference: https://docs.openalgo.in/api-documentation/v1/order-constants +""" + +# Exchange Types +EXCHANGE_NSE = 'NSE' # NSE Equity +EXCHANGE_NFO = 'NFO' # NSE Futures & Options +EXCHANGE_CDS = 'CDS' # NSE Currency +EXCHANGE_BSE = 'BSE' # BSE Equity +EXCHANGE_BFO = 'BFO' # BSE Futures & Options +EXCHANGE_BCD = 'BCD' # BSE Currency +EXCHANGE_MCX = 'MCX' # MCX Commodity +EXCHANGE_NCDEX = 'NCDEX' # NCDEX Commodity + +VALID_EXCHANGES = [ + EXCHANGE_NSE, + EXCHANGE_NFO, + EXCHANGE_CDS, + EXCHANGE_BSE, + EXCHANGE_BFO, + EXCHANGE_BCD, + EXCHANGE_MCX, + EXCHANGE_NCDEX +] + +# Product Types +PRODUCT_CNC = 'CNC' # Cash & Carry for equity +PRODUCT_NRML = 'NRML' # Normal for futures and options +PRODUCT_MIS = 'MIS' # Intraday Square off + +VALID_PRODUCT_TYPES = [ + PRODUCT_CNC, + PRODUCT_NRML, + PRODUCT_MIS +] + +# Price Types +PRICE_TYPE_MARKET = 'MARKET' # Market Order +PRICE_TYPE_LIMIT = 'LIMIT' # Limit Order +PRICE_TYPE_SL = 'SL' # Stop Loss Limit Order +PRICE_TYPE_SLM = 'SL-M' # Stop Loss Market Order + +VALID_PRICE_TYPES = [ + PRICE_TYPE_MARKET, + PRICE_TYPE_LIMIT, + PRICE_TYPE_SL, + PRICE_TYPE_SLM +] + +# Order Actions +ACTION_BUY = 'BUY' # Buy +ACTION_SELL = 'SELL' # Sell + +VALID_ACTIONS = [ + ACTION_BUY, + ACTION_SELL +] + +# Exchange Badge Colors (for UI) +EXCHANGE_BADGE_COLORS = { + EXCHANGE_NSE: 'badge-accent', + EXCHANGE_NFO: 'badge-secondary', + EXCHANGE_CDS: 'badge-info', + EXCHANGE_BSE: 'badge-neutral', + EXCHANGE_BFO: 'badge-warning', + EXCHANGE_BCD: 'badge-error', + EXCHANGE_MCX: 'badge-primary', + EXCHANGE_NCDEX: 'badge-success' +} + +# Required Fields for Order Placement +REQUIRED_ORDER_FIELDS = ['symbol', 'exchange', 'quantity', 'action'] From 25df29075abc1becd5a52248f4d775c813c8f283 Mon Sep 17 00:00:00 2001 From: marketcalls Date: Tue, 3 Dec 2024 18:06:18 +0530 Subject: [PATCH 11/41] Analyzer Mode Vs Live Mode Control via DB --- .sample.env | 3 -- app.py | 4 +++ blueprints/settings.py | 36 ++++++++++++++++++++++ database/settings_db.py | 65 ++++++++++++++++++++++++++++++++++++++++ restx_api/place_order.py | 28 +++++++++-------- static/js/mode-toggle.js | 27 +++++++++++++++++ templates/navbar.html | 20 ++++++++++++- 7 files changed, 166 insertions(+), 17 deletions(-) create mode 100644 blueprints/settings.py create mode 100644 database/settings_db.py create mode 100644 static/js/mode-toggle.js diff --git a/.sample.env b/.sample.env index f69033f2..f19cd188 100644 --- a/.sample.env +++ b/.sample.env @@ -10,9 +10,6 @@ VALID_BROKERS = 'fivepaisa,aliceblue,angel,dhan,fyers,icici,kotak,shoonya,upstox # OpenAlgo Application Key - Change the Key to Some Random Values APP_KEY = 'dfsd98tyhgfrtk34ghuu85df' -# API Analyzer Mode - Set to true to analyze requests without placing orders -ANALYZE_MODE = 'true' - # OpenAlgo Database Configuration DATABASE_URL = 'sqlite:///db/openalgo.db' diff --git a/app.py b/app.py index 2c2f7123..2ec33b01 100644 --- a/app.py +++ b/app.py @@ -18,6 +18,7 @@ from blueprints.brlogin import brlogin_bp from blueprints.core import core_bp from blueprints.analyzer import analyzer_bp # Import the analyzer blueprint +from blueprints.settings import settings_bp # Import the settings blueprint from restx_api import api_v1_bp @@ -26,6 +27,7 @@ from database.symbol import init_db as ensure_master_contract_tables_exists from database.apilog_db import init_db as ensure_api_log_tables_exists from database.analyzer_db import init_db as ensure_analyzer_tables_exists +from database.settings_db import init_db as ensure_settings_tables_exists from utils.plugin_loader import load_broker_auth_functions @@ -63,6 +65,7 @@ def create_app(): app.register_blueprint(brlogin_bp) app.register_blueprint(core_bp) app.register_blueprint(analyzer_bp) # Register the analyzer blueprint + app.register_blueprint(settings_bp) # Register the settings blueprint # Register RESTx API blueprint app.register_blueprint(api_v1_bp) @@ -88,6 +91,7 @@ def setup_environment(app): ensure_master_contract_tables_exists() ensure_api_log_tables_exists() ensure_analyzer_tables_exists() + ensure_settings_tables_exists() # Conditionally setup ngrok in development environment if os.getenv('NGROK_ALLOW') == 'TRUE': diff --git a/blueprints/settings.py b/blueprints/settings.py new file mode 100644 index 00000000..b63736e4 --- /dev/null +++ b/blueprints/settings.py @@ -0,0 +1,36 @@ +# blueprints/settings.py + +from flask import Blueprint, jsonify, request +from database.settings_db import get_analyze_mode, set_analyze_mode +from utils.session import check_session_validity +import logging + +logger = logging.getLogger(__name__) + +settings_bp = Blueprint('settings_bp', __name__, url_prefix='/settings') + +@settings_bp.route('/analyze-mode') +@check_session_validity +def get_mode(): + """Get current analyze mode setting""" + try: + return jsonify({'analyze_mode': get_analyze_mode()}) + except Exception as e: + logger.error(f"Error getting analyze mode: {str(e)}") + return jsonify({'error': 'Failed to get analyze mode'}), 500 + +@settings_bp.route('/analyze-mode/', methods=['POST']) +@check_session_validity +def set_mode(mode): + """Set analyze mode setting""" + try: + set_analyze_mode(bool(mode)) + mode_name = 'Analyze' if mode else 'Live' + return jsonify({ + 'success': True, + 'analyze_mode': bool(mode), + 'message': f'Switched to {mode_name} Mode' + }) + except Exception as e: + logger.error(f"Error setting analyze mode: {str(e)}") + return jsonify({'error': 'Failed to set analyze mode'}), 500 diff --git a/database/settings_db.py b/database/settings_db.py new file mode 100644 index 00000000..9fcf9617 --- /dev/null +++ b/database/settings_db.py @@ -0,0 +1,65 @@ +# database/settings_db.py + +from sqlalchemy import create_engine, Column, Integer, String, Boolean, MetaData +from sqlalchemy.orm import scoped_session, sessionmaker +from sqlalchemy.ext.declarative import declarative_base +import os +from dotenv import load_dotenv + +load_dotenv() + +DATABASE_URL = os.getenv('DATABASE_URL') + +engine = create_engine( + DATABASE_URL, + pool_size=50, + max_overflow=100, + pool_timeout=10 +) + +db_session = scoped_session(sessionmaker(autocommit=False, autoflush=False, bind=engine)) +Base = declarative_base() +Base.query = db_session.query_property() + +class Settings(Base): + __tablename__ = 'settings' + id = Column(Integer, primary_key=True) + analyze_mode = Column(Boolean, default=False) # Default to Live Mode + +def init_db(): + """Initialize the settings database""" + print("Initializing Settings DB") + + # Drop existing settings table if it exists + metadata = MetaData() + metadata.reflect(bind=engine) + if 'settings' in metadata.tables: + Settings.__table__.drop(engine) + + # Create tables + Base.metadata.create_all(bind=engine) + + # Create default settings + if not Settings.query.first(): + default_settings = Settings(analyze_mode=False) # Default to Live Mode + db_session.add(default_settings) + db_session.commit() + +def get_analyze_mode(): + """Get current analyze mode setting""" + settings = Settings.query.first() + if not settings: + settings = Settings(analyze_mode=False) # Default to Live Mode + db_session.add(settings) + db_session.commit() + return settings.analyze_mode + +def set_analyze_mode(mode: bool): + """Set analyze mode setting""" + settings = Settings.query.first() + if not settings: + settings = Settings(analyze_mode=mode) + db_session.add(settings) + else: + settings.analyze_mode = mode + db_session.commit() diff --git a/restx_api/place_order.py b/restx_api/place_order.py index 66da6118..d8fa99b1 100644 --- a/restx_api/place_order.py +++ b/restx_api/place_order.py @@ -3,6 +3,7 @@ from marshmallow import ValidationError from database.auth_db import get_auth_token_broker from database.apilog_db import async_log_order, executor +from database.settings_db import get_analyze_mode from extensions import socketio from limiter import limiter from utils.api_analyzer import analyze_request @@ -22,7 +23,6 @@ load_dotenv() API_RATE_LIMIT = os.getenv("API_RATE_LIMIT", "10 per second") -ANALYZE_MODE = os.getenv("ANALYZE_MODE", "true").lower() == "true" api = Namespace('place_order', description='Place Order API') # Configure logging @@ -58,7 +58,7 @@ def post(self): 'status': 'error', 'message': f'Missing mandatory field(s): {", ".join(missing_fields)}' } - if not ANALYZE_MODE: + if not get_analyze_mode(): executor.submit(async_log_order, 'placeorder', data, error_response) return make_response(jsonify(error_response), 400) @@ -68,7 +68,7 @@ def post(self): 'status': 'error', 'message': f'Invalid exchange. Must be one of: {", ".join(VALID_EXCHANGES)}' } - if not ANALYZE_MODE: + if not get_analyze_mode(): executor.submit(async_log_order, 'placeorder', data, error_response) return make_response(jsonify(error_response), 400) @@ -78,7 +78,7 @@ def post(self): 'status': 'error', 'message': f'Invalid action. Must be one of: {", ".join(VALID_ACTIONS)}' } - if not ANALYZE_MODE: + if not get_analyze_mode(): executor.submit(async_log_order, 'placeorder', data, error_response) return make_response(jsonify(error_response), 400) @@ -88,7 +88,7 @@ def post(self): 'status': 'error', 'message': f'Invalid price type. Must be one of: {", ".join(VALID_PRICE_TYPES)}' } - if not ANALYZE_MODE: + if not get_analyze_mode(): executor.submit(async_log_order, 'placeorder', data, error_response) return make_response(jsonify(error_response), 400) @@ -98,7 +98,7 @@ def post(self): 'status': 'error', 'message': f'Invalid product type. Must be one of: {", ".join(VALID_PRODUCT_TYPES)}' } - if not ANALYZE_MODE: + if not get_analyze_mode(): executor.submit(async_log_order, 'placeorder', data, error_response) return make_response(jsonify(error_response), 400) @@ -109,18 +109,19 @@ def post(self): 'status': 'error', 'message': 'Invalid openalgo apikey' } - if not ANALYZE_MODE: + if not get_analyze_mode(): executor.submit(async_log_order, 'placeorder', data, error_response) return make_response(jsonify(error_response), 403) # If in analyze mode, analyze the request and store in analyzer_logs - if ANALYZE_MODE: + if get_analyze_mode(): _, analysis = analyze_request(order_data) response_data = { 'status': analysis.get('status', 'error'), 'message': analysis.get('message', 'Analysis failed'), 'warnings': analysis.get('warnings', []), - 'broker': broker + 'broker': broker, + 'mode': 'analyze' } return make_response(jsonify(response_data), 200) @@ -154,7 +155,8 @@ def post(self): 'orderid': order_id, 'exchange': order_data.get('exchange', 'Unknown'), 'price_type': order_data.get('price_type', 'Unknown'), - 'product_type': order_data.get('product_type', 'Unknown') + 'product_type': order_data.get('product_type', 'Unknown'), + 'mode': 'live' }) order_response_data = {'status': 'success', 'orderid': order_id} executor.submit(async_log_order, 'placeorder', order_data, order_response_data) @@ -171,7 +173,7 @@ def post(self): except ValidationError as err: logger.warning(f"Validation error: {err.messages}") error_response = {'status': 'error', 'message': err.messages} - if not ANALYZE_MODE: + if not get_analyze_mode(): executor.submit(async_log_order, 'placeorder', data, error_response) return make_response(jsonify(error_response), 400) @@ -182,7 +184,7 @@ def post(self): 'status': 'error', 'message': f"A required field is missing: {missing_field}" } - if not ANALYZE_MODE: + if not get_analyze_mode(): executor.submit(async_log_order, 'placeorder', data, error_response) return make_response(jsonify(error_response), 400) @@ -193,6 +195,6 @@ def post(self): 'status': 'error', 'message': 'An unexpected error occurred' } - if not ANALYZE_MODE: + if not get_analyze_mode(): executor.submit(async_log_order, 'placeorder', data, error_response) return make_response(jsonify(error_response), 500) diff --git a/static/js/mode-toggle.js b/static/js/mode-toggle.js new file mode 100644 index 00000000..9b299848 --- /dev/null +++ b/static/js/mode-toggle.js @@ -0,0 +1,27 @@ +document.addEventListener('DOMContentLoaded', function() { + const modeToggle = document.querySelector('.mode-controller'); + if (!modeToggle) return; + + // Initialize mode from server + fetch('/settings/analyze-mode') + .then(response => response.json()) + .then(data => { + modeToggle.checked = data.analyze_mode; + }); + + // Handle mode toggle + modeToggle.addEventListener('change', function(e) { + const mode = e.target.checked ? 1 : 0; + fetch(`/settings/analyze-mode/${mode}`, { + method: 'POST', + }) + .then(response => response.json()) + .then(data => { + if (data.success) { + showToast(data.message, 'success'); + // Reload page to ensure all components update + setTimeout(() => window.location.reload(), 1000); + } + }); + }); +}); diff --git a/templates/navbar.html b/templates/navbar.html index a6e3d80e..1c48e10d 100644 --- a/templates/navbar.html +++ b/templates/navbar.html @@ -60,8 +60,23 @@ - + + + + From d3f1dec61c49a32fcc142b771fdaddbc0abf9954 Mon Sep 17 00:00:00 2001 From: marketcalls Date: Tue, 3 Dec 2024 18:11:07 +0530 Subject: [PATCH 12/41] Analyze Mode Vs Live Mode Badge --- static/js/mode-toggle.js | 17 ++++++++++++++++- templates/navbar.html | 3 +++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/static/js/mode-toggle.js b/static/js/mode-toggle.js index 9b299848..1d9084b3 100644 --- a/static/js/mode-toggle.js +++ b/static/js/mode-toggle.js @@ -1,12 +1,26 @@ document.addEventListener('DOMContentLoaded', function() { const modeToggle = document.querySelector('.mode-controller'); - if (!modeToggle) return; + const modeBadge = document.getElementById('mode-badge'); + if (!modeToggle || !modeBadge) return; + + function updateBadge(isAnalyzeMode) { + if (isAnalyzeMode) { + modeBadge.textContent = 'Analyze Mode'; + modeBadge.classList.remove('badge-success'); + modeBadge.classList.add('badge-warning'); + } else { + modeBadge.textContent = 'Live Mode'; + modeBadge.classList.remove('badge-warning'); + modeBadge.classList.add('badge-success'); + } + } // Initialize mode from server fetch('/settings/analyze-mode') .then(response => response.json()) .then(data => { modeToggle.checked = data.analyze_mode; + updateBadge(data.analyze_mode); }); // Handle mode toggle @@ -18,6 +32,7 @@ document.addEventListener('DOMContentLoaded', function() { .then(response => response.json()) .then(data => { if (data.success) { + updateBadge(data.analyze_mode); showToast(data.message, 'success'); // Reload page to ensure all components update setTimeout(() => window.location.reload(), 1000); diff --git a/templates/navbar.html b/templates/navbar.html index 1c48e10d..b07382cc 100644 --- a/templates/navbar.html +++ b/templates/navbar.html @@ -62,6 +62,9 @@ @@ -148,20 +148,6 @@

    First Time Setup

    placeholder="Enter your email"> - -
    - - -
    -
  • -
  • - - API Analyzer - -
  • - - {% block scripts %}{% endblock %} diff --git a/templates/layout.html b/templates/layout.html index 9c6f6712..d5531ef9 100644 --- a/templates/layout.html +++ b/templates/layout.html @@ -1,6 +1,15 @@ - + + + @@ -15,12 +24,16 @@ - + + + + + @@ -50,6 +63,10 @@ 'base-200': 'hsl(var(--b2))', 'base-300': 'hsl(var(--b3))', 'base-content': 'hsl(var(--bc))', + info: 'hsl(var(--in))', + success: 'hsl(var(--su))', + warning: 'hsl(var(--wa))', + error: 'hsl(var(--er))', } } }, @@ -57,48 +74,56 @@ themes: [ { light: { - primary: "#2563eb", // Blue-600 - secondary: "#7c3aed", // Violet-600 - accent: "#0891b2", // Cyan-600 - neutral: "#1f2937", // Gray-800 + "primary": "#2563eb", // Blue-600 + "primary-focus": "#1d4ed8", // Blue-700 + "primary-content": "#ffffff", + "secondary": "#7c3aed", // Violet-600 + "secondary-focus": "#6d28d9", // Violet-700 + "secondary-content": "#ffffff", + "accent": "#0891b2", // Cyan-600 + "accent-focus": "#0e7490", // Cyan-700 + "accent-content": "#ffffff", + "neutral": "#1f2937", // Gray-800 + "neutral-focus": "#111827", // Gray-900 + "neutral-content": "#ffffff", "base-100": "#ffffff", "base-200": "#f3f4f6", "base-300": "#e5e7eb", + "base-content": "#1f2937", + "info": "#3b82f6", + "success": "#22c55e", + "warning": "#f59e0b", + "error": "#ef4444" }, dark: { - primary: "#3b82f6", // Blue-500 - secondary: "#8b5cf6", // Violet-500 - accent: "#06b6d4", // Cyan-500 - "base-100": "#1f2937", // Gray-800 - "base-200": "#111827", // Gray-900 - "base-300": "#0f172a", // Gray-950 + "primary": "#3b82f6", // Blue-500 + "primary-focus": "#2563eb", // Blue-600 + "primary-content": "#ffffff", + "secondary": "#8b5cf6", // Violet-500 + "secondary-focus": "#7c3aed", // Violet-600 + "secondary-content": "#ffffff", + "accent": "#06b6d4", // Cyan-500 + "accent-focus": "#0891b2", // Cyan-600 + "accent-content": "#ffffff", + "neutral": "#1f2937", // Gray-800 + "neutral-focus": "#111827", // Gray-900 + "neutral-content": "#ffffff", + "base-100": "#1f2937", // Gray-800 + "base-200": "#111827", // Gray-900 + "base-300": "#0f172a", // Gray-950 + "base-content": "#f3f4f6", + "info": "#60a5fa", + "success": "#34d399", + "warning": "#fbbf24", + "error": "#f87171" } } ], - darkTheme: "dark", + darkTheme: "dark" } } - - - {% block head %}{% endblock %} @@ -195,19 +220,6 @@ @@ -33,11 +37,6 @@ - - - - - @@ -47,6 +46,7 @@ + + + + {% block head %}{% endblock %} @@ -240,9 +276,11 @@ + + + + + + diff --git a/templates/navbar.html b/templates/navbar.html index 8a7b131a..aeb1152d 100644 --- a/templates/navbar.html +++ b/templates/navbar.html @@ -65,7 +65,7 @@
    -
    +
    Live Mode
    @@ -165,6 +165,3 @@
    - - - From d49c7e5aa41ba4dc2f8369c9e4109f1830404c1c Mon Sep 17 00:00:00 2001 From: marketcalls Date: Thu, 5 Dec 2024 12:23:51 +0530 Subject: [PATCH 30/41] removed browser console logs --- static/js/mode-toggle.js | 56 +++------------------------------------- static/js/theme.js | 50 +++-------------------------------- templates/base.html | 10 ------- 3 files changed, 7 insertions(+), 109 deletions(-) diff --git a/static/js/mode-toggle.js b/static/js/mode-toggle.js index c89f1770..d598ab1e 100644 --- a/static/js/mode-toggle.js +++ b/static/js/mode-toggle.js @@ -1,42 +1,29 @@ // Mode toggle functionality document.addEventListener('DOMContentLoaded', function() { - console.log('[Mode] DOM Content Loaded'); const modeToggle = document.querySelector('.mode-controller'); const modeBadge = document.getElementById('mode-badge'); if (!modeToggle || !modeBadge) { - console.log('[Mode] Required elements not found:', { - modeToggle: !!modeToggle, - modeBadge: !!modeBadge - }); + console.error('[Mode] Required elements not found'); return; } - console.log('[Mode] Setting initial badge state'); // Set initial badge text to prevent flash of empty content modeBadge.textContent = 'Live Mode'; modeBadge.classList.add('badge-success'); function updateBadge(isAnalyzeMode) { - console.log(`[Mode] Updating badge for analyze mode: ${isAnalyzeMode}`); - if (isAnalyzeMode) { - console.log('[Mode] Switching to Analyze Mode'); // Store current theme before switching to garden const currentTheme = document.documentElement.getAttribute('data-theme'); - console.log(`[Mode] Current theme before garden: ${currentTheme}`); - if (currentTheme !== 'garden') { localStorage.setItem('previousTheme', currentTheme); sessionStorage.setItem('previousTheme', currentTheme); - console.log(`[Mode] Stored previous theme: ${currentTheme}`); } // Set garden theme when switching to analyze mode - console.log('[Mode] Setting garden theme'); window.themeManager.setTheme('garden'); } else { - console.log('[Mode] Setting Live Mode badge'); modeBadge.textContent = 'Live Mode'; modeBadge.classList.remove('badge-warning'); modeBadge.classList.add('badge-success'); @@ -44,60 +31,42 @@ document.addEventListener('DOMContentLoaded', function() { // Only restore theme if we're switching from analyze mode const currentTheme = document.documentElement.getAttribute('data-theme'); if (currentTheme === 'garden') { - console.log('[Mode] Switching from analyze mode, restoring theme'); const previousTheme = localStorage.getItem('previousTheme') || sessionStorage.getItem('previousTheme') || 'light'; - console.log(`[Mode] Restoring previous theme: ${previousTheme}`); window.themeManager.setTheme(previousTheme, true); - } else { - console.log('[Mode] Already in live mode, keeping current theme:', currentTheme); } // Clear stored previous theme localStorage.removeItem('previousTheme'); sessionStorage.removeItem('previousTheme'); - console.log('[Mode] Cleared previous theme from storage'); // Re-enable theme controllers const themeControllers = document.querySelectorAll('.theme-controller'); - console.log(`[Mode] Re-enabling ${themeControllers.length} theme controllers`); - themeControllers.forEach((controller, index) => { + themeControllers.forEach(controller => { const currentTheme = document.documentElement.getAttribute('data-theme'); controller.checked = currentTheme === 'dark'; controller.disabled = false; - console.log(`[Mode] Controller ${index} updated: checked=${controller.checked}, disabled=false`); }); // Remove disabled class from theme switcher const themeSwitcher = document.querySelector('.theme-switcher'); if (themeSwitcher) { themeSwitcher.classList.remove('disabled'); - console.log('[Mode] Theme switcher enabled'); } } } // Initialize mode from server - console.log('[Mode] Fetching initial mode from server'); fetch('/settings/analyze-mode') - .then(response => { - console.log('[Mode] Server response received:', response.status); - return response.json(); - }) + .then(response => response.json()) .then(data => { - console.log('[Mode] Server data:', data); modeToggle.checked = data.analyze_mode; - console.log(`[Mode] Toggle checked state set to: ${data.analyze_mode}`); if (data.analyze_mode) { - console.log('[Mode] Initializing analyze mode'); // If in analyze mode, ensure we store the current theme before switching const currentTheme = document.documentElement.getAttribute('data-theme'); - console.log(`[Mode] Current theme: ${currentTheme}`); - if (currentTheme !== 'garden') { localStorage.setItem('previousTheme', currentTheme); sessionStorage.setItem('previousTheme', currentTheme); - console.log(`[Mode] Stored initial theme: ${currentTheme}`); } window.themeManager.setTheme('garden'); } else { @@ -113,29 +82,21 @@ document.addEventListener('DOMContentLoaded', function() { // Handle mode toggle modeToggle.addEventListener('change', function(e) { - console.log(`[Mode] Toggle changed: ${e.target.checked}`); const mode = e.target.checked ? 1 : 0; - console.log(`[Mode] Sending mode update to server: ${mode}`); fetch(`/settings/analyze-mode/${mode}`, { method: 'POST', }) - .then(response => { - console.log('[Mode] Server response received:', response.status); - return response.json(); - }) + .then(response => response.json()) .then(data => { - console.log('[Mode] Server update response:', data); if (data.success) { updateBadge(data.analyze_mode); showToast(data.message, 'success'); // Store current state in sessionStorage before reload sessionStorage.setItem('analyzeMode', data.analyze_mode); - console.log(`[Mode] Stored analyze mode in session: ${data.analyze_mode}`); // Reload page to ensure all components update - console.log('[Mode] Reloading page in 1 second'); setTimeout(() => window.location.reload(), 1000); } }) @@ -144,24 +105,17 @@ document.addEventListener('DOMContentLoaded', function() { showToast('Failed to update mode', 'error'); // Reset toggle state e.target.checked = !e.target.checked; - console.log(`[Mode] Reset toggle to: ${e.target.checked}`); }); }); // Handle page visibility changes document.addEventListener('visibilitychange', function() { - console.log(`[Mode] Visibility changed: ${document.hidden ? 'hidden' : 'visible'}`); if (!document.hidden) { // When page becomes visible, check and restore theme state const analyzeMode = sessionStorage.getItem('analyzeMode') === 'true'; - console.log(`[Mode] Retrieved analyze mode from session: ${analyzeMode}`); - if (analyzeMode) { const previousTheme = localStorage.getItem('previousTheme') || sessionStorage.getItem('previousTheme'); - console.log(`[Mode] Previous theme found: ${previousTheme}`); - if (previousTheme) { - console.log('[Mode] Setting garden theme on visibility change'); window.themeManager.setTheme('garden'); } } @@ -170,10 +124,8 @@ document.addEventListener('DOMContentLoaded', function() { // Handle storage events for cross-tab consistency window.addEventListener('storage', function(e) { - console.log(`[Mode] Storage event: key=${e.key}, newValue=${e.newValue}`); if (e.key === 'analyzeMode') { const isAnalyzeMode = e.newValue === 'true'; - console.log(`[Mode] Analyze mode changed in another tab: ${isAnalyzeMode}`); modeToggle.checked = isAnalyzeMode; updateBadge(isAnalyzeMode); } diff --git a/static/js/theme.js b/static/js/theme.js index c6e4ec8f..187049a5 100644 --- a/static/js/theme.js +++ b/static/js/theme.js @@ -6,103 +6,79 @@ const themes = ['light', 'dark', 'garden']; // Set theme and persist to localStorage function setTheme(theme, force = false) { - console.log(`[Theme] Setting theme to: ${theme}, force: ${force}`); - console.log(`[Theme] Current theme before change: ${document.documentElement.getAttribute('data-theme')}`); - if (!themes.includes(theme)) { - console.log(`[Theme] Invalid theme: ${theme}, using default: ${defaultTheme}`); theme = defaultTheme; } // Store current theme before changing to garden if (!force && theme === 'garden') { const currentTheme = document.documentElement.getAttribute('data-theme'); - console.log(`[Theme] Storing current theme before garden: ${currentTheme}`); if (currentTheme !== 'garden') { localStorage.setItem(previousThemeKey, currentTheme); sessionStorage.setItem(previousThemeKey, currentTheme); - console.log(`[Theme] Stored previous theme: ${currentTheme}`); } } document.documentElement.setAttribute('data-theme', theme); - console.log(`[Theme] Theme attribute set to: ${document.documentElement.getAttribute('data-theme')}`); // Only update localStorage if not garden theme or if forced if (theme !== 'garden' || force) { localStorage.setItem(themeKey, theme); sessionStorage.setItem(themeKey, theme); - console.log(`[Theme] Theme stored in storage: ${theme}`); } // Update theme controller checkboxes and visibility const themeControllers = document.querySelectorAll('.theme-controller'); const themeSwitcher = document.querySelector('.theme-switcher'); - console.log(`[Theme] Found ${themeControllers.length} theme controllers`); - themeControllers.forEach((controller, index) => { + themeControllers.forEach(controller => { controller.checked = theme === 'dark'; controller.disabled = theme === 'garden'; - console.log(`[Theme] Controller ${index}: checked=${controller.checked}, disabled=${controller.disabled}`); }); // Toggle theme switcher disabled state if (themeSwitcher) { if (theme === 'garden') { themeSwitcher.classList.add('disabled'); - console.log('[Theme] Theme switcher disabled'); } else { themeSwitcher.classList.remove('disabled'); - console.log('[Theme] Theme switcher enabled'); } } // Update mode badge if in garden theme const modeBadge = document.getElementById('mode-badge'); if (modeBadge) { - console.log(`[Theme] Updating mode badge for theme: ${theme}`); if (theme === 'garden') { modeBadge.textContent = 'Analyze Mode'; modeBadge.classList.remove('badge-success'); modeBadge.classList.add('badge-warning'); - console.log('[Theme] Mode badge set to Analyze Mode'); } else { - // Don't update badge text here, let mode-toggle.js handle it modeBadge.classList.remove('badge-warning'); modeBadge.classList.add('badge-success'); - console.log('[Theme] Mode badge classes updated for Live Mode'); } - } else { - console.log('[Theme] Mode badge element not found'); } } // Theme toggle event handler function handleThemeToggle(e) { - console.log(`[Theme] Theme toggle triggered, checked: ${e.target.checked}`); const newTheme = e.target.checked ? 'dark' : 'light'; setTheme(newTheme); } // Initialize theme from localStorage or sessionStorage function initializeTheme() { - console.log('[Theme] Initializing theme'); // Check sessionStorage first for navigation consistency const sessionTheme = sessionStorage.getItem(themeKey); const savedTheme = sessionTheme || localStorage.getItem(themeKey) || defaultTheme; - console.log(`[Theme] Retrieved theme - session: ${sessionTheme}, local: ${localStorage.getItem(themeKey)}, using: ${savedTheme}`); // Set theme without triggering storage events document.documentElement.setAttribute('data-theme', savedTheme); - console.log(`[Theme] Initial theme attribute set to: ${savedTheme}`); // Update controllers const themeControllers = document.querySelectorAll('.theme-controller'); - console.log(`[Theme] Found ${themeControllers.length} theme controllers during initialization`); - themeControllers.forEach((controller, index) => { + themeControllers.forEach(controller => { controller.checked = savedTheme === 'dark'; controller.disabled = savedTheme === 'garden'; - console.log(`[Theme] Initialized controller ${index}: checked=${controller.checked}, disabled=${controller.disabled}`); }); // Update theme switcher state @@ -110,28 +86,21 @@ function initializeTheme() { if (themeSwitcher) { if (savedTheme === 'garden') { themeSwitcher.classList.add('disabled'); - console.log('[Theme] Theme switcher initialized as disabled'); } else { themeSwitcher.classList.remove('disabled'); - console.log('[Theme] Theme switcher initialized as enabled'); } } // Update mode badge classes only, let mode-toggle.js handle the text const modeBadge = document.getElementById('mode-badge'); if (modeBadge) { - console.log(`[Theme] Initializing mode badge for theme: ${savedTheme}`); if (savedTheme === 'garden') { modeBadge.classList.remove('badge-success'); modeBadge.classList.add('badge-warning'); - console.log('[Theme] Mode badge classes set for Analyze Mode'); } else { modeBadge.classList.remove('badge-warning'); modeBadge.classList.add('badge-success'); - console.log('[Theme] Mode badge classes set for Live Mode'); } - } else { - console.log('[Theme] Mode badge element not found during initialization'); } return savedTheme; @@ -139,53 +108,41 @@ function initializeTheme() { // Function to restore previous theme function restorePreviousTheme() { - console.log('[Theme] Restoring previous theme'); const previousTheme = localStorage.getItem(previousThemeKey) || sessionStorage.getItem(previousThemeKey) || defaultTheme; - console.log(`[Theme] Previous theme found: ${previousTheme}`); if (previousTheme && previousTheme !== 'garden') { setTheme(previousTheme, true); // Clear the stored previous theme after restoration localStorage.removeItem(previousThemeKey); sessionStorage.removeItem(previousThemeKey); - console.log('[Theme] Previous theme restored and cleared from storage'); } else { setTheme(defaultTheme, true); - console.log(`[Theme] No valid previous theme, restored to default: ${defaultTheme}`); } } // Initialize theme immediately to prevent flash (function() { - console.log('[Theme] Immediate theme initialization'); const savedTheme = localStorage.getItem(themeKey) || sessionStorage.getItem(themeKey) || defaultTheme; document.documentElement.setAttribute('data-theme', savedTheme); - console.log(`[Theme] Initial theme set to: ${savedTheme}`); })(); // Add event listeners when DOM is loaded document.addEventListener('DOMContentLoaded', function() { - console.log('[Theme] DOM Content Loaded'); // Initialize theme const currentTheme = initializeTheme(); - console.log(`[Theme] Theme initialized to: ${currentTheme}`); // Set initial state of theme toggles and add event listeners const themeControllers = document.querySelectorAll('.theme-controller'); - console.log(`[Theme] Setting up ${themeControllers.length} theme controllers`); - themeControllers.forEach((controller, index) => { + themeControllers.forEach(controller => { controller.checked = currentTheme === 'dark'; controller.addEventListener('change', handleThemeToggle); - console.log(`[Theme] Controller ${index} setup: checked=${controller.checked}`); }); // Handle page visibility changes document.addEventListener('visibilitychange', function() { - console.log(`[Theme] Visibility changed: ${document.hidden ? 'hidden' : 'visible'}`); if (!document.hidden) { // When page becomes visible, ensure theme is consistent const sessionTheme = sessionStorage.getItem(themeKey); - console.log(`[Theme] Retrieved theme from session: ${sessionTheme}`); if (sessionTheme) { setTheme(sessionTheme, true); } @@ -194,7 +151,6 @@ document.addEventListener('DOMContentLoaded', function() { // Handle storage events for cross-tab consistency window.addEventListener('storage', function(e) { - console.log(`[Theme] Storage event: key=${e.key}, newValue=${e.newValue}`); if (e.key === themeKey) { const newTheme = e.newValue || defaultTheme; setTheme(newTheme, true); diff --git a/templates/base.html b/templates/base.html index aadf99f9..d014c9a8 100644 --- a/templates/base.html +++ b/templates/base.html @@ -4,13 +4,9 @@ @@ -46,7 +42,6 @@ @@ -278,9 +272,7 @@