CP_Week17_예습과제_한예송.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -U vega_datasets notebook vega","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import Libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\nfrom tqdm import tqdm_notebook\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.svm import NuSVR, SVR\nfrom sklearn.metrics import mean_absolute_error\npd.options.display.precision = 15\n\nimport lightgbm as lgb\nimport xgboost as xgb\nimport time\nimport datetime\nfrom catboost import CatBoostRegressor\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import StratifiedKFold, KFold, RepeatedKFold, GroupKFold, GridSearchCV, train_test_split, TimeSeriesSplit\nfrom sklearn import metrics\nfrom sklearn import linear_model\nimport gc\nimport seaborn as sns\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport eli5\nimport shap\nfrom IPython.display import HTML\nimport json\nimport altair as alt\n\nimport networkx as nx\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\n# alt.renderers.enable('notebook')\n\n%env JOBLIB_TEMP_FOLDER=/tmp","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Functions used in this kernel","metadata":{}},{"cell_type":"code","source":"import os\nimport time\nimport datetime\nimport json\nimport gc\nfrom numba import jit\n\nimport numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm import tqdm_notebook\n\nimport lightgbm as lgb\nimport xgboost as xgb\nfrom catboost import CatBoostRegressor, CatBoostClassifier\nfrom sklearn import metrics\n\nfrom itertools import product\n\nimport altair as alt\nfrom altair.vega import v5\nfrom IPython.display import HTML\n\n# using ideas from this kernel: https://www.kaggle.com/notslush/altair-visualization-2018-stackoverflow-survey\ndef prepare_altair():\n    \"\"\"\n    Helper function to prepare altair for working.\n    \"\"\"\n    vega_url = 'https://cdn.jsdelivr.net/npm/vega@' + v5.SCHEMA_VERSION\n    vega_lib_url = 'https://cdn.jsdelivr.net/npm/vega-lib'\n    vega_lite_url = 'https://cdn.jsdelivr.net/npm/vega-lite@' + alt.SCHEMA_VERSION\n    vega_embed_url = 'https://cdn.jsdelivr.net/npm/vega-embed@3'\n    noext = \"?noext\"\n    \n    paths = {\n        'vega': vega_url + noext,\n        'vega-lib': vega_lib_url + noext,\n        'vega-lite': vega_lite_url + noext,\n        'vega-embed': vega_embed_url + noext\n    }\n    \n    workaround = f\"\"\"    requirejs.config({{\n        baseUrl: 'https://cdn.jsdelivr.net/npm/',\n        paths: {paths}\n    }});\n    \"\"\"\n    \n    return workaround\n    \n\ndef add_autoincrement(render_func):\n    # Keep track of unique <div/> IDs\n    cache = {}\n    def wrapped(chart, id=\"vega-chart\", autoincrement=True):\n        if autoincrement:\n            if id in cache:\n                counter = 1 + cache[id]\n                cache[id] = counter\n            else:\n                cache[id] = 0\n            actual_id = id if cache[id] == 0 else id + '-' + str(cache[id])\n        else:\n            if id not in cache:\n                cache[id] = 0\n            actual_id = id\n        return render_func(chart, id=actual_id)\n    # Cache will stay outside and \n    return wrapped\n           \n\n@add_autoincrement\ndef render(chart, id=\"vega-chart\"):\n    \"\"\"\n    Helper function to plot altair visualizations.\n    \"\"\"\n    chart_str = \"\"\"\n    <div id=\"{id}\"></div><script>\n    require([\"vega-embed\"], function(vg_embed) {{\n        const spec = {chart};     \n        vg_embed(\"#{id}\", spec, {{defaultStyle: true}}).catch(console.warn);\n        console.log(\"anything?\");\n    }});\n    console.log(\"really...anything?\");\n    </script>\n    \"\"\"\n    return HTML(\n        chart_str.format(\n            id=id,\n            chart=json.dumps(chart) if isinstance(chart, dict) else chart.to_json(indent=None)\n        )\n    )\n    \n\ndef reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage(deep=True).sum() / 1024**2\n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)\n            else:\n                c_prec = df[col].apply(lambda x: np.finfo(x).precision).max()\n                if c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max and c_prec == np.finfo(np.float32).precision:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n    end_mem = df.memory_usage(deep=True).sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df\n    \n\n@jit\ndef fast_auc(y_true, y_prob):\n    \"\"\"\n    fast roc_auc computation: https://www.kaggle.com/c/microsoft-malware-prediction/discussion/76013\n    \"\"\"\n    y_true = np.asarray(y_true)\n    y_true = y_true[np.argsort(y_prob)]\n    nfalse = 0\n    auc = 0\n    n = len(y_true)\n    for i in range(n):\n        y_i = y_true[i]\n        nfalse += (1 - y_i)\n        auc += y_i * nfalse\n    auc /= (nfalse * (n - nfalse))\n    return auc\n\n\ndef eval_auc(y_true, y_pred):\n    \"\"\"\n    Fast auc eval function for lgb.\n    \"\"\"\n    return 'auc', fast_auc(y_true, y_pred), True\n\n\ndef group_mean_log_mae(y_true, y_pred, types, floor=1e-9):\n    \"\"\"\n    Fast metric computation for this competition: https://www.kaggle.com/c/champs-scalar-coupling\n    Code is from this kernel: https://www.kaggle.com/uberkinder/efficient-metric\n    \"\"\"\n    maes = (y_true-y_pred).abs().groupby(types).mean()\n    return np.log(maes.map(lambda x: max(x, floor))).mean()\n    \n\ndef train_model_regression(X, X_test, y, params, folds=None, model_type='lgb', eval_metric='mae', columns=None, plot_feature_importance=False, model=None,\n                               verbose=10000, early_stopping_rounds=200, n_estimators=50000, splits=None, n_folds=3):\n    \"\"\"\n    A function to train a variety of regression models.\n    Returns dictionary with oof predictions, test predictions, scores and, if necessary, feature importances.\n    \n    :params: X - training data, can be pd.DataFrame or np.ndarray (after normalizing)\n    :params: X_test - test data, can be pd.DataFrame or np.ndarray (after normalizing)\n    :params: y - target\n    :params: folds - folds to split data\n    :params: model_type - type of model to use\n    :params: eval_metric - metric to use\n    :params: columns - columns to use. If None - use all columns\n    :params: plot_feature_importance - whether to plot feature importance of LGB\n    :params: model - sklearn model, works only for \"sklearn\" model type\n    \"\"\"\n    columns = X.columns if columns is None else columns\n    X_test = X_test[columns]\n    splits = folds.split(X) if splits is None else splits\n    n_splits = folds.n_splits if splits is None else n_folds\n    \n    # to set up scoring parameters\n    metrics_dict = {'mae': {'lgb_metric_name': 'mae',\n                        'catboost_metric_name': 'MAE',\n                        'sklearn_scoring_function': metrics.mean_absolute_error},\n                    'group_mae': {'lgb_metric_name': 'mae',\n                        'catboost_metric_name': 'MAE',\n                        'scoring_function': group_mean_log_mae},\n                    'mse': {'lgb_metric_name': 'mse',\n                        'catboost_metric_name': 'MSE',\n                        'sklearn_scoring_function': metrics.mean_squared_error}\n                    }\n\n    \n    result_dict = {}\n    \n    # out-of-fold predictions on train data\n    oof = np.zeros(len(X))\n    \n    # averaged predictions on train data\n    prediction = np.zeros(len(X_test))\n    \n    # list of scores on folds\n    scores = []\n    feature_importance = pd.DataFrame()\n    \n    # split and train on folds\n    for fold_n, (train_index, valid_index) in enumerate(splits):\n        if verbose:\n            print(f'Fold {fold_n + 1} started at {time.ctime()}')\n        if type(X) == np.ndarray:\n            X_train, X_valid = X[columns][train_index], X[columns][valid_index]\n            y_train, y_valid = y[train_index], y[valid_index]\n        else:\n            X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]\n            y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]\n            \n        if model_type == 'lgb':\n            model = lgb.LGBMRegressor(**params, n_estimators = n_estimators, n_jobs = -1)\n            model.fit(X_train, y_train, \n                    eval_set=[(X_train, y_train), (X_valid, y_valid)], eval_metric=metrics_dict[eval_metric]['lgb_metric_name'],\n                    verbose=verbose, early_stopping_rounds=early_stopping_rounds)\n            \n            y_pred_valid = model.predict(X_valid)\n            y_pred = model.predict(X_test, num_iteration=model.best_iteration_)\n            \n        if model_type == 'xgb':\n            train_data = xgb.DMatrix(data=X_train, label=y_train, feature_names=X.columns)\n            valid_data = xgb.DMatrix(data=X_valid, label=y_valid, feature_names=X.columns)\n\n            watchlist = [(train_data, 'train'), (valid_data, 'valid_data')]\n            model = xgb.train(dtrain=train_data, num_boost_round=20000, evals=watchlist, early_stopping_rounds=200, verbose_eval=verbose, params=params)\n            y_pred_valid = model.predict(xgb.DMatrix(X_valid, feature_names=X.columns), ntree_limit=model.best_ntree_limit)\n            y_pred = model.predict(xgb.DMatrix(X_test, feature_names=X.columns), ntree_limit=model.best_ntree_limit)\n        \n        if model_type == 'sklearn':\n            model = model\n            model.fit(X_train, y_train)\n            \n            y_pred_valid = model.predict(X_valid).reshape(-1,)\n            score = metrics_dict[eval_metric]['sklearn_scoring_function'](y_valid, y_pred_valid)\n            print(f'Fold {fold_n}. {eval_metric}: {score:.4f}.')\n            print('')\n            \n            y_pred = model.predict(X_test).reshape(-1,)\n        \n        if model_type == 'cat':\n            model = CatBoostRegressor(iterations=20000,  eval_metric=metrics_dict[eval_metric]['catboost_metric_name'], **params,\n                                      loss_function=metrics_dict[eval_metric]['catboost_metric_name'])\n            model.fit(X_train, y_train, eval_set=(X_valid, y_valid), cat_features=[], use_best_model=True, verbose=False)\n\n            y_pred_valid = model.predict(X_valid)\n            y_pred = model.predict(X_test)\n        \n        oof[valid_index] = y_pred_valid.reshape(-1,)\n        if eval_metric != 'group_mae':\n            scores.append(metrics_dict[eval_metric]['sklearn_scoring_function'](y_valid, y_pred_valid))\n        else:\n            scores.append(metrics_dict[eval_metric]['scoring_function'](y_valid, y_pred_valid, X_valid['type']))\n\n        prediction += y_pred    \n        \n        if model_type == 'lgb' and plot_feature_importance:\n            # feature importance\n            fold_importance = pd.DataFrame()\n            fold_importance[\"feature\"] = columns\n            fold_importance[\"importance\"] = model.feature_importances_\n            fold_importance[\"fold\"] = fold_n + 1\n            feature_importance = pd.concat([feature_importance, fold_importance], axis=0)\n\n    prediction /= n_splits\n    print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))\n    \n    result_dict['oof'] = oof\n    result_dict['prediction'] = prediction\n    result_dict['scores'] = scores\n    \n    if model_type == 'lgb':\n        if plot_feature_importance:\n            feature_importance[\"importance\"] /= n_splits\n            cols = feature_importance[[\"feature\", \"importance\"]].groupby(\"feature\").mean().sort_values(\n                by=\"importance\", ascending=False)[:50].index\n\n            best_features = feature_importance.loc[feature_importance.feature.isin(cols)]\n\n            plt.figure(figsize=(16, 12));\n            sns.barplot(x=\"importance\", y=\"feature\", data=best_features.sort_values(by=\"importance\", ascending=False));\n            plt.title('LGB Features (avg over folds)');\n            \n            result_dict['feature_importance'] = feature_importance\n        \n    return result_dict\n    \n\n\ndef train_model_classification(X, X_test, y, params, folds, model_type='lgb', eval_metric='auc', columns=None, plot_feature_importance=False, model=None,\n                               verbose=10000, early_stopping_rounds=200, n_estimators=50000, splits=None, n_folds=3, averaging='usual', n_jobs=-1):\n    \"\"\"\n    A function to train a variety of classification models.\n    Returns dictionary with oof predictions, test predictions, scores and, if necessary, feature importances.\n    \n    :params: X - training data, can be pd.DataFrame or np.ndarray (after normalizing)\n    :params: X_test - test data, can be pd.DataFrame or np.ndarray (after normalizing)\n    :params: y - target\n    :params: folds - folds to split data\n    :params: model_type - type of model to use\n    :params: eval_metric - metric to use\n    :params: columns - columns to use. If None - use all columns\n    :params: plot_feature_importance - whether to plot feature importance of LGB\n    :params: model - sklearn model, works only for \"sklearn\" model type\n    \"\"\"\n    columns = X.columns if columns is None else columns\n    n_splits = folds.n_splits if splits is None else n_folds\n    X_test = X_test[columns]\n    \n    # to set up scoring parameters\n    metrics_dict = {'auc': {'lgb_metric_name': eval_auc,\n                        'catboost_metric_name': 'AUC',\n                        'sklearn_scoring_function': metrics.roc_auc_score},\n                    }\n    \n    result_dict = {}\n    if averaging == 'usual':\n        # out-of-fold predictions on train data\n        oof = np.zeros((len(X), 1))\n\n        # averaged predictions on train data\n        prediction = np.zeros((len(X_test), 1))\n        \n    elif averaging == 'rank':\n        # out-of-fold predictions on train data\n        oof = np.zeros((len(X), 1))\n\n        # averaged predictions on train data\n        prediction = np.zeros((len(X_test), 1))\n\n    \n    # list of scores on folds\n    scores = []\n    feature_importance = pd.DataFrame()\n    \n    # split and train on folds\n    for fold_n, (train_index, valid_index) in enumerate(folds.split(X)):\n        print(f'Fold {fold_n + 1} started at {time.ctime()}')\n        if type(X) == np.ndarray:\n            X_train, X_valid = X[columns][train_index], X[columns][valid_index]\n            y_train, y_valid = y[train_index], y[valid_index]\n        else:\n            X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]\n            y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]\n            \n        if model_type == 'lgb':\n            model = lgb.LGBMClassifier(**params, n_estimators=n_estimators, n_jobs = n_jobs)\n            model.fit(X_train, y_train, \n                    eval_set=[(X_train, y_train), (X_valid, y_valid)], eval_metric=metrics_dict[eval_metric]['lgb_metric_name'],\n                    verbose=verbose, early_stopping_rounds=early_stopping_rounds)\n            \n            y_pred_valid = model.predict_proba(X_valid)[:, 1]\n            y_pred = model.predict_proba(X_test, num_iteration=model.best_iteration_)[:, 1]\n            \n        if model_type == 'xgb':\n            train_data = xgb.DMatrix(data=X_train, label=y_train, feature_names=X.columns)\n            valid_data = xgb.DMatrix(data=X_valid, label=y_valid, feature_names=X.columns)\n\n            watchlist = [(train_data, 'train'), (valid_data, 'valid_data')]\n            model = xgb.train(dtrain=train_data, num_boost_round=n_estimators, evals=watchlist, early_stopping_rounds=early_stopping_rounds, verbose_eval=verbose, params=params)\n            y_pred_valid = model.predict(xgb.DMatrix(X_valid, feature_names=X.columns), ntree_limit=model.best_ntree_limit)\n            y_pred = model.predict(xgb.DMatrix(X_test, feature_names=X.columns), ntree_limit=model.best_ntree_limit)\n        \n        if model_type == 'sklearn':\n            model = model\n            model.fit(X_train, y_train)\n            \n            y_pred_valid = model.predict(X_valid).reshape(-1,)\n            score = metrics_dict[eval_metric]['sklearn_scoring_function'](y_valid, y_pred_valid)\n            print(f'Fold {fold_n}. {eval_metric}: {score:.4f}.')\n            print('')\n            \n            y_pred = model.predict_proba(X_test)\n        \n        if model_type == 'cat':\n            model = CatBoostClassifier(iterations=n_estimators, eval_metric=metrics_dict[eval_metric]['catboost_metric_name'], **params,\n                                      loss_function=Logloss)\n            model.fit(X_train, y_train, eval_set=(X_valid, y_valid), cat_features=[], use_best_model=True, verbose=False)\n\n            y_pred_valid = model.predict(X_valid)\n            y_pred = model.predict(X_test)\n        \n        if averaging == 'usual':\n            \n            oof[valid_index] = y_pred_valid.reshape(-1, 1)\n            scores.append(metrics_dict[eval_metric]['sklearn_scoring_function'](y_valid, y_pred_valid))\n            \n            prediction += y_pred.reshape(-1, 1)\n\n        elif averaging == 'rank':\n                                  \n            oof[valid_index] = y_pred_valid.reshape(-1, 1)\n            scores.append(metrics_dict[eval_metric]['sklearn_scoring_function'](y_valid, y_pred_valid))\n                                  \n            prediction += pd.Series(y_pred).rank().values.reshape(-1, 1)        \n        \n        if model_type == 'lgb' and plot_feature_importance:\n            # feature importance\n            fold_importance = pd.DataFrame()\n            fold_importance[\"feature\"] = columns\n            fold_importance[\"importance\"] = model.feature_importances_\n            fold_importance[\"fold\"] = fold_n + 1\n            feature_importance = pd.concat([feature_importance, fold_importance], axis=0)\n\n    prediction /= n_splits\n    \n    print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))\n    \n    result_dict['oof'] = oof\n    result_dict['prediction'] = prediction\n    result_dict['scores'] = scores\n    \n    if model_type == 'lgb':\n        if plot_feature_importance:\n            feature_importance[\"importance\"] /= n_splits\n            cols = feature_importance[[\"feature\", \"importance\"]].groupby(\"feature\").mean().sort_values(\n                by=\"importance\", ascending=False)[:50].index\n\n            best_features = feature_importance.loc[feature_importance.feature.isin(cols)]\n\n            plt.figure(figsize=(16, 12));\n            sns.barplot(x=\"importance\", y=\"feature\", data=best_features.sort_values(by=\"importance\", ascending=False));\n            plt.title('LGB Features (avg over folds)');\n            \n            result_dict['feature_importance'] = feature_importance\n            result_dict['top_columns'] = cols\n        \n    return result_dict\n\n# setting up altair\nworkaround = prepare_altair()\nHTML(\"\".join((\n    \"<script>\",\n    workaround,\n    \"</script>\",\n)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data loading and overview\nData is separated into two datasets: information about the identity of the customer and transaction information.  \nNot all transactions belong to identities, which are available.  \nMaybe it would be possible to use additional transactions to generate new features.","metadata":{}},{"cell_type":"code","source":"folder_path = '../input/ieee-fraud-detection/'\ntrain_identity = pd.read_csv(f'{folder_path}train_identity.csv')\ntrain_transaction = pd.read_csv(f'{folder_path}train_transaction.csv')\ntest_identity = pd.read_csv(f'{folder_path}test_identity.csv')\ntest_transaction = pd.read_csv(f'{folder_path}test_transaction.csv')\nsub = pd.read_csv(f'{folder_path}sample_submission.csv')\n# let's combine the data and work with the whole dataset\ntrain = pd.merge(train_transaction, train_identity, on='TransactionID', how='left')\ntest = pd.merge(test_transaction, test_identity, on='TransactionID', how='left')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Train dataset has {train.shape[0]} rows and {train.shape[1]} columns.')\nprint(f'Test dataset has {test.shape[0]} rows and {test.shape[1]} columns.')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_transaction.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_identity.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_identity, train_transaction, test_identity, test_transaction","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'There are {train.isnull().any().sum()} columns in train dataset with missing values.')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"one_value_cols = [col for col in train.columns if train[col].nunique() <= 1]\none_value_cols_test = [col for col in test.columns if test[col].nunique() <= 1]\none_value_cols == one_value_cols_test","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'There are {len(one_value_cols)} columns in train dataset with one unique value.')\nprint(f'There are {len(one_value_cols_test)} columns in test dataset with one unique value.')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Exploration","metadata":{}},{"cell_type":"code","source":"plt.hist(train['id_01'], bins=77);\nplt.title('Distribution of id_01 variable');","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"id_01 has an interesting distribution: it has 77 unique non-positive values with skeweness to 0","metadata":{}},{"cell_type":"code","source":"train['id_03'].value_counts(dropna=False, normalize=True).head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"id_03 has 88% of missing values and 98% of values are either missing or equal to 0","metadata":{}},{"cell_type":"code","source":"train['id_11'].value_counts(dropna=False, normalize=True).head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"22% of values in id_11 are equal to 100and 76% are missing. Quite strange","metadata":{}},{"cell_type":"code","source":"plt.hist(train['id_07']);\nplt.title('Distribution of id_07 variable');","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Some of features seem to be normalized. So if someone wants to normalize all variables, it would be necessary to separate such variables which seem to be already normalized.","metadata":{}},{"cell_type":"code","source":"charts = {}\nfor i in ['id_12', 'id_15', 'id_16', 'id_28', 'id_29', 'id_30', 'id_31', 'id_32', 'id_33', 'id_34', 'id_35', 'id_36', 'id_37', 'id_38']:\n    feature_count = train[i].value_counts(dropna=False).reset_index().rename(columns={i: 'count', 'index': i})\n    chart = alt.Chart(feature_count).mark_bar().encode(\n                y=alt.Y(f\"{i}:N\", axis=alt.Axis(title=i)),\n                x=alt.X('count:Q', axis=alt.Axis(title='Count')),\n                tooltip=[i, 'count']\n            ).properties(title=f\"Counts of {i}\", width=400)\n    charts[i] = chart                         \nrender((charts['id_12'] | charts['id_15'] | charts['id_16']) & (charts['id_28'] | charts['id_29'] | charts['id_32']) & (charts['id_34'] | charts['id_35'] | charts['id_36']) & (charts['id_37'] | charts['id_38']))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We have several features showing some kind of \"found\" status and several binary columns.","metadata":{}},{"cell_type":"code","source":"charts = {}\nfor i in ['id_30', 'id_31', 'id_33', 'DeviceType', 'DeviceInfo']:\n    feature_count = train[i].value_counts(dropna=False)[:40].reset_index().rename(columns={i: 'count', 'index': i})\n    chart = alt.Chart(feature_count).mark_bar().encode(\n                x=alt.X(f\"{i}:N\", axis=alt.Axis(title=i)),\n                y=alt.Y('count:Q', axis=alt.Axis(title='Count')),\n                tooltip=[i, 'count']\n            ).properties(title=f\"Counts of {i}\", width=800)\n    charts[i] = chart\nrender(charts['id_30'] & charts['id_31'] & charts['id_33'] & charts['DeviceType'] & charts['DeviceInfo'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here we can see some information about client's device.  \nIt is important to be careful here - some of info could be for old devices and may be absent from test data.  \nNow let's have a look at transaction data.","metadata":{}},{"cell_type":"code","source":"plt.hist(train['TransactionDT'], label='train');\nplt.hist(test['TransactionDT'], label='test');\nplt.legend();\nplt.title('Distribution of transactiond dates');","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"A very important idea: it seems that train and test transaction dates don't overlap, so it would be prudent to use time-based split for validation. ","metadata":{}},{"cell_type":"code","source":"charts = {}\nfor i in ['ProductCD', 'card4', 'card6', 'M4', 'M1', 'M2', 'M3', 'M5', 'M6', 'M7', 'M8', 'M9']:\n    feature_count = train[i].value_counts(dropna=False).reset_index().rename(columns={i: 'count', 'index': i})\n    chart = alt.Chart(feature_count).mark_bar().encode(\n                y=alt.Y(f\"{i}:N\", axis=alt.Axis(title=i)),\n                x=alt.X('count:Q', axis=alt.Axis(title='Count')),\n                tooltip=[i, 'count']\n            ).properties(title=f\"Counts of {i}\", width=400)\n    charts[i] = chart                         \n\nrender((charts['ProductCD'] | charts['card4']) & (charts['card6'] | charts['M4']) & (charts['card6'] | charts['M4']) & (charts['M1'] | charts['M2']) & (charts['M3'] | charts['M5']) & (charts['M6'] | charts['M7']) & (charts['M8'] | charts['M9']))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So card6 is type of card, card4 is credit card company.","metadata":{}},{"cell_type":"code","source":"charts = {}\nfor i in ['P_emaildomain', 'R_emaildomain', 'card1', 'card2', 'card3',  'card5', 'addr1', 'addr2']:\n    feature_count = train[i].value_counts(dropna=False).reset_index()[:40].rename(columns={i: 'count', 'index': i})\n    chart = alt.Chart(feature_count).mark_bar().encode(\n                x=alt.X(f\"{i}:N\", axis=alt.Axis(title=i)),\n                y=alt.Y('count:Q', axis=alt.Axis(title='Count')),\n                tooltip=[i, 'count']\n            ).properties(title=f\"Counts of {i}\", width=600)\n    charts[i] = chart\nrender((charts['P_emaildomain'] | charts['R_emaildomain']) & (charts['card1'] | charts['card2']) & (charts['card3'] | charts['card5']) & (charts['addr1'] | charts['addr2']))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature engineering\nLet's create some aggregations. There is no logic in them - simply aggregations on top features.","metadata":{}},{"cell_type":"code","source":"train['TransactionAmt_to_mean_card1'] = train['TransactionAmt'] / train.groupby(['card1'])['TransactionAmt'].transform('mean')\ntrain['TransactionAmt_to_mean_card4'] = train['TransactionAmt'] / train.groupby(['card4'])['TransactionAmt'].transform('mean')\ntrain['TransactionAmt_to_std_card1'] = train['TransactionAmt'] / train.groupby(['card1'])['TransactionAmt'].transform('std')\ntrain['TransactionAmt_to_std_card4'] = train['TransactionAmt'] / train.groupby(['card4'])['TransactionAmt'].transform('std')\n\ntest['TransactionAmt_to_mean_card1'] = test['TransactionAmt'] / test.groupby(['card1'])['TransactionAmt'].transform('mean')\ntest['TransactionAmt_to_mean_card4'] = test['TransactionAmt'] / test.groupby(['card4'])['TransactionAmt'].transform('mean')\ntest['TransactionAmt_to_std_card1'] = test['TransactionAmt'] / test.groupby(['card1'])['TransactionAmt'].transform('std')\ntest['TransactionAmt_to_std_card4'] = test['TransactionAmt'] / test.groupby(['card4'])['TransactionAmt'].transform('std')\n\ntrain['id_02_to_mean_card1'] = train['id_02'] / train.groupby(['card1'])['id_02'].transform('mean')\ntrain['id_02_to_mean_card4'] = train['id_02'] / train.groupby(['card4'])['id_02'].transform('mean')\ntrain['id_02_to_std_card1'] = train['id_02'] / train.groupby(['card1'])['id_02'].transform('std')\ntrain['id_02_to_std_card4'] = train['id_02'] / train.groupby(['card4'])['id_02'].transform('std')\n\ntest['id_02_to_mean_card1'] = test['id_02'] / test.groupby(['card1'])['id_02'].transform('mean')\ntest['id_02_to_mean_card4'] = test['id_02'] / test.groupby(['card4'])['id_02'].transform('mean')\ntest['id_02_to_std_card1'] = test['id_02'] / test.groupby(['card1'])['id_02'].transform('std')\ntest['id_02_to_std_card4'] = test['id_02'] / test.groupby(['card4'])['id_02'].transform('std')\n\ntrain['D15_to_mean_card1'] = train['D15'] / train.groupby(['card1'])['D15'].transform('mean')\ntrain['D15_to_mean_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('mean')\ntrain['D15_to_std_card1'] = train['D15'] / train.groupby(['card1'])['D15'].transform('std')\ntrain['D15_to_std_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('std')\n\ntest['D15_to_mean_card1'] = test['D15'] / test.groupby(['card1'])['D15'].transform('mean')\ntest['D15_to_mean_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('mean')\ntest['D15_to_std_card1'] = test['D15'] / test.groupby(['card1'])['D15'].transform('std')\ntest['D15_to_std_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('std')\n\ntrain['D15_to_mean_addr1'] = train['D15'] / train.groupby(['addr1'])['D15'].transform('mean')\ntrain['D15_to_mean_addr2'] = train['D15'] / train.groupby(['addr2'])['D15'].transform('mean')\ntrain['D15_to_std_addr1'] = train['D15'] / train.groupby(['addr1'])['D15'].transform('std')\ntrain['D15_to_std_addr2'] = train['D15'] / train.groupby(['addr2'])['D15'].transform('std')\n\ntest['D15_to_mean_addr1'] = test['D15'] / test.groupby(['addr1'])['D15'].transform('mean')\ntest['D15_to_mean_addr2'] = test['D15'] / test.groupby(['addr2'])['D15'].transform('mean')\ntest['D15_to_std_addr1'] = test['D15'] / test.groupby(['addr1'])['D15'].transform('std')\ntest['D15_to_std_addr2'] = test['D15'] / test.groupby(['addr2'])['D15'].transform('std')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[['P_emaildomain_1', 'P_emaildomain_2', 'P_emaildomain_3']] = train['P_emaildomain'].str.split('.', expand=True)\ntrain[['R_emaildomain_1', 'R_emaildomain_2', 'R_emaildomain_3']] = train['R_emaildomain'].str.split('.', expand=True)\ntest[['P_emaildomain_1', 'P_emaildomain_2', 'P_emaildomain_3']] = test['P_emaildomain'].str.split('.', expand=True)\ntest[['R_emaildomain_1', 'R_emaildomain_2', 'R_emaildomain_3']] = test['R_emaildomain'].str.split('.', expand=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare data for modeling","metadata":{}},{"cell_type":"code","source":"many_null_cols = [col for col in train.columns if train[col].isnull().sum() / train.shape[0] > 0.9]\nmany_null_cols_test = [col for col in test.columns if test[col].isnull().sum() / test.shape[0] > 0.9]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"big_top_value_cols = [col for col in train.columns if train[col].value_counts(dropna=False, normalize=True).values[0] > 0.9]\nbig_top_value_cols_test = [col for col in test.columns if test[col].value_counts(dropna=False, normalize=True).values[0] > 0.9]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols_to_drop = list(set(many_null_cols + many_null_cols_test + big_top_value_cols + big_top_value_cols_test + one_value_cols+ one_value_cols_test))\ncols_to_drop.remove('isFraud')\nlen(cols_to_drop)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.drop(cols_to_drop, axis=1)\ntest = test.drop(cols_to_drop, axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_cols = ['id_12', 'id_13', 'id_14', 'id_15', 'id_16', 'id_17', 'id_18', 'id_19', 'id_20', 'id_21', 'id_22', 'id_23', 'id_24', 'id_25', 'id_26', 'id_27', 'id_28', 'id_29',\n            'id_30', 'id_31', 'id_32', 'id_33', 'id_34', 'id_35', 'id_36', 'id_37', 'id_38', 'DeviceType', 'DeviceInfo', 'ProductCD', 'card4', 'card6', 'M4','P_emaildomain',\n            'R_emaildomain', 'card1', 'card2', 'card3',  'card5', 'addr1', 'addr2', 'M1', 'M2', 'M3', 'M5', 'M6', 'M7', 'M8', 'M9',\n            'P_emaildomain_1', 'P_emaildomain_2', 'P_emaildomain_3', 'R_emaildomain_1', 'R_emaildomain_2', 'R_emaildomain_3']\nfor col in cat_cols:\n    if col in train.columns:\n        le = LabelEncoder()\n        le.fit(list(train[col].astype(str).values) + list(test[col].astype(str).values))\n        train[col] = le.transform(list(train[col].astype(str).values))\n        test[col] = le.transform(list(test[col].astype(str).values))   ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train.sort_values('TransactionDT').drop(['isFraud', 'TransactionDT', 'TransactionID'], axis=1)\ny = train.sort_values('TransactionDT')['isFraud']\n#X_test = test.sort_values('TransactionDT').drop(['TransactionDT', 'TransactionID'], axis=1)\nX_test = test.drop(['TransactionDT', 'TransactionID'], axis=1)\ndel train\ntest = test[[\"TransactionDT\", 'TransactionID']]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_inf_nan(df):\n    return df.replace([np.inf, -np.inf], np.nan)   \n\n# Cleaning infinite values to NaN\nX = clean_inf_nan(X)\nX_test = clean_inf_nan(X_test )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# LGBM","metadata":{}},{"cell_type":"code","source":"n_fold = 5\nfolds = TimeSeriesSplit(n_splits=n_fold)\nfolds = KFold(n_splits=5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {'num_leaves': 256,\n          'min_child_samples': 79,\n          'objective': 'binary',\n          'max_depth': 13,\n          'learning_rate': 0.03,\n          \"boosting_type\": \"gbdt\",\n          \"subsample_freq\": 3,\n          \"subsample\": 0.9,\n          \"bagging_seed\": 11,\n          \"metric\": 'auc',\n          \"verbosity\": -1,\n          'reg_alpha': 0.3,\n          'reg_lambda': 0.3,\n          'colsample_bytree': 0.9\n         }\nresult_dict_lgb = train_model_classification(X=X, X_test=X_test, y=y, params=params, folds=folds, model_type='lgb', eval_metric='auc', plot_feature_importance=True,\n                                                      verbose=500, early_stopping_rounds=200, n_estimators=5000, averaging='usual', n_jobs=-1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub['isFraud'] = result_dict_lgb['prediction']\nsub.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(result_dict_lgb['oof']).to_csv('lgb_oof.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}