Pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To run the Pipeline (all steps from Sensor events to Activity events) : Please execute following code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-19T05:03:28.752895Z",
     "start_time": "2022-05-19T05:03:28.749469Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import general.utils\n",
    "data=general.utils.loadState('temp')\n",
    "import pandas as pd\n",
    "target=pd.DataFrame(data['meta_targets'])\n",
    "feat=pd.DataFrame(data['meta_features'])\n",
    "print(feat.describe())\n",
    "print(target.describe())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-18T23:44:00.510360Z",
     "start_time": "2022-05-18T23:44:00.329334Z"
    }
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'X' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-b5fec669aca1>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mX\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'X' is not defined"
     ]
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-19T05:03:36.294670Z",
     "start_time": "2022-05-19T05:03:36.288760Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "target=pd.DataFrame(data['meta_targets'])\n",
    "feat=pd.DataFrame(data['meta_features'])\n",
    "\n",
    "# feat['weekday']=feat['time'].dt.dayofweek\n",
    "# feat['month']=feat['time'].dt.month"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-19T05:06:42.884965Z",
     "start_time": "2022-05-19T05:06:42.881182Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.48"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target['f1'].quantile(.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-18T23:44:00.516419Z",
     "start_time": "2022-05-18T23:43:58.000Z"
    }
   },
   "outputs": [],
   "source": [
    "preprocessor.transform(X).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-18T23:44:00.517740Z",
     "start_time": "2022-05-18T23:43:58.000Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.datasets import fetch_openml\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.preprocessing import StandardScaler, OneHotEncoder\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV,cross_validate\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.ensemble import HistGradientBoostingRegressor\n",
    "\n",
    "from sklearn.multioutput import MultiOutputRegressor\n",
    "from sklearn.preprocessing import SplineTransformer\n",
    "\n",
    "\n",
    "\n",
    "# y=target.loc[target['method']=='FixedSlidingWindow'][['size','shift']]\n",
    "# X=feat.loc[target['method']=='FixedSlidingWindow']\n",
    "\n",
    "y=target.drop(['accuracy','precision','recall','f1'],axis=1)\n",
    "X=feat\n",
    "\n",
    "# from sklearn import preprocessing\n",
    "# le = preprocessing.LabelEncoder()\n",
    "# y['method']=le.fit_transform(y['method'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-18T23:44:00.518342Z",
     "start_time": "2022-05-18T23:43:58.001Z"
    }
   },
   "outputs": [],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-18T23:44:00.519092Z",
     "start_time": "2022-05-18T23:43:58.002Z"
    }
   },
   "outputs": [],
   "source": [
    "from feature_extraction.features import TimeSpliter,periodic_spline_transformer\n",
    "numeric_features = feat.columns.drop(['time'])\n",
    "numeric_transformer = StandardScaler()\n",
    "\n",
    "preprocessor = ColumnTransformer(\n",
    "            transformers=[\n",
    "                (\"num\", numeric_transformer, numeric_features),\n",
    "#                 ('date',DateEncoder(),['time']),\n",
    "                ('date',make_pipeline(TimeSpliter(),ColumnTransformer(\n",
    "                    transformers=[\n",
    "                        (\"weekday\", periodic_spline_transformer(7,n_splines=3), [\"time_dayofweek\"]),\n",
    "                        (\"month\", periodic_spline_transformer(12,n_splines=6), [\"time_month\"]),\n",
    "                    ]\n",
    "                )),['time']),\n",
    "            ]\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-18T23:44:00.519664Z",
     "start_time": "2022-05-18T23:43:58.002Z"
    }
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)\n",
    "pipeline=make_pipeline(preprocessor, \n",
    "#                        HistGradientBoostingRegressor()\n",
    "                       MultiOutputRegressor(SVC())\n",
    "                      )\n",
    "# pipeline.fit(X_train,y_train['size'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-18T23:44:00.520219Z",
     "start_time": "2022-05-18T23:43:58.003Z"
    }
   },
   "outputs": [],
   "source": [
    "def evaluate(model, X, y, cv=None):\n",
    "    cv_results = cross_validate(\n",
    "        model,\n",
    "        X,\n",
    "        y,\n",
    "        cv=cv,\n",
    "        scoring=[\"neg_mean_absolute_error\", \"neg_root_mean_squared_error\"],\n",
    "    )\n",
    "    mae = -cv_results[\"test_neg_mean_absolute_error\"]\n",
    "    rmse = -cv_results[\"test_neg_root_mean_squared_error\"]\n",
    "    print(\n",
    "        f\"Mean Absolute Error:     {mae.mean():.3f} +/- {mae.std():.3f}\\n\"\n",
    "        f\"Root Mean Squared Error: {rmse.mean():.3f} +/- {rmse.std():.3f}\"\n",
    "    )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-18T23:44:00.520776Z",
     "start_time": "2022-05-18T23:43:58.004Z"
    }
   },
   "outputs": [],
   "source": [
    "\n",
    "evaluate(pipeline,X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-18T23:44:00.521409Z",
     "start_time": "2022-05-18T23:43:58.005Z"
    }
   },
   "outputs": [],
   "source": [
    "pipeline.score(X_test,y_test['size'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-18T23:44:00.522078Z",
     "start_time": "2022-05-18T23:43:58.006Z"
    }
   },
   "outputs": [],
   "source": [
    "pipeline.fit(X,y)\n",
    "ty=pipeline.predict(X)\n",
    "for i,y2 in enumerate(ty):\n",
    "#     print((y.iloc[i]['size']-y2)*100//y.iloc[i]['size'])\n",
    "    print(y2)\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-18T23:44:00.522795Z",
     "start_time": "2022-05-18T23:43:58.006Z"
    }
   },
   "outputs": [],
   "source": [
    "x_new=preprocessor.fit_transform(X)\n",
    "x_new.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-18T23:44:00.523451Z",
     "start_time": "2022-05-18T23:43:58.007Z"
    }
   },
   "outputs": [],
   "source": [
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "inputs = keras.Input(shape=(x_new.shape[1],))\n",
    "layer1 = layers.Dense(100, activation='relu')(inputs)\n",
    "layer2 = layers.Dense(200, activation='relu')(layer1)\n",
    "layer3 = layers.Dense(100, activation='relu')(layer2)\n",
    "classifier = layers.Dense(1, activation='softmax',name='method')(layer3)\n",
    "regressions = [layers.Dense(1, activation='linear',name=x)(layer3) for x in y.columns.drop('method')]\n",
    "\n",
    "mdl = keras.Model(inputs=inputs, outputs=[classifier, *regressions])\n",
    "\n",
    "mdl.compile(loss=['categorical_crossentropy',*(['mse']*len(regressions))], optimizer='adam', metrics=['accuracy'])\n",
    "from tensorflow.keras.utils import plot_model\n",
    "plot_model(mdl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-18T23:44:00.524524Z",
     "start_time": "2022-05-18T23:43:58.008Z"
    }
   },
   "outputs": [],
   "source": [
    "['a',*(['mse']*10)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-18T23:44:00.525206Z",
     "start_time": "2022-05-18T23:43:58.008Z"
    }
   },
   "outputs": [],
   "source": [
    "mdl.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-18T23:44:00.525837Z",
     "start_time": "2022-05-18T23:43:58.009Z"
    }
   },
   "outputs": [],
   "source": [
    "mdl.fit(x_new,m.transform(y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-18T23:44:00.526557Z",
     "start_time": "2022-05-18T23:43:58.010Z"
    }
   },
   "outputs": [],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-18T23:44:00.527181Z",
     "start_time": "2022-05-18T23:43:58.011Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "y_pr=mdl.predict(x_new)\n",
    "m.inverse_transform(y_pr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-18T23:44:00.527827Z",
     "start_time": "2022-05-18T23:43:58.011Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "y_pr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-18T23:44:00.528455Z",
     "start_time": "2022-05-18T23:43:58.012Z"
    }
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "        \n",
    "m=MyTargetTransformer()\n",
    "m.fit(y)        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-18T12:09:38.158934Z",
     "start_time": "2022-05-18T12:09:38.155127Z"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-18T12:03:34.320170Z",
     "start_time": "2022-05-18T12:03:34.312121Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-18T23:44:00.529190Z",
     "start_time": "2022-05-18T23:43:58.013Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "y2=m.transform(y)\n",
    "# d={c:m.trans[c].inverse_transform(y2[i]).reshape(-1) for i,c in enumerate(m.trans)}\n",
    "# print(pd.DataFrame(d))\n",
    "# for i,c in enumerate(m.trans):\n",
    "#     print(c,i,y_pr[i])\n",
    "#     print(m.trans[c].inverse_transform(y2[i]).reshape(-1))\n",
    "# #     d= {c:self.trans[c].inverse_transform(y[i]) }\n",
    "m.inverse_transform(m.transform(y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-18T23:44:00.529877Z",
     "start_time": "2022-05-18T23:43:58.014Z"
    }
   },
   "outputs": [],
   "source": [
    "pd.DataFrame(np.hstack(y2))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}