{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "asian-tracker",
   "metadata": {},
   "source": [
    "# Briefkastensensor: Analyse der Temperaturabhängigkeit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "subject-cambridge",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import sqlite3"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "tough-construction",
   "metadata": {},
   "source": [
    "## Datengrundlage\n",
    "\n",
    "* Homeassistant SQLite Database (home-assistants.db)\n",
    "* Entity Helligkeit: sensor.briefkasten_zda_measure\n",
    "* Entity Lorapark: sensor.aussentemperatur_lorapark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "civilian-causing",
   "metadata": {},
   "outputs": [
    {
     "ename": "DatabaseError",
     "evalue": "Execution failed on sql 'SELECT state, created FROM states WHERE entity_id = 'sensor.briefkasten_zda_measure'': no such table: states",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mOperationalError\u001b[0m                          Traceback (most recent call last)",
      "\u001b[0;32m~/.local/lib/python3.8/site-packages/pandas/io/sql.py\u001b[0m in \u001b[0;36mexecute\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1696\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1697\u001b[0;31m             \u001b[0mcur\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1698\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mcur\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mOperationalError\u001b[0m: no such table: states",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[0;31mDatabaseError\u001b[0m                             Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-11-8bcec3e0fc38>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mquery2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"SELECT state, created FROM states WHERE entity_id = 'sensor.aussentemperatur_lorapark'\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0mmessung\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_sql\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mquery1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcon\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparse_dates\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex_col\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'created'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m \u001b[0mtemperatur\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_sql\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mquery2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcon\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparse_dates\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex_col\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'created'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.8/site-packages/pandas/io/sql.py\u001b[0m in \u001b[0;36mread_sql\u001b[0;34m(sql, con, index_col, coerce_float, params, parse_dates, columns, chunksize)\u001b[0m\n\u001b[1;32m    482\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    483\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpandas_sql\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mSQLiteDatabase\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 484\u001b[0;31m         return pandas_sql.read_query(\n\u001b[0m\u001b[1;32m    485\u001b[0m             \u001b[0msql\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    486\u001b[0m             \u001b[0mindex_col\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mindex_col\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.8/site-packages/pandas/io/sql.py\u001b[0m in \u001b[0;36mread_query\u001b[0;34m(self, sql, index_col, coerce_float, params, parse_dates, chunksize)\u001b[0m\n\u001b[1;32m   1741\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1742\u001b[0m         \u001b[0margs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_convert_params\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msql\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1743\u001b[0;31m         \u001b[0mcursor\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1744\u001b[0m         \u001b[0mcolumns\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mcol_desc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mcol_desc\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcursor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdescription\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1745\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.8/site-packages/pandas/io/sql.py\u001b[0m in \u001b[0;36mexecute\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1707\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1708\u001b[0m             \u001b[0mex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDatabaseError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"Execution failed on sql '{args[0]}': {exc}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1709\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mex\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1710\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1711\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mstaticmethod\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mDatabaseError\u001b[0m: Execution failed on sql 'SELECT state, created FROM states WHERE entity_id = 'sensor.briefkasten_zda_measure'': no such table: states"
     ]
    }
   ],
   "source": [
    "# SQLite lesen\n",
    "con = sqlite3.connect(\"./home-assistant.db\")\n",
    "\n",
    "query1 = \"SELECT state, created FROM states WHERE entity_id = 'sensor.briefkasten_zda_measure'\"\n",
    "query2 = \"SELECT state, created FROM states WHERE entity_id = 'sensor.aussentemperatur_lorapark'\"\n",
    "\n",
    "messung = pd.read_sql(query1, con, parse_dates=True, index_col='created')\n",
    "temperatur = pd.read_sql(query2, con, parse_dates=True, index_col='created')\n",
    "\n",
    "def drop_non_numeric (df, data_columns):\n",
    "    df_new = (df.drop(data_columns, axis=1).join(df[data_columns].apply(pd.to_numeric, errors='coerce')))\n",
    "    return df_new[df_new[data_columns].notnull().all(axis=1)]\n",
    "\n",
    "# Drop rows where state is \"unavailable\"\n",
    "messung = drop_non_numeric(messung, ['state'])\n",
    "temperatur = drop_non_numeric(temperatur, ['state'])\n",
    "\n",
    "# Daten in Pivot Table zusammenführen\n",
    "messung['sensor'] = 'brightness'\n",
    "temperatur['sensor'] = 'temperature'\n",
    "time_df = pd.concat([messung, temperatur], axis=0)\n",
    "time_df = time_df.pivot_table(index='created', columns='sensor', values='state', aggfunc=np.sum)\n",
    "\n",
    "# Jetzt sind die allermeisten Datensätze nur mit einem Wert (z.B. Helligkeit), der andere (z.B. Temperatur) ist null\n",
    "# Deswegen mit vorherigen Werten auffüllen\n",
    "time_df = time_df.fillna(method='ffill', limit=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "unauthorized-india",
   "metadata": {},
   "source": [
    "## Optional: Daten eingrenzen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "adjacent-traveler",
   "metadata": {},
   "outputs": [],
   "source": [
    "# time_df = time_df[\"2021-03-23\":\"2021-03-24\"]\n",
    "# time_df = time_df[\"2021-03-29\":\"2021-03-30\"]\n",
    "# time_df = time_df[\"2021-04-13\":\"2021-04-14\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "junior-basketball",
   "metadata": {},
   "source": [
    "## Optional: CSV-Export"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "silver-collins",
   "metadata": {},
   "outputs": [],
   "source": [
    "time_df.to_csv(\"export.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "collaborative-range",
   "metadata": {},
   "source": [
    "## Temperatur und Brightness über die Zeit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "proper-mounting",
   "metadata": {},
   "outputs": [],
   "source": [
    "time_df.plot(subplots=True,figsize=(20,20))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "velvet-combining",
   "metadata": {},
   "source": [
    "## Modellierung von Regressionsfunktionen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "emotional-respect",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Nur Datensätze verwenden, wo beide Werte vorhanden sind\n",
    "time_df.dropna(inplace=True)\n",
    "\n",
    "# Generator für Regressionsfunktionen (mit Numpy) - Degree: Grad der gewünschten ganzrationalen Funktion\n",
    "def reg_func (x, y, degree=1):\n",
    "    d = np.polyfit(x, y, degree)\n",
    "    \n",
    "    # <--- Print out function -->\n",
    "    printstr = f\"\\n\"\n",
    "    for i, n in enumerate(d):\n",
    "        if (n >= 0): \n",
    "            n = f\"+{n}\"\n",
    "        if (len(d)-i-1 == 0):\n",
    "            printstr += f\" {n}\"\n",
    "        elif (len(d)-i-1 == 1):\n",
    "            printstr += f\" {n}x\"\n",
    "        else:\n",
    "            printstr += f\" {n}x^{len(d)-i-1}\"\n",
    "    print(printstr)\n",
    "    # <------------------------>\n",
    "    \n",
    "    f = np.poly1d(d)\n",
    "    return f(x)\n",
    "\n",
    "\n",
    "# ------- Add regression functions here -------\n",
    "time_df.loc[:, 'reg-01'] = reg_func(time_df['temperature'], time_df['brightness'], degree=1)\n",
    "time_df.loc[:, 'reg-02'] = reg_func(time_df['temperature'], time_df['brightness'], degree=2)\n",
    "time_df.loc[:, 'reg-03'] = reg_func(time_df['temperature'], time_df['brightness'], degree=3)\n",
    "time_df.loc[:, 'reg-04'] = reg_func(time_df['temperature'], time_df['brightness'], degree=4)\n",
    "time_df.loc[:, 'reg-05'] = reg_func(time_df['temperature'], time_df['brightness'], degree=5)\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "timely-utility",
   "metadata": {},
   "outputs": [],
   "source": [
    "ax = time_df.plot.scatter(x=\"temperature\", y=\"brightness\", figsize=(10,10))\n",
    "\n",
    "\n",
    "# ------- Add regression functions here -------\n",
    "time_df.plot(x='temperature',y='reg-01',color='Red', ax=ax)\n",
    "time_df.plot(x='temperature',y='reg-02',color='Green', ax=ax)\n",
    "time_df.plot(x='temperature',y='reg-03',color='Purple', ax=ax)\n",
    "time_df.plot(x='temperature',y='reg-04',color='Yellow', ax=ax)\n",
    "time_df.plot(x='temperature',y='reg-05',color='Orange', ax=ax)\n",
    "\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}