3-4 files update

Author: LoveNui

.gitignore

@@ -319,3 +319,4 @@ venv.bak/
 
 /extraction/
 /load/
+/test.py

Tester.ipynb

@@ -0,0 +1,149 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "collapsed": true
+   },
+   "source": [
+    "### Check outputs from Data Engineering Capstone Project for Udacity"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### Read config"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# import necessary configurations\n",
+    "import configparser\n",
+    "\n",
+    "config = configparser.ConfigParser()\n",
+    "config.read_file(open('dwh.cfg'))\n",
+    "\n",
+    "# amazon aws\n",
+    "KEY = config.get('AWS', 'key')\n",
+    "SECRET = config.get('AWS', 'secret')\n",
+    "\n",
+    "# Redshift\n",
+    "DWH_DB = config.get('DWH', 'DWH_DB')\n",
+    "DWH_DB_USER = config.get('DWH', 'DWH_DB_USER')\n",
+    "DWH_DB_PASSWORD = config.get('DWH', 'DWH_DB_PASSWORD')\n",
+    "DWH_PORT = config.get('DWH', 'DWH_PORT')\n",
+    "DWH_CLUSTER_TYPE = config.get('DWH', 'DWH_CLUSTER_TYPE')\n",
+    "DWH_NUM_NODES = config.get('DWH', 'DWH_NUM_NODES')\n",
+    "DWH_NODE_TYPE = config.get('DWH', 'DWH_NODE_TYPE')\n",
+    "DWH_IAM_ROLE_NAME = config.get('DWH', 'DWH_IAM_ROLE_NAME')\n",
+    "DWH_CLUSTER_IDENTIFIER = config.get('DWH', 'DWH_CLUSTER_IDENTIFIER')\n",
+    "DWH_SCHEMA = config.get('DWH', 'DWH_SCHEMA')\n",
+    "DWH_LOG_STAGING_TABLE = config.get('DWH', 'DWH_LOG_STAGING_TABLE')\n",
+    "DWH_SONG_STAGING_TABLE = config.get('DWH', 'DWH_SONG_STAGING_TABLE')\n",
+    "DWH_REGION = config.get('DWH', 'DWH_REGION')\n",
+    "\n",
+    "# s3\n",
+    "LOG_JSON_FORMAT = config.get('S3', 'LOG_JSON_FORMAT')\n",
+    "S3_BUCKET_LOG_JSON_PATH = config.get('S3', 'S3_BUCKET_LOG_JSON_PATH')\n",
+    "S3_BUCKET_SONG_JSON_PATH = config.get('S3', 'S3_BUCKET_SONG_JSON_PATH')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### Create SQL Connection String"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%load_ext sql"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# save your DWH_ENDPOINT here:\n",
+    "DWH_ROLE_ARN = '<fill_it_up_here>'\n",
+    "DWH_ENDPOINT = '<fill_it_up_here>'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# creating connection to your DB in Amazon Redshift\n",
+    "conn_string=\"postgresql://{}:{}@{}:{}/{}\".format(DWH_DB_USER, DWH_DB_PASSWORD, DWH_ENDPOINT, DWH_PORT, DWH_DB)\n",
+    "print(conn_string)\n",
+    "%sql $conn_string"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### Get result from query"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "from sqlalchemy import create_engine\n",
+    " \n",
+    "test_query = \"\"\"\n",
+    "\n",
+    "\"\"\"\n",
+    " \n",
+    "redshift_engine = create_engine('redshift+psycopg2://username:password@your.redshift.host.123456abcdef.us-east-1.redshift.amazonaws.com:port/database')\n",
+    "test = pd.read_sql_query(test_query, redshift_engine)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}

dwh.cfg

@@ -9,7 +9,7 @@ DWH_NODE_TYPE=dc2.large
 
 DWH_IAM_ROLE_NAME=
 DWH_CLUSTER_IDENTIFIER=
-DWH_DB=
+DWH_DB=udacity
 DWH_DB_USER=
 DWH_DB_PASSWORD=
 DWH_PORT=5439