Permalink
Browse files

Merge pull request #6 from annetkerr/anne/news2

Anne/news2
  • Loading branch information...
annetkerr committed Apr 18, 2019
2 parents 9e711c5 + 424e2ac commit 52436b90ebda93ed15d26992ee22f8635342a662
Showing with 22,223 additions and 0 deletions.
  1. +142 −0 code/EDA_VIS.ipynb
  2. +22,081 −0 code/News_Table_Create_Interval.ipynb
@@ -0,0 +1,142 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import requests\n",
"import time\n",
"import pandas as pd\n",
"import datetime"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"#save a local copy file\n",
"now = time.time()\n",
"\n",
"import warnings;\n",
"warnings.simplefilter('ignore')\n",
"\n",
"from sqlalchemy import create_engine\n",
"import psycopg2\n",
"import hashlib"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"# Write the posts data to a SQL table\n",
"def write_to_database(df, engine = None, table = None, chunksize = 1000, exist_action='replace'):\n",
" # write posts to the posts table\n",
" if engine == None:\n",
" return\n",
" else:\n",
" df.to_sql(table, con=engine, index=False, if_exists=exist_action)\n",
"\n",
" \n",
"# Read in data from SQL database\n",
"def read_from_database(SQL, engine = None):\n",
" # write posts to the posts table\n",
" if engine == None:\n",
" return\n",
" return pd.read_sql(SQL, con = engine)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"# conn_str = 'postgres://postgres:' + str(password) + '@54.200.180.8:5432'\n",
"conn_str = 'postgres://postgres:A%A$TSHBSA@54.200.180.8:5432'\n",
"\n",
"# Database engine\n",
"engine = create_engine(conn_str)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"#read news data\n",
"news_table_name = 'news_camp_fire'\n",
"SQL = f'SELECT * FROM {news_table_name}'\n",
"dfnews = read_from_database(SQL, engine = engine)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(20869, 9)"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dfnews.shape"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python [conda env:root]",
"language": "python",
"name": "conda-root-py"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.8"
},
"toc": {
"base_numbering": 1,
"nav_menu": {},
"number_sections": true,
"sideBar": true,
"skip_h1_title": false,
"title_cell": "Table of Contents",
"title_sidebar": "Contents",
"toc_cell": false,
"toc_position": {},
"toc_section_display": true,
"toc_window_display": false
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Oops, something went wrong.

0 comments on commit 52436b9

Please sign in to comment.