Web Scraping
JEEVAN MEGAVATH
Data Modelling Analyst
Data Scraper
Data Engineer
Jupyter Notebook
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "f236cbb9",
"metadata": {},
"outputs": [],
"source": [
"# import libraries \n",
"\n",
"from bs4 import BeautifulSoup\n",
"import requests\n",
"import time\n",
"import datetime\n",
"\n",
"import smtplib\n"
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "9b531b61",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
" Funny Got Data MIS Data Systems Business Analyst T-Shirt\n",
" \n",
"\n",
" $16.99\n",
" \n"
]
}
],
"source": [
"# Connect to Website and pull in data\n",
"\n",
"URL = 'https://www.amazon.com/Funny-Data-Systems-Business-Analyst/dp/B07FNW9FGJ/ref=sr_1_3?dchild=1&keywords=data%2Banalyst%2Btshirt&qid=1626655184&sr=8-3&customId=B0752XJYNL&th=1'\n",
"\n",
"headers = {\"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36\", \"Accept-Encoding\":\"gzip, deflate\", \"Accept\":\"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\", \"DNT\":\"1\",\"Connection\":\"close\", \"Upgrade-Insecure-Requests\":\"1\"}\n",
"\n",
"page = requests.get(URL, headers=headers)\n",
"\n",
"soup1 = BeautifulSoup(page.content, \"html.parser\")\n",
"\n",
"soup2 = BeautifulSoup(soup1.prettify(), \"html.parser\")\n",
"\n",
"title = soup2.find(id='productTitle').get_text()\n",
"\n",
"price = soup2.find(id='priceblock_ourprice').get_text()\n",
"\n",
"\n",
"print(title)\n",
"print(price)\n",
"\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 17,
"id": "b6f7d66e",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Funny Got Data MIS Data Systems Business Analyst T-Shirt\n",
"16.99\n"
]
}
],
"source": [
"# Clean up the data a little bit\n",
"\n",
"price = price.strip()[1:]\n",
"title = title.strip()\n",
"\n",
"print(title)\n",
"print(price)"
]
},
{
"cell_type": "code",
"execution_count": 21,
"id": "4f021c23",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"2021-08-21\n"
]
}
],
"source": [
"# Create a Timestamp for your output to track when data was collected\n",
"\n",
"import datetime\n",
"\n",
"today = datetime.date.today()\n",
"\n",
"print(today)\n"
]
},
{
"cell_type": "code",
"execution_count": 22,
"id": "14d703ca",
"metadata": {},
"outputs": [],
"source": [
"# Create CSV and write headers and data into the file\n",
"\n",
"import csv \n",
"\n",
"header = ['Title', 'Price', 'Date']\n",
"data = [title, price, today]\n",
"\n",
"\n",
"with open('AmazonWebScraperDataset.csv', 'w', newline='', encoding='UTF8') as f:\n",
" writer = csv.writer(f)\n",
" writer.writerow(header)\n",
" writer.writerow(data)\n",
" \n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d07eeb86",
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"\n",
"df = pd.read_csv(r'C:\\Users\\alexf\\AmazonWebScraperDataset.csv')\n",
"\n",
"print(df)\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 29,
"id": "6b05c1eb",
"metadata": {},
"outputs": [],
"source": [
"#Now we are appending data to the csv\n",
"\n",
"with open('AmazonWebScraperDataset.csv', 'a+', newline='', encoding='UTF8') as f:\n",
" writer = csv.writer(f)\n",
" writer.writerow(data)"
]
},
{
"cell_type": "code",
"execution_count": 31,
"id": "8e95b9e0",
"metadata": {},
"outputs": [],
"source": [
"#Combine all of the above code into one function\n",
"\n",
"\n",
"def check_price():\n",
" URL = 'https://www.amazon.com/Funny-Data-Systems-Business-Analyst/dp/B07FNW9FGJ/ref=sr_1_3?dchild=1&keywords=data%2Banalyst%2Btshirt&qid=1626655184&sr=8-3&customId=B0752XJYNL&th=1'\n",
"\n",
" headers = {\"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36\", \"Accept-Encoding\":\"gzip, deflate\", \"Accept\":\"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\", \"DNT\":\"1\",\"Connection\":\"close\", \"Upgrade-Insecure-Requests\":\"1\"}\n",
"\n",
" page = requests.get(URL, headers=headers)\n",
"\n",
" soup1 = BeautifulSoup(page.content, \"html.parser\")\n",
"\n",
" soup2 = BeautifulSoup(soup1.prettify(), \"html.parser\")\n",
"\n",
" title = soup2.find(id='productTitle').get_text()\n",
"\n",
" price = soup2.find(id='priceblock_ourprice').get_text()\n",
"\n",
" price = price.strip()[1:]\n",
" title = title.strip()\n",
"\n",
" import datetime\n",
"\n",
" today = datetime.date.today()\n",
" \n",
" import csv \n",
"\n",
" header = ['Title', 'Price', 'Date']\n",
" data = [title, price, today]\n",
"\n",
" with open('AmazonWebScraperDataset.csv', 'a+', newline='', encoding='UTF8') as f:\n",
" writer = csv.writer(f)\n",
" writer.writerow(data)\n",
" \n",
" "
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c72f2c4e",
"metadata": {},
"outputs": [],
"source": [
"# Runs check_price after a set time and inputs data into your CSV\n",
"\n",
"while(True):\n",
" check_price()\n",
" time.sleep(86400)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "00af7126",
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"\n",
"df = pd.read_csv(r'C:\\Users\\alexf\\AmazonWebScraperDataset.csv')\n",
"\n",
"print(df)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d14fce5f",
"metadata": {},
"outputs": [],
"source": [
"# If uou want to try sending yourself an email (just for fun) when a price hits below a certain level you can try it\n",
"# out with this script\n",
"\n",
"def send_mail():\n",
" server = smtplib.SMTP_SSL('smtp.gmail.com',465)\n",
" server.ehlo()\n",
" #server.starttls()\n",
" server.ehlo()\n",
" server.login('AlexTheAnalyst95@gmail.com','xxxxxxxxxxxxxx')\n",
" \n",
" subject = \"The Shirt you want is below $15! Now is your chance to buy!\"\n",
" body = \"Alex, This is the moment we have been waiting for. Now is your chance to pick up the shirt of your dreams. Don't mess it up! Link here: https://www.amazon.com/Funny-Data-Systems-Business-Analyst/dp/B07FNW9FGJ/ref=sr_1_3?dchild=1&keywords=data+analyst+tshirt&qid=1626655184&sr=8-3\"\n",
" \n",
" msg = f\"Subject: {subject}\\n\\n{body}\"\n",
" \n",
" server.sendmail(\n",
" 'AlexTheAnalyst95@gmail.com',\n",
" msg\n",
" \n",
" )"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.8"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
More Projects by JEEVAN
Resources
Dіscover Freelancers