amazon_spending_2020.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### notebook to find the amount i spent at Amazon. Amazon said I could save 5K with their 5% cashback card. I wanted to confirm it. \n",
    "### websites said one can get reports from accounts tab but I didn't find any. so wrote this makeshift code to find out what I spent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.chdir('C:\\\\Users\\\\anubr\\\\jupyter_notebooks\\\\web_scrape')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['.ipynb_checkpoints',\n",
       " 'all_text',\n",
       " 'amazon_orders.txt',\n",
       " 'amazon_spend.ipynb',\n",
       " 'amazon_spend_2.ipynb',\n",
       " 'data',\n",
       " 'How to Extract Data from PDF Forms Using Python _ by ankur garg _ Towards Data Science.pdf',\n",
       " 'orders_placed.txt',\n",
       " 'pdf2txt.py',\n",
       " 'test.txt',\n",
       " 'text.txt']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from io import StringIO\n",
    "import io\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pdfminer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pdfminer.converter import TextConverter\n",
    "from pdfminer.layout import LAParams\n",
    "from pdfminer.pdfdocument import PDFDocument\n",
    "from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter\n",
    "from pdfminer.pdfpage import PDFPage\n",
    "from pdfminer.pdfparser import PDFParser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "lst_file=os.listdir(os.getcwd()) # list of files in folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "pdf=lambda x:re.search('.pdf$',x) # lambda function to search for pdf files "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "pdf_file=list(filter(pdf,lst_file)) # filter only pdf files in the list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['page1.pdf',\n",
       " 'page10.pdf',\n",
       " 'page11.pdf',\n",
       " 'page12.pdf',\n",
       " 'page13.pdf',\n",
       " 'page14.pdf',\n",
       " 'page15.pdf',\n",
       " 'page16.pdf',\n",
       " 'page17.pdf',\n",
       " 'page2.pdf',\n",
       " 'page3.pdf',\n",
       " 'page4.pdf',\n",
       " 'page5.pdf',\n",
       " 'page6.pdf',\n",
       " 'page7.pdf',\n",
       " 'page8.pdf',\n",
       " 'page9.pdf']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pdf_file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_string=StringIO()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to convert pdf to text with pdfminer\n",
    "def pdf_to_txt(pdf_list):    \n",
    "    for file in enumerate(pdf_file):\n",
    "        infile=open(file[1],'rb')\n",
    "        outfile=open(str(file[1])[:-4]+'_output.txt','w')\n",
    "        parser=PDFParser(infile)\n",
    "        doc = PDFDocument(parser)\n",
    "        rsrcmgr = PDFResourceManager()\n",
    "        device = TextConverter(rsrcmgr, output_string, laparams=LAParams())\n",
    "        interpreter = PDFPageInterpreter(rsrcmgr, device)\n",
    "        for page in PDFPage.create_pages(doc):\n",
    "            interpreter.process_page(page)\n",
    "\n",
    "        outfile.write(output_string.getvalue())\n",
    "        infile.close()\n",
    "        outfile.close()    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to join output of individual pdf files processed\n",
    "def join_files():   \n",
    "    lst=[]\n",
    "    pattern=re.compile('output.txt$')\n",
    "    for item in os.listdir():\n",
    "        if re.search(pattern,item):\n",
    "            lst.append(item)\n",
    "    outfile=open ('all_text','a')\n",
    "    for file in lst:\n",
    "        infile=open(file,'r')\n",
    "        data=infile.read()\n",
    "        outfile.write(data)\n",
    "        infile.close()\n",
    "    outfile.close()        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "join_files()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to compile the orders placed\n",
    "def spend_list(string=False,file=False):    \n",
    "    with open (file,'r') as txtinfile:\n",
    "        datum=txtinfile.read()\n",
    "    spnd_lst=[]\n",
    "    for inp in re.finditer(string,datum):\n",
    "        strt=inp.start()\n",
    "        new_txt=datum[strt:strt+100]\n",
    "        new_lst=new_txt.split('\\n')\n",
    "        for item in new_lst:\n",
    "            if 'PLACED' in item:\n",
    "                y=new_txt.index(item)        \n",
    "                date=str(new_txt[y:y+30].split('\\n')[1])        \n",
    "\n",
    "            if 'TOTAL' in item:\n",
    "                n=new_txt.index(item)\n",
    "                amount=new_txt[n:n+20].split('\\n')[1].replace(u'\\xa0','').encode('utf-8')\n",
    "                amount=amount.decode('utf-8')\n",
    "                if ',' in amount:\n",
    "                    inp1=amount.split(',')\n",
    "                    amount=\"\".join(inp1)\n",
    "                #namount=float(amount)    \n",
    "        inp_val=(date,amount)\n",
    "        spnd_lst.append(inp_val)\n",
    "    return spnd_lst"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "lst1=spend_list('ORDER PLACED','all_text')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import  pandas as pd\n",
    "pd.set_option('display.max_rows',None)\n",
    "lst_arr=np.asarray(lst1)\n",
    "lst_df=pd.DataFrame(lst_arr)\n",
    "lst_df.columns=['date','amount']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>amount</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3 December 2020</td>\n",
       "      <td>299.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3 December 2020</td>\n",
       "      <td>699.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>29 November 2020</td>\n",
       "      <td>900.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>28 November 2020</td>\n",
       "      <td>399.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>28 November 2020</td>\n",
       "      <td>1170.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               date   amount\n",
       "0   3 December 2020   299.00\n",
       "1   3 December 2020   699.00\n",
       "2  29 November 2020   900.00\n",
       "3  28 November 2020   399.00\n",
       "4  28 November 2020  1170.00"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lst_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>date</td>\n",
       "      <td>object</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>amount</td>\n",
       "      <td>object</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     type   count\n",
       "0    date  object\n",
       "1  amount  object"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_col_type=lst_df.dtypes.reset_index()\n",
    "df_col_type.columns=['type','count']\n",
    "df_col_type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert date to date and amount to float\n",
    "lst_df['date']=lst_df['date'].astype('datetime64[ns]')\n",
    "lst_df['amount']=pd.to_numeric(lst_df['amount'],downcast='float')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>amount</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020-12-03</td>\n",
       "      <td>299.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020-12-03</td>\n",
       "      <td>699.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2020-11-29</td>\n",
       "      <td>900.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2020-11-28</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2020-11-28</td>\n",
       "      <td>1170.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        date  amount\n",
       "0 2020-12-03   299.0\n",
       "1 2020-12-03   699.0\n",
       "2 2020-11-29   900.0\n",
       "3 2020-11-28   399.0\n",
       "4 2020-11-28  1170.0"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lst_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1521, 2)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lst_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>amount</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1440</th>\n",
       "      <td>2020-01-03</td>\n",
       "      <td>265.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>864</th>\n",
       "      <td>2020-01-03</td>\n",
       "      <td>265.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>636</th>\n",
       "      <td>2020-01-03</td>\n",
       "      <td>265.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>993</th>\n",
       "      <td>2020-01-03</td>\n",
       "      <td>265.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1281</th>\n",
       "      <td>2020-01-03</td>\n",
       "      <td>265.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           date  amount\n",
       "1440 2020-01-03   265.0\n",
       "864  2020-01-03   265.0\n",
       "636  2020-01-03   265.0\n",
       "993  2020-01-03   265.0\n",
       "1281 2020-01-03   265.0"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lst_df.sort_values('date').head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "amazon_purchase=lst_df.drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "165 number of times purchases were made in 2020\n"
     ]
    }
   ],
   "source": [
    "print(f'{amazon_purchase.shape[0]} number of times purchases were made in 2020')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total amount spent was Rs.106711.39\n"
     ]
    }
   ],
   "source": [
    "bill=amazon_purchase['amount'].sum(skipna=True,axis=0)\n",
    "print(f'total amount spent was Rs.{bill:6.2f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}