Scraping

Author: HammadAli08

Web Sraping/Dawn Latest news Scraping.ipynb

@@ -0,0 +1,78 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "initial_id",
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "# STEP 1\n",
+    "BASE_DIR = 'images/'\n",
+    "\n",
+    "# STEP 2\n",
+    "SUB_DIRS = [topic + '/' for topic in df['Topic'].unique()]\n",
+    "\n",
+    "# Print a message to the user\n",
+    "print('Image Download Started...')\n",
+    "start_time = datetime.datetime.now()\n",
+    "\n",
+    "# STEP 3\n",
+    "for sub_dir in SUB_DIRS:\n",
+    "    if not os.path.exists(BASE_DIR + sub_dir):\n",
+    "        os.makedirs(BASE_DIR + sub_dir)\n",
+    "\n",
+    "# STEP 4\n",
+    "for topic in SUB_DIRS:\n",
+    "    # filtering based on topic values\n",
+    "    for ind, row in df[df['Topic'] == topic.split('/')[0]].iterrows():\n",
+    "\n",
+    "        # STEP 5\n",
+    "        urllib.request.urlretrieve(\n",
+    "            row['Thumbnail'],\n",
+    "            '{}{}.jpg'.format(\n",
+    "                BASE_DIR + topic,\n",
+    "                (row['Title']\n",
+    "                     # removing unwanted characters\n",
+    "                    .lower()\n",
+    "                    .replace(' ', '_')\n",
+    "                    .replace('.', '')\n",
+    "                    .replace(',', '')\n",
+    "                    .replace(':', '')\n",
+    "                    .replace('\\'', '')\n",
+    "                    .replace('’', '')\n",
+    "                    .replace('#', '')\n",
+    "                    .replace('*', ''))\n",
+    "            )\n",
+    "        )\n",
+    "\n",
+    "# Tell the user that download has finished\n",
+    "end_time = datetime.datetime.now()\n",
+    "print('\\tDownload Finished! It took {} seconds.'.format(int((end_time - start_time).total_seconds())))\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}