From 630dba83bc18aa0efe6343c4adbe9b1d6fd64b15 Mon Sep 17 00:00:00 2001
From: Pranav25191 <62637230+Pranav25191@users.noreply.github.com>
Date: Sun, 6 Sep 2020 14:32:44 +0530
Subject: [PATCH] Panav_Balijapelly_Assignment2_Pandas
---
Assignment2 pandas.ipynb | 1107 ++++++++++++++++++++++++++++++++++++++
1 file changed, 1107 insertions(+)
create mode 100644 Assignment2 pandas.ipynb
diff --git a/Assignment2 pandas.ipynb b/Assignment2 pandas.ipynb
new file mode 100644
index 0000000..12aaceb
--- /dev/null
+++ b/Assignment2 pandas.ipynb
@@ -0,0 +1,1107 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 288,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import numpy as np\n",
+ "import pandas as pd\n",
+ "#%matplotlib notebook\n",
+ "%matplotlib inline"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "import the dataset into a dataframe"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 289,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " Id | \n",
+ " EmployeeName | \n",
+ " JobTitle | \n",
+ " BasePay | \n",
+ " OvertimePay | \n",
+ " OtherPay | \n",
+ " Benefits | \n",
+ " TotalPay | \n",
+ " TotalPayBenefits | \n",
+ " Year | \n",
+ " Notes | \n",
+ " Agency | \n",
+ " Status | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " 0 | \n",
+ " 1 | \n",
+ " NATHANIEL FORD | \n",
+ " GENERAL MANAGER-METROPOLITAN TRANSIT AUTHORITY | \n",
+ " 167411.18 | \n",
+ " 0.00 | \n",
+ " 400184.25 | \n",
+ " NaN | \n",
+ " 567595.43 | \n",
+ " 567595.43 | \n",
+ " 2011 | \n",
+ " NaN | \n",
+ " San Francisco | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ " 1 | \n",
+ " 2 | \n",
+ " GARY JIMENEZ | \n",
+ " CAPTAIN III (POLICE DEPARTMENT) | \n",
+ " 155966.02 | \n",
+ " 245131.88 | \n",
+ " 137811.38 | \n",
+ " NaN | \n",
+ " 538909.28 | \n",
+ " 538909.28 | \n",
+ " 2011 | \n",
+ " NaN | \n",
+ " San Francisco | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ " 2 | \n",
+ " 3 | \n",
+ " ALBERT PARDINI | \n",
+ " CAPTAIN III (POLICE DEPARTMENT) | \n",
+ " 212739.13 | \n",
+ " 106088.18 | \n",
+ " 16452.60 | \n",
+ " NaN | \n",
+ " 335279.91 | \n",
+ " 335279.91 | \n",
+ " 2011 | \n",
+ " NaN | \n",
+ " San Francisco | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ " 3 | \n",
+ " 4 | \n",
+ " CHRISTOPHER CHONG | \n",
+ " WIRE ROPE CABLE MAINTENANCE MECHANIC | \n",
+ " 77916.00 | \n",
+ " 56120.71 | \n",
+ " 198306.90 | \n",
+ " NaN | \n",
+ " 332343.61 | \n",
+ " 332343.61 | \n",
+ " 2011 | \n",
+ " NaN | \n",
+ " San Francisco | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ " 4 | \n",
+ " 5 | \n",
+ " PATRICK GARDNER | \n",
+ " DEPUTY CHIEF OF DEPARTMENT,(FIRE DEPARTMENT) | \n",
+ " 134401.60 | \n",
+ " 9737.00 | \n",
+ " 182234.59 | \n",
+ " NaN | \n",
+ " 326373.19 | \n",
+ " 326373.19 | \n",
+ " 2011 | \n",
+ " NaN | \n",
+ " San Francisco | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
"
+ ],
+ "text/plain": [
+ " Id EmployeeName JobTitle \\\n",
+ "0 1 NATHANIEL FORD GENERAL MANAGER-METROPOLITAN TRANSIT AUTHORITY \n",
+ "1 2 GARY JIMENEZ CAPTAIN III (POLICE DEPARTMENT) \n",
+ "2 3 ALBERT PARDINI CAPTAIN III (POLICE DEPARTMENT) \n",
+ "3 4 CHRISTOPHER CHONG WIRE ROPE CABLE MAINTENANCE MECHANIC \n",
+ "4 5 PATRICK GARDNER DEPUTY CHIEF OF DEPARTMENT,(FIRE DEPARTMENT) \n",
+ "\n",
+ " BasePay OvertimePay OtherPay Benefits TotalPay TotalPayBenefits \\\n",
+ "0 167411.18 0.00 400184.25 NaN 567595.43 567595.43 \n",
+ "1 155966.02 245131.88 137811.38 NaN 538909.28 538909.28 \n",
+ "2 212739.13 106088.18 16452.60 NaN 335279.91 335279.91 \n",
+ "3 77916.00 56120.71 198306.90 NaN 332343.61 332343.61 \n",
+ "4 134401.60 9737.00 182234.59 NaN 326373.19 326373.19 \n",
+ "\n",
+ " Year Notes Agency Status \n",
+ "0 2011 NaN San Francisco NaN \n",
+ "1 2011 NaN San Francisco NaN \n",
+ "2 2011 NaN San Francisco NaN \n",
+ "3 2011 NaN San Francisco NaN \n",
+ "4 2011 NaN San Francisco NaN "
+ ]
+ },
+ "execution_count": 289,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "df=pd.read_csv(\"Salaries.csv\")\n",
+ "df.head()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "display the column names"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 290,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "Index(['Id', 'EmployeeName', 'JobTitle', 'BasePay', 'OvertimePay', 'OtherPay',\n",
+ " 'Benefits', 'TotalPay', 'TotalPayBenefits', 'Year', 'Notes', 'Agency',\n",
+ " 'Status'],\n",
+ " dtype='object')"
+ ]
+ },
+ "execution_count": 290,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "df.columns"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "display the number of rows and cols"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 291,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "(148654, 13)"
+ ]
+ },
+ "execution_count": 291,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "df.shape"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "display the dataframe info (types of data in columns and not null values etc.)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 293,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\n",
+ "RangeIndex: 148654 entries, 0 to 148653\n",
+ "Data columns (total 13 columns):\n",
+ " # Column Non-Null Count Dtype \n",
+ "--- ------ -------------- ----- \n",
+ " 0 Id 148654 non-null int64 \n",
+ " 1 EmployeeName 148654 non-null object \n",
+ " 2 JobTitle 148654 non-null object \n",
+ " 3 BasePay 148045 non-null float64\n",
+ " 4 OvertimePay 148650 non-null float64\n",
+ " 5 OtherPay 148650 non-null float64\n",
+ " 6 Benefits 112491 non-null float64\n",
+ " 7 TotalPay 148654 non-null float64\n",
+ " 8 TotalPayBenefits 148654 non-null float64\n",
+ " 9 Year 148654 non-null int64 \n",
+ " 10 Notes 0 non-null float64\n",
+ " 11 Agency 148654 non-null object \n",
+ " 12 Status 0 non-null float64\n",
+ "dtypes: float64(8), int64(2), object(3)\n",
+ "memory usage: 14.7+ MB\n"
+ ]
+ }
+ ],
+ "source": [
+ "df.info()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "display stats of the dataframe like count, mean, std, max, 25% etc....."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 294,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " Id | \n",
+ " BasePay | \n",
+ " OvertimePay | \n",
+ " OtherPay | \n",
+ " Benefits | \n",
+ " TotalPay | \n",
+ " TotalPayBenefits | \n",
+ " Year | \n",
+ " Notes | \n",
+ " Status | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " count | \n",
+ " 148654.000000 | \n",
+ " 148045.000000 | \n",
+ " 148650.000000 | \n",
+ " 148650.000000 | \n",
+ " 112491.000000 | \n",
+ " 148654.000000 | \n",
+ " 148654.000000 | \n",
+ " 148654.000000 | \n",
+ " 0.0 | \n",
+ " 0.0 | \n",
+ "
\n",
+ " \n",
+ " mean | \n",
+ " 74327.500000 | \n",
+ " 66325.448840 | \n",
+ " 5066.059886 | \n",
+ " 3648.767297 | \n",
+ " 25007.893151 | \n",
+ " 74768.321972 | \n",
+ " 93692.554811 | \n",
+ " 2012.522643 | \n",
+ " NaN | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ " std | \n",
+ " 42912.857795 | \n",
+ " 42764.635495 | \n",
+ " 11454.380559 | \n",
+ " 8056.601866 | \n",
+ " 15402.215858 | \n",
+ " 50517.005274 | \n",
+ " 62793.533483 | \n",
+ " 1.117538 | \n",
+ " NaN | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ " min | \n",
+ " 1.000000 | \n",
+ " -166.010000 | \n",
+ " -0.010000 | \n",
+ " -7058.590000 | \n",
+ " -33.890000 | \n",
+ " -618.130000 | \n",
+ " -618.130000 | \n",
+ " 2011.000000 | \n",
+ " NaN | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ " 25% | \n",
+ " 37164.250000 | \n",
+ " 33588.200000 | \n",
+ " 0.000000 | \n",
+ " 0.000000 | \n",
+ " 11535.395000 | \n",
+ " 36168.995000 | \n",
+ " 44065.650000 | \n",
+ " 2012.000000 | \n",
+ " NaN | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ " 50% | \n",
+ " 74327.500000 | \n",
+ " 65007.450000 | \n",
+ " 0.000000 | \n",
+ " 811.270000 | \n",
+ " 28628.620000 | \n",
+ " 71426.610000 | \n",
+ " 92404.090000 | \n",
+ " 2013.000000 | \n",
+ " NaN | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ " 75% | \n",
+ " 111490.750000 | \n",
+ " 94691.050000 | \n",
+ " 4658.175000 | \n",
+ " 4236.065000 | \n",
+ " 35566.855000 | \n",
+ " 105839.135000 | \n",
+ " 132876.450000 | \n",
+ " 2014.000000 | \n",
+ " NaN | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ " max | \n",
+ " 148654.000000 | \n",
+ " 319275.010000 | \n",
+ " 245131.880000 | \n",
+ " 400184.250000 | \n",
+ " 96570.660000 | \n",
+ " 567595.430000 | \n",
+ " 567595.430000 | \n",
+ " 2014.000000 | \n",
+ " NaN | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
"
+ ],
+ "text/plain": [
+ " Id BasePay OvertimePay OtherPay \\\n",
+ "count 148654.000000 148045.000000 148650.000000 148650.000000 \n",
+ "mean 74327.500000 66325.448840 5066.059886 3648.767297 \n",
+ "std 42912.857795 42764.635495 11454.380559 8056.601866 \n",
+ "min 1.000000 -166.010000 -0.010000 -7058.590000 \n",
+ "25% 37164.250000 33588.200000 0.000000 0.000000 \n",
+ "50% 74327.500000 65007.450000 0.000000 811.270000 \n",
+ "75% 111490.750000 94691.050000 4658.175000 4236.065000 \n",
+ "max 148654.000000 319275.010000 245131.880000 400184.250000 \n",
+ "\n",
+ " Benefits TotalPay TotalPayBenefits Year Notes \\\n",
+ "count 112491.000000 148654.000000 148654.000000 148654.000000 0.0 \n",
+ "mean 25007.893151 74768.321972 93692.554811 2012.522643 NaN \n",
+ "std 15402.215858 50517.005274 62793.533483 1.117538 NaN \n",
+ "min -33.890000 -618.130000 -618.130000 2011.000000 NaN \n",
+ "25% 11535.395000 36168.995000 44065.650000 2012.000000 NaN \n",
+ "50% 28628.620000 71426.610000 92404.090000 2013.000000 NaN \n",
+ "75% 35566.855000 105839.135000 132876.450000 2014.000000 NaN \n",
+ "max 96570.660000 567595.430000 567595.430000 2014.000000 NaN \n",
+ "\n",
+ " Status \n",
+ "count 0.0 \n",
+ "mean NaN \n",
+ "std NaN \n",
+ "min NaN \n",
+ "25% NaN \n",
+ "50% NaN \n",
+ "75% NaN \n",
+ "max NaN "
+ ]
+ },
+ "execution_count": 294,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "df.describe()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "display null values per column"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 295,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "Id 0\n",
+ "EmployeeName 0\n",
+ "JobTitle 0\n",
+ "BasePay 609\n",
+ "OvertimePay 4\n",
+ "OtherPay 4\n",
+ "Benefits 36163\n",
+ "TotalPay 0\n",
+ "TotalPayBenefits 0\n",
+ "Year 0\n",
+ "Notes 148654\n",
+ "Agency 0\n",
+ "Status 148654\n",
+ "dtype: int64"
+ ]
+ },
+ "execution_count": 295,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "df.isnull().sum()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "remove columns will all values as NaN"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 296,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " Id | \n",
+ " EmployeeName | \n",
+ " JobTitle | \n",
+ " BasePay | \n",
+ " OvertimePay | \n",
+ " OtherPay | \n",
+ " Benefits | \n",
+ " TotalPay | \n",
+ " TotalPayBenefits | \n",
+ " Year | \n",
+ " Agency | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " 0 | \n",
+ " 1 | \n",
+ " NATHANIEL FORD | \n",
+ " GENERAL MANAGER-METROPOLITAN TRANSIT AUTHORITY | \n",
+ " 167411.18 | \n",
+ " 0.00 | \n",
+ " 400184.25 | \n",
+ " NaN | \n",
+ " 567595.43 | \n",
+ " 567595.43 | \n",
+ " 2011 | \n",
+ " San Francisco | \n",
+ "
\n",
+ " \n",
+ " 1 | \n",
+ " 2 | \n",
+ " GARY JIMENEZ | \n",
+ " CAPTAIN III (POLICE DEPARTMENT) | \n",
+ " 155966.02 | \n",
+ " 245131.88 | \n",
+ " 137811.38 | \n",
+ " NaN | \n",
+ " 538909.28 | \n",
+ " 538909.28 | \n",
+ " 2011 | \n",
+ " San Francisco | \n",
+ "
\n",
+ " \n",
+ " 2 | \n",
+ " 3 | \n",
+ " ALBERT PARDINI | \n",
+ " CAPTAIN III (POLICE DEPARTMENT) | \n",
+ " 212739.13 | \n",
+ " 106088.18 | \n",
+ " 16452.60 | \n",
+ " NaN | \n",
+ " 335279.91 | \n",
+ " 335279.91 | \n",
+ " 2011 | \n",
+ " San Francisco | \n",
+ "
\n",
+ " \n",
+ " 3 | \n",
+ " 4 | \n",
+ " CHRISTOPHER CHONG | \n",
+ " WIRE ROPE CABLE MAINTENANCE MECHANIC | \n",
+ " 77916.00 | \n",
+ " 56120.71 | \n",
+ " 198306.90 | \n",
+ " NaN | \n",
+ " 332343.61 | \n",
+ " 332343.61 | \n",
+ " 2011 | \n",
+ " San Francisco | \n",
+ "
\n",
+ " \n",
+ " 4 | \n",
+ " 5 | \n",
+ " PATRICK GARDNER | \n",
+ " DEPUTY CHIEF OF DEPARTMENT,(FIRE DEPARTMENT) | \n",
+ " 134401.60 | \n",
+ " 9737.00 | \n",
+ " 182234.59 | \n",
+ " NaN | \n",
+ " 326373.19 | \n",
+ " 326373.19 | \n",
+ " 2011 | \n",
+ " San Francisco | \n",
+ "
\n",
+ " \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ "
\n",
+ " \n",
+ " 148649 | \n",
+ " 148650 | \n",
+ " Roy I Tillery | \n",
+ " Custodian | \n",
+ " 0.00 | \n",
+ " 0.00 | \n",
+ " 0.00 | \n",
+ " 0.0 | \n",
+ " 0.00 | \n",
+ " 0.00 | \n",
+ " 2014 | \n",
+ " San Francisco | \n",
+ "
\n",
+ " \n",
+ " 148650 | \n",
+ " 148651 | \n",
+ " Not provided | \n",
+ " Not provided | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " 0.00 | \n",
+ " 0.00 | \n",
+ " 2014 | \n",
+ " San Francisco | \n",
+ "
\n",
+ " \n",
+ " 148651 | \n",
+ " 148652 | \n",
+ " Not provided | \n",
+ " Not provided | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " 0.00 | \n",
+ " 0.00 | \n",
+ " 2014 | \n",
+ " San Francisco | \n",
+ "
\n",
+ " \n",
+ " 148652 | \n",
+ " 148653 | \n",
+ " Not provided | \n",
+ " Not provided | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " 0.00 | \n",
+ " 0.00 | \n",
+ " 2014 | \n",
+ " San Francisco | \n",
+ "
\n",
+ " \n",
+ " 148653 | \n",
+ " 148654 | \n",
+ " Joe Lopez | \n",
+ " Counselor, Log Cabin Ranch | \n",
+ " 0.00 | \n",
+ " 0.00 | \n",
+ " -618.13 | \n",
+ " 0.0 | \n",
+ " -618.13 | \n",
+ " -618.13 | \n",
+ " 2014 | \n",
+ " San Francisco | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
148654 rows × 11 columns
\n",
+ "
"
+ ],
+ "text/plain": [
+ " Id EmployeeName \\\n",
+ "0 1 NATHANIEL FORD \n",
+ "1 2 GARY JIMENEZ \n",
+ "2 3 ALBERT PARDINI \n",
+ "3 4 CHRISTOPHER CHONG \n",
+ "4 5 PATRICK GARDNER \n",
+ "... ... ... \n",
+ "148649 148650 Roy I Tillery \n",
+ "148650 148651 Not provided \n",
+ "148651 148652 Not provided \n",
+ "148652 148653 Not provided \n",
+ "148653 148654 Joe Lopez \n",
+ "\n",
+ " JobTitle BasePay \\\n",
+ "0 GENERAL MANAGER-METROPOLITAN TRANSIT AUTHORITY 167411.18 \n",
+ "1 CAPTAIN III (POLICE DEPARTMENT) 155966.02 \n",
+ "2 CAPTAIN III (POLICE DEPARTMENT) 212739.13 \n",
+ "3 WIRE ROPE CABLE MAINTENANCE MECHANIC 77916.00 \n",
+ "4 DEPUTY CHIEF OF DEPARTMENT,(FIRE DEPARTMENT) 134401.60 \n",
+ "... ... ... \n",
+ "148649 Custodian 0.00 \n",
+ "148650 Not provided NaN \n",
+ "148651 Not provided NaN \n",
+ "148652 Not provided NaN \n",
+ "148653 Counselor, Log Cabin Ranch 0.00 \n",
+ "\n",
+ " OvertimePay OtherPay Benefits TotalPay TotalPayBenefits Year \\\n",
+ "0 0.00 400184.25 NaN 567595.43 567595.43 2011 \n",
+ "1 245131.88 137811.38 NaN 538909.28 538909.28 2011 \n",
+ "2 106088.18 16452.60 NaN 335279.91 335279.91 2011 \n",
+ "3 56120.71 198306.90 NaN 332343.61 332343.61 2011 \n",
+ "4 9737.00 182234.59 NaN 326373.19 326373.19 2011 \n",
+ "... ... ... ... ... ... ... \n",
+ "148649 0.00 0.00 0.0 0.00 0.00 2014 \n",
+ "148650 NaN NaN NaN 0.00 0.00 2014 \n",
+ "148651 NaN NaN NaN 0.00 0.00 2014 \n",
+ "148652 NaN NaN NaN 0.00 0.00 2014 \n",
+ "148653 0.00 -618.13 0.0 -618.13 -618.13 2014 \n",
+ "\n",
+ " Agency \n",
+ "0 San Francisco \n",
+ "1 San Francisco \n",
+ "2 San Francisco \n",
+ "3 San Francisco \n",
+ "4 San Francisco \n",
+ "... ... \n",
+ "148649 San Francisco \n",
+ "148650 San Francisco \n",
+ "148651 San Francisco \n",
+ "148652 San Francisco \n",
+ "148653 San Francisco \n",
+ "\n",
+ "[148654 rows x 11 columns]"
+ ]
+ },
+ "execution_count": 296,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "df.dropna(how='all',axis=1)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "display number of unique values in each column"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 297,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "Id 148654\n",
+ "EmployeeName 110811\n",
+ "JobTitle 2159\n",
+ "BasePay 109489\n",
+ "OvertimePay 65998\n",
+ "OtherPay 83225\n",
+ "Benefits 98465\n",
+ "TotalPay 138486\n",
+ "TotalPayBenefits 142098\n",
+ "Year 4\n",
+ "Notes 0\n",
+ "Agency 1\n",
+ "Status 0\n",
+ "dtype: int64"
+ ]
+ },
+ "execution_count": 297,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "df.nunique()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "mean of total pay of all people based on year"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 305,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "74768.32197169267"
+ ]
+ },
+ "execution_count": 305,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "df['TotalPay'].mean()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "how many people have 0 overtime pay"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 306,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "77321"
+ ]
+ },
+ "execution_count": 306,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "df['OvertimePay'].value_counts()[0]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "max, min, mean, median and other stats of TotalPay of people having 0 OvertimePay"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 303,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "count 77321.000000\n",
+ "mean 60229.348901\n",
+ "std 49307.912350\n",
+ "min -618.130000\n",
+ "25% 13290.450000\n",
+ "50% 58158.590000\n",
+ "75% 91115.090000\n",
+ "max 567595.430000\n",
+ "Name: TotalPay, dtype: float64"
+ ]
+ },
+ "execution_count": 303,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "df1=df[(df['OvertimePay']==0)]\n",
+ "df1[\"TotalPay\"].describe()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "find Id of that person with max TotalPay you got in previous question"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 304,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "1"
+ ]
+ },
+ "execution_count": 304,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "df1[(df1['TotalPay']==df1['TotalPay'].max())].iloc[0,0]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "name of employee with total pay benefits = 87619.78"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 131,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "'REBECCA CHIU'"
+ ]
+ },
+ "execution_count": 131,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "df[(df['TotalPay']==87619.78)].iloc[0,1]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "how many people have BasePay > 150000 and OvertimePay > 100000"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 140,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "12"
+ ]
+ },
+ "execution_count": 140,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "df[(df['BasePay']>150000) & (df[\"OvertimePay\"]>100000)].shape[0]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "which job title generally has highest average TotalPayBenefits"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 225,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "Index(['Chief Investment Officer'], dtype='object', name='JobTitle')"
+ ]
+ },
+ "execution_count": 225,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "df3=df.groupby(\"JobTitle\").mean()\n",
+ "df3[(df3['TotalPayBenefits']==df3['TotalPayBenefits'].max())].index"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "How many employees are POLICE"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 274,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "The number of employees are with police 2512\n"
+ ]
+ }
+ ],
+ "source": [
+ "k=df[\"JobTitle\"].value_counts().index\n",
+ "n=df[\"JobTitle\"].value_counts()\n",
+ "count=0\n",
+ "for i in range(len(k)):\n",
+ " if \"POLICE\" in k[i]:\n",
+ " count+=n[i]\n",
+ "print(\"The number of employees are with police\",count)\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 285,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "2512"
+ ]
+ },
+ "execution_count": 285,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "b=df[\"JobTitle\"]\n",
+ "a=b.str.contains(pat=\"POLICE\")\n",
+ "\n",
+ "sum=0\n",
+ "for i in range(len(b)):\n",
+ " if a[i]:\n",
+ " sum=sum+1\n",
+ " else:\n",
+ " continue\n",
+ "sum"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.4rc1"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}