2019-01-05

Python 数据实战：北京市 PM 2.5 数据的统计分析

import csv
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn
%matplotlib inline

files = ['BeijingPM20100101_20151231.csv',
       'ChengduPM20100101_20151231.csv',
       'GuangzhouPM20100101_20151231.csv',
       'ShanghaiPM20100101_20151231.csv',
       'ShenyangPM20100101_20151231.csv']

out_columns = ['No', 'year', 'month', 'day', 'hour', 'season', 'PM_US Post']

# create a void dataframe
df_all_cities = pd.DataFrame()

# iterate to write diffrent files
for inx, val in enumerate(files):
    df = pd.read_csv(val)
    df = df[out_columns]
    # create a city column
    df['city'] = val.split('P')[0]
    # append each file and merge all files into one
    df_all_cities = df_all_cities.append(df)

# replace the space in variable names with '_'
df_all_cities.columns = [c.replace(' ', '_') for c in df_all_cities.columns]

df_all_cities.head()

def filter_data(data, condition):
    """
    Remove elements that do not match the condition provided.
    Takes a data list as input and returns a filtered list.
    Conditions should be a list of strings of the following format:
      '<field> <op> <value>'
    where the following operations are valid: >, <, >=, <=, ==, !=
    
    Example: ["duration < 15", "start_city == 'San Francisco'"]
    """

    # Only want to split on first two spaces separating field from operator and
    # operator from value: spaces within value should be retained.
    field, op, value = condition.split(" ", 2)
    
    # check if field is valid
    if field not in data.columns.values :
        raise Exception("'{}' is not a feature of the dataframe. Did you spell something wrong?".format(field))

    # convert value into number or strip excess quotes if string
    try:
        value = float(value)
    except:
        value = value.strip("\'\"")

    # get booleans for filtering
    if op == ">":
        matches = data[field] > value
    elif op == "<":
        matches = data[field] < value
    elif op == ">=":
        matches = data[field] >= value
    elif op == "<=":
        matches = data[field] <= value
    elif op == "==":
        matches = data[field] == value
    elif op == "!=":
        matches = data[field] != value
    else: # catch invalid operation codes
        raise Exception("Invalid comparison operator. Only >, <, >=, <=, ==, != allowed.")
    
    # filter data and outcomes
    data = data[matches].reset_index(drop = True)
    return data

data1 = filter_data(df_all_cities, "year == 2015")
data1.head()

def reading_stats(data, filters = [], verbose = True):
    """
    Report number of readings and average PM2.5 readings for data points that meet
    specified filtering criteria.
    """

    n_data_all = data.shape[0]

    # Apply filters to data
    for condition in filters:
        data = filter_data(data, condition)

    # Compute number of data points that met the filter criteria.
    n_data = data.shape[0]

    # Compute statistics for PM 2.5 readings.
    pm_mean = data['PM_US_Post'].mean()
    pm_qtiles = data['PM_US_Post'].quantile([.25, .5, .75]).as_matrix()
    
    # Report computed statistics if verbosity is set to True (default).
    if verbose:
        if filters:
            print('There are {:d} readings ({:.2f}%) matching the filter criteria.'.format(n_data, 100. * n_data / n_data_all))
        else:
            print('There are {:d} reading in the dataset.'.format(n_data))

        print('The average readings of PM 2.5 is {:.2f} ug/m^3.'.format(pm_mean))
        print('The median readings of PM 2.5 is {:.2f} ug/m^3.'.format(pm_qtiles[1]))
        print('25% of readings of PM 2.5 are smaller than {:.2f} ug/m^3.'.format(pm_qtiles[0]))
        print('25% of readings of PM 2.5 are larger than {:.2f} ug/m^3.'.format(pm_qtiles[2]))
        seaborn.boxplot(data['PM_US_Post'], showfliers=False)
        plt.title('Boxplot of PM 2.5 of filtered data')
        plt.xlabel('PM_US Post (ug/m^3)')

    # Return three-number summary
    return data

def univariate_plot(data, key = '', bar = False):
    """
    Plot average PM 2.5 readings, given a feature of interest
    """
    
    # Check if the key exists
    if not key:
        raise Exception("No key has been provided. Make sure you provide a variable on which to plot the data.")
    if key not in data.columns.values :
        raise Exception("'{}' is not a feature of the dataframe. Did you spell something wrong?".format(key))

    # Create plot
    if bar:
        plt.figure(figsize=(8,6))
        data.groupby(key)['PM_US_Post'].mean().plot(kind = 'bar', color = 'grey')
        #plt.plot(data.groupby(key)[key], data.groupby(key)['PM_US_Post'].mean(), color = color)
        plt.ylabel('PM 2.5 (ug/m^3)')
        plt.title('Average PM 2.5 Reading by {:s}'.format(key), fontsize =14)
        plt.show()
    return None

df_test = reading_stats(df_all_cities, ["city == 'Beijing'", "year == 2015"])

univariate_plot(data1, 'city', True)

data2 = filter_data(df_all_cities, "city == Beijing")
data2.head()

univariate_plot(data2, 'hour', True)
univariate_plot(data2, 'day', True)
univariate_plot(data2, 'month', True)
univariate_plot(data2, 'season', True)
univariate_plot(data2, 'year', True)

plt.figure(figsize=(12,9))
data3 = filter_data(data2, "year == 2010")
data2010 = data3.groupby('season')['PM_US_Post'].mean()
data2010.plot(kind = 'line', color = 'red', marker = 'o', label = 'year = 2010')

data3 = filter_data(data2, "year == 2011")
data2011 = data3.groupby('season')['PM_US_Post'].mean()
data2011.plot(kind = 'line', color = 'orange', marker = 'o', label = 'year = 2011')

data3 = filter_data(data2, "year == 2012")
data2012 = data3.groupby('season')['PM_US_Post'].mean()
data2012.plot(kind = 'line', color = 'yellow', marker = 'o', label = 'year = 2012')

data3 = filter_data(data2, "year == 2013")
data2013 = data3.groupby('season')['PM_US_Post'].mean()
data2013.plot(kind = 'line', color = 'green', marker = 'o', label = 'year = 2013')

data3 = filter_data(data2, "year == 2014")
data2014 = data3.groupby('season')['PM_US_Post'].mean()
data2014.plot(kind = 'line', color = 'blue', marker = 'o', label = 'year = 2014')

data3 = filter_data(data2, "year == 2015")
data2015 = data3.groupby('season')['PM_US_Post'].mean()
data2015.plot(kind = 'line', color = 'purple', marker = 'o', label = 'year = 2015')

plt.ylabel('PM 2.5 (ug/m^3)')
plt.title('Average PM 2.5 Reading by Season', fontsize =14)
new_ticks = np.linspace(1, 4, 4)
plt.xticks(new_ticks)
plt.legend()

pd.concat([data2012, data2013, data2014, data2015])