(修复bug)开源线下得分测试代码

Discussion

Home Discussion Board KDD CUP of Fresh Air (修复bug)开源线下得分测试代码

智浪淘沙

(修复bug)开源线下得分测试代码

posted in KDD CUP of Fresh Air

2018-05-08 03:27

```python
# -*- coding: utf-8 -*-
"""
Created on Tue May  8 10:46:55 2018

@author: ZHILANGTAOSHA
"""

import numpy as np
import pandas as pd
from requests.exceptions import RequestException
import requests

stat_latlon = pd.read_csv('stat1.csv',header = None)
stat_latlon = stat_latlon[[0,1,2,3]]
stat_latlon.columns = ['dq','stationId','lat','lon']

headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36'
}

cw = {'wanshouxig_aq':'wanshouxigong_aq', 'aotizhongx_aq':'aotizhongxin_aq', 'nongzhangu_aq':'nongzhanguan_aq', 'fengtaihua_aq':'fengtaihuayuan_aq',
       'miyunshuik_aq':'miyunshuiku_aq', 'yongdingme_aq':'yongdingmennei_aq', 'xizhimenbe_aq':'xizhimenbei_aq'}
gz = {value:key for key,value in cw.items()}

def colchuli(x):
    if x == 'time':return 'utc_time'
    if x == 'station_id':return 'stationId'
    x = str(x).replace('_Concentration','').replace('.','').upper()
    if x == 'PM25':
        return 'PM2.5'
    else:
        return x

def chulifloat(x):
    try:
        return float(x)
    except:
        return np.NaN

def get_xianshang_true(times,city):
    time_start = (pd.to_datetime(times) - pd.DateOffset(days = 1)).strftime('%Y-%m-%d-%H')
    time_end = (pd.to_datetime(times) + pd.DateOffset(days = 2)).strftime('%Y-%m-%d-%H')
    url = 'https://biendata.com/competition/airquality/%s/%s/%s/2k0d1d8'%(city,time_start,time_end)
    print(url)
    zt = 0
    while zt == 0:
        try:
            respones= requests.get(url,headers=headers)
            zt = 1
            print(respones.status_code)
        except RequestException:
            print('Error')
            zt = 0
    df = pd.DataFrame([i.decode('utf8').split(',') for i in respones.content.splitlines()])
    df.columns = df.loc[0]
    df = df.shift(-1,axis = 0).dropna()
    df.columns = df.columns.map(colchuli)
    df.utc_time = pd.to_datetime(df.utc_time)
    floatcl = [i for i in df.columns if i not in ['stationId', 'utc_time']]
    df.loc[:,floatcl] = df.loc[:,floatcl].applymap(chulifloat)
    return df

def smape(actual, predicted):
    a = np.abs(np.array(actual) - np.array(predicted))
    b = np.array(actual) + np.array(predicted)
    return 2 * np.mean(np.divide(a, b, out=np.zeros_like(a), where=b!=0, casting='unsafe'))

def get_true(times,smaplev = 'old'):
 xlj = get_xianshang_true(times,'bj')
 bjtrue = xlj[(xlj.utc_time >= times)& (xlj.utc_time < pd.to_datetime(times) + pd.DateOffset(days = 2))]
 xlj = get_xianshang_true(times,'ld')
 ldtrue = xlj[(xlj.utc_time >= times)& (xlj.utc_time < pd.to_datetime(times) + pd.DateOffset(days = 2))]
 ldtrue['O3'] = 0
 true = pd.concat([bjtrue[['stationId', 'utc_time', 'PM2.5', 'PM10','O3']]
 ,ldtrue[['stationId', 'utc_time', 'PM2.5', 'PM10','O3']]])
 true['hour'] = true.utc_time.dt.hour
 true['hour'] = true.apply(lambda x: x['hour'] + 24 if x['utc_time'] >= pd.to_datetime(times) + pd.DateOffset(days = 1) else x['hour'],axis=1)
 if smaplev == 'old':
 true['test_id'] = true.stationId.map(lambda x:gz[x] if x in gz.keys() else x )+'#'+(true.hour).astype(str)
 else:
 true['test_id'] = true.stationId +'#'+(true.hour).astype(str)
 true = true[true.stationId.isin(stat_latlon.stationId)]
 return true

def smape_pf_sub(true,sub2):
    smapedf = []
    truez = []
    predz = []    
    for label in ['PM2.5','PM10','O3']:
        ck = pd.merge(true[['test_id',label]],sub2[['test_id',label]],on = 'test_id',how = 'left')
        ck = ck[ck['%s_x'%label].notnull()]
        ck = ck[(ck['%s_x'%label] != 0) & (ck['%s_y'%label] != 0)]
        smapedf.append(smape(ck['%s_x'%label],ck['%s_y'%label]))
        predz.extend(ck['%s_x'%label].tolist())
        truez.extend(ck['%s_y'%label].tolist())
        print('%s:'%label,smape(ck['%s_x'%label],ck['%s_y'%label]))
    print(smape(np.array(truez),np.array(predz)))
    return smapedf

true = get_true('2018-05-05 00:00:00')
true = true.dropna()
sub2 = pd.read_csv('baseline_201805050734_0.295193965064.csv')

true1 = true[true['O3'] == 0]
true2 = true[true['O3'] != 0]

jg2 = smape_pf_sub(true1,sub2)
jg1 = smape_pf_sub(true2,sub2)

print((np.mean(jg1) + np.mean(jg2[0:2])) / 2)

```

stat1.csv

bj	aotizhongxin_aq	116.397	39.982
bj	badaling_aq	115.988	40.365
bj	beibuxinqu_aq	116.174	40.09
bj	daxing_aq	116.404	39.718
bj	dingling_aq	116.22	40.292
bj	donggaocun_aq	117.12	40.1
bj	dongsi_aq	116.417	39.929
bj	dongsihuan_aq	116.483	39.939
bj	fangshan_aq	116.136	39.742
bj	fengtaihuayuan_aq	116.279	39.863
bj	guanyuan_aq	116.339	39.929
bj	gucheng_aq	116.184	39.914
bj	huairou_aq	116.628	40.328
bj	liulihe_aq	116	39.58
bj	mentougou_aq	116.106	39.937
bj	miyun_aq	116.832	40.37
bj	miyunshuiku_aq	116.911	40.499
bj	nansanhuan_aq	116.368	39.856
bj	nongzhanguan_aq	116.461	39.937
bj	pingchang_aq	116.23	40.217
bj	pinggu_aq	117.1	40.143
bj	qianmen_aq	116.395	39.899
bj	shunyi_aq	116.655	40.127
bj	tiantan_aq	116.407	39.886
bj	tongzhou_aq	116.663	39.886
bj	wanliu_aq	116.287	39.987
bj	wanshouxigong_aq	116.352	39.878
bj	xizhimenbei_aq	116.349	39.954
bj	yanqin_aq	115.972	40.453
bj	yizhuang_aq	116.506	39.795
bj	yongdingmennei_aq	116.394	39.876
bj	yongledian_aq	116.783	39.712
bj	yufa_aq	116.3	39.52
bj	yungang_aq	116.146	39.824
bj	zhiwuyuan_aq	116.207	40.002
ld	BL0	51.522287	-0.125848
ld	CD1	51.544219	-0.175284
ld	CD9	51.52770662	-0.129053205
ld	GN0	51.490532	0.074003
ld	GN3	51.486957	0.095111
ld	GR4	51.45258	0.070766
ld	GR9	51.456357	0.040725
ld	HV1	51.52078746	0.205460706
ld	KF1	51.52104675	-0.21349214
ld	LW2	51.474954	-0.039641
ld	MY7	51.52254	-0.15459
ld	ST5	51.3892869	-0.141661525
ld	TH4	51.51504617	-0.008418493

因为sample有两个版本，在get_true中请自行判断