Machine Learning Anomaly Detection

this use case is a customized timeseries version of making a cnn anomaly detection model from the tensorflow website import numpy as np import tensorflow as tf from tensorflow import keras import pandas as pd import seaborn as sns from pylab import rcparams import matplotlib pyplot as plt from matplotlib import rc from pandas plotting import register matplotlib converters ​ register matplotlib converters() sns set(style='whitegrid', palette='muted') \# rcparams\['figure figsize'] = 22, 10 csv path = '20161003 085624 csv' df = pd read csv(csv path) ​ print ("\ncolumns ", df columns, "data frame length ", len(df), " rows\n") ​ features considered = \['temperature (c)'] ​ features = df\[features considered] features index = np arange(start=0, stop=len(df), step = 1) \# print(features index) print(features head())​ x = features values df = features print(x) \# plt plot(df) \# plt show() train size = int(len(df) 0 90) test size = len(df) train size train, test = df\[0\ train size], df\[train size\ len(df)] print(train, test) print(train shape, test shape) ​ from sklearn preprocessing import standardscaler ​ scaler = standardscaler() scaler = scaler fit(train\[\['temperature (c)']]) ​ train\['close'] = scaler transform(train\[\['temperature (c)']]) test\['close'] = scaler transform(test\[\['temperature (c)']]) \# print(train, test) def create dataset(x, y, time steps=1) xs, ys = \[], \[] for i in range(len(x) time steps) v = x iloc\[i (i + time steps)] values xs append(v)ys append(y iloc\[i + time steps]) return np array(xs), np array(ys) ​ time steps = 30 \# reshape to \[samples, time steps, n features] x train, y train = create dataset(train\[\['temperature (c)']], train close, time steps) ​ x test, y test = create dataset(test\[\['temperature (c)']], test close, time steps) print(x train shape) print(y test shape) print(x train shape\[1], x train shape\[2], y train\[1]) \# model = keras sequential() \# model add(keras layers lstm( \# units=64, \# input shape=(x train shape\[1], x train shape\[2]) \# )) \# model add(keras layers dropout(rate=0 2)) \# model add(keras layers repeatvector(n=x train shape\[1])) \# model add(keras layers lstm(units=64, return sequences=true)) \# model add(keras layers dropout(rate=0 2)) \# model add(keras layers timedistributed(keras layers dense(units=x train shape\[2]))) \# model compile(loss='mae', optimizer='adam') verbose, epochs, batch size = 0, 2, 128 n timesteps, n features, n outputs = x train shape\[1], x train shape\[2], y train\[1] ​ model = keras sequential() model add(tf keras layers conv1d(filters=64, kernel size=3, activation='relu', input shape=(n timesteps,n features))) model add(tf keras layers conv1d(filters=64, kernel size=3, activation='relu')) model add(tf keras layers dropout(0 5)) model add(tf keras layers maxpooling1d(pool size=2)) model add(tf keras layers flatten()) model add(tf keras layers dense(100, activation='relu')) model add(tf keras layers dense(n outputs, activation='softmax')) model compile(loss='mae', optimizer='adam', metrics=\['accuracy']) \# fit network history = model fit( x train, y train, epochs=2, batch size=256, validation split=0 1, shuffle=false ) ​ model summary() x train pred = model predict(x train)​ \# model save("/anomalymodel/") ​ train mae loss = np mean(np abs(x train pred x train), axis=1) plt figure() sns distplot(train mae loss, bins=50, kde=true); ​ ​ x test pred = model predict(x test) ​ test mae loss = np mean(np abs(x test pred x test), axis=1) print(test mae loss) plt figure() sns distplot(train mae loss, bins=50, kde=true) plt figure() ​ threshold = 0 65 ​ test score df = pd dataframe(index=test\[time steps ] index) test score df\['loss'] = test mae loss test score df\['threshold'] = threshold test score df\['anomaly'] = test score df loss > test score df threshold test score df\['close'] = test\[time steps ] close ​ plt plot(test score df index, test score df loss, label='loss') plt plot(test score df index, test score df threshold, label='threshold') plt xticks(rotation=25) ​ anomalies = test score df\[test score df anomaly == true] print(anomalies head()) ​ plt plot( test\[time steps ] index, scaler inverse transform(test\[time steps ] close), label='temp' ); ​ sns scatterplot( anomalies index, scaler inverse transform(anomalies close), color=sns color palette()\[3], s=52, label='anomaly' ) plt xticks(rotation=25) ​ plt show()