Network Programming in .NET

ROOT_DIR = 'C:\\Users\\Andrew\\Documents\\Work\\string-prediction\\Final'  # The home location of this project

​

SEED = 42  # The seed to ensure consistency across experiments, not used in production

INPUT_SIZE = 24  # A helper variable to define the size of the input

OUTPUT_SIZE = 17  # A helper variable to define the size of the output

import pandas as pd  # pandas is a commonly used data processing library, used here to load the csv initially

​

DATA_FILE = '{}/{}'.format(ROOT_DIR, 'RealWorkload.csv')  # the filepath of the csv file

DATA_PICKLE = '{}/{}'.format(ROOT_DIR, 'data.pickle')  # the cached version of the csv file for faster loading in development

​

data = pd.read_csv(DATA_FILE, header=None, names=['input', 'output'])  # use pandas to read the input

data = data.dropna()  # drop any rows with null values just in case

data.to_pickle(DATA_PICKLE)  # cache csv file in a "pickle" format for faster loading in development

# data = pd.read_pickle(DATA_PICKLE)  # loading csv file in pickle format, not used here

data.head()  # display the first five rows in the console

import numpy as np  # numpy is a utility mathematical library

x = np.zeros((len(data), INPUT_SIZE))  # construct an empty container that will track the input values

for i, row in enumerate(data['input'].values):  # iterate through the input values

    arr = row.split('|')  # split the string by the '|' delimiter

    for j, value in enumerate(arr):  # iterate through each value in a row

        x[i, j] = int(value)  # set the appropriate value in the container to the value in the input as an integer

y = data['output'].values  # simply copy over the output values into the y container

​

# note that even though we are constructing the x and y containers separately, it is guaranteed that their values will sync up.

# Such that the first x element will correspond to the first y element

import string

from sklearn.model_selection import train_test_split  # sklearn is a commonly used data science library

​

SIZE = 100000  # we define our working-set size

x_work, y_work, x_test, y_test = x[:SIZE], y[:SIZE], x[SIZE:], y[SIZE:]  # we split the data into a working set and test set

x_train, x_val, y_train, y_val = train_test_split(x_work, y_work, test_size=0.2, random_state=SEED)

# we then further split the working set into a training and validation using the SEED variable to 

# control randomness during development, the validation set is 20% of the working set

from sklearn.ensemble import RandomForestClassifier  

# a random forest classifier is an advanced implementation of decision trees and 

# has been shown to be the best statistical learning method

​

models = []

for i in range(OUTPUT_SIZE):

    target = list(map(lambda v: v[i], y_train))  # we fetch the appropriate target digits

    model = RandomForestClassifier(random_state=SEED, n_estimators=10)  # we use 100 estimators in our random forest setup

    model.fit(x_train, target)

    models += [model]

    print('finished with model #{}'.format(i))

from sklearn.metrics import accuracy_score  # a utility helper function that evaluates models

import matplotlib.pyplot as plt  # import plotting library

​

def measure(x, y):

    output = []

    for i in range(OUTPUT_SIZE):  # we iterate through each character in the output string

        target = list(map(lambda x: x[i], y))  # we fetch the appropriate expected target

        predicted = models[i].predict(x)  # we use our trained model to predict the the character

        accuracy = accuracy_score(target, predicted)  # we check the accuracy of the predictions against the expected

        output += [accuracy * 100]

    return output

​

def graph_results(results, title):

    plt.bar(np.arange(OUTPUT_SIZE), results, align='center', alpha=0.5)

    plt.xticks(np.arange(OUTPUT_SIZE), np.arange(1, OUTPUT_SIZE + 1))

    plt.ylabel('accuracy %')

    plt.xlabel('character in output')

    plt.title(title)

    plt.show()

train_results = measure(x_train, y_train)

val_results = measure(x_val, y_val)

test_results = measure(x_test, y_test)

graph_results(train_results, 'train accuracy')

graph_results(val_results, 'validation accuracy')

graph_results(test_results, 'test accuracy')

models = []

for i in range(OUTPUT_SIZE):

    target = list(map(lambda v: v[i], y))  # we fetch the appropriate target digits

    model = RandomForestClassifier(random_state=SEED, n_estimators=10)  # we use 100 estimators in our random forest setup

    model.fit(x, target)

    models += [model]

    print('finished with model #{}'.format(i))

import pickle

​

for i in range(OUTPUT_SIZE):

    pickle.dump(models[i], open('{}/model_{}.pickle'.format(ROOT_DIR, i), 'wb'))  # serializing models to pickle format

    print('finished {}'.format(i))