Multi-label classification for the dice game Ten Thousand.

import numpy as np
import pandas as pd

from collections import Counter
from itertools import combinations_with_replacement as combos
from itertools import permutations as perms
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.metrics import f1_score, label_ranking_average_precision_score, average_precision_score
from sklearn.model_selection import train_test_split, GridSearchCV

scoring_rules = [[100, 200, 1000, 2000, 4000, 5000],
                 [0, 0, 200, 400, 800, 5000],
                 [0, 0, 300, 600, 1200, 5000],
                 [0, 0, 400, 800, 1600, 5000],
                 [50, 100, 500, 1000, 2000, 5000],
                 [0, 0, 600, 1200, 2400, 5000]
                 ]


def is_three_pair(roll):
    """Return true if roll contains three pairs.
    
    :var roll: list - A roll of 1 - 6 dice.
    """
    roll = sorted(roll)
    return (len(roll) == 6 and roll[0] == roll[1] and
            roll[2] == roll[3] and roll[4] == roll[5])


def is_straight(roll):
    """Return true if roll contains dice 1 - 6.

    :var roll: list - A roll of 1 - 6 dice.
    """
    return sorted(roll) == list(range(1, 7))


def score_all():
    """Return a list of floats == 1.0."""
    return [1.] * 6


def make_labels(roll):
    """Returns a label for each roll.
    
    :var roll: list - A roll of 1 - 6 dice.
    """
    counts = Counter(roll)
    if is_three_pair(roll) and (sum(scoring_rules[die - 1][count - 1] for die, count in counts.items()) < 1500):
        choice = score_all()
    elif is_straight(roll):
        choice = score_all()
    else:
        picks = set()
        for die, count in counts.items():
            if scoring_rules[die - 1][count - 1] > 0:
                picks.add(die)
        choice = [0.] * 6
        for i, x in enumerate(roll):
            if x in picks:
                choice[i] = 1.
    return np.array(choice)

def make_some_features(clip):
    """Creat a set of outcomes for rolling 6 dice.
    
    Find combinations of 6 dice, then permute them.
    
    :var clip: keep only combinations where its index % clip = 0
    """
    features = set()
    numbers = list(range(1, 7))
    combinations = (combo for combo in combos(numbers, 6))
    for i, comb in enumerate(combinations):
        if i % clip == 0:  # Keeping size reasonable
            for perm in perms(comb):
                features.add(perm)
    return features

features = make_some_features(2)

all_features = np.array([np.array(feature) for feature in features])

all_labels = np.array([make_labels(feature) for feature in all_features])

all_features.shape, all_labels.shape

def create_dataset(features, labels):
    """Create a DataFrame for dice throws and their labels.
    
    A column for each die in a roll and a label for each die.
    
    :var features: np.array of rolls
    :var labels: np.array of labels
    """
    # DataFrame for features.
    data = {str(i): features[:,i] for i in range(6)}
    dataset = pd.DataFrame(data)

    # DataFrame for labels.
    label = {'{}_l'.format(i): labels[:,i] for i in range(6)}
    label_df = pd.DataFrame(label)
    
    # Stick em together.
    df = pd.concat([dataset, label_df], axis=1, sort=False)
    return df

df = create_dataset(all_features, all_labels)

df.sample(10)

X = df[['0', '1', '2', '3', '4', '5']]
y = df[['0_l', '1_l', '2_l', '3_l', '4_l', '5_l']]

X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, shuffle=True)

X_train.shape, y_train.shape

X_test.shape, y_test.shape

extra = ExtraTreesClassifier(bootstrap=True,
                             oob_score=True,
                             n_jobs=-1,
                             n_estimators=2250)

params = {'min_samples_split': [4, 5, 6],
          'max_depth': [27, 30, 33]}
grid = GridSearchCV(extra,
                    param_grid=params,
                    scoring='average_precision',
                    n_jobs=-1,
                    cv=5,
                    verbose=1)
grid.fit(X_train, y_train)
grid.best_params_, grid.best_score_

params = {'n_estimators': [1250, 1500, 1750, 2000, 2250, 2500]}
grid = GridSearchCV(grid.best_estimator_,
                    param_grid=params,
                    scoring='average_precision',
                    n_jobs=-1,
                    cv=5,
                    verbose=1)
grid.fit(X_train, y_train)
grid.best_params_, grid.best_score_

best = grid.best_estimator_

y_pred = np.array([best.predict([test])[0] for test in X_test.values])

f1_score(y_test, y_pred, average='samples')

label_ranking_average_precision_score(y_test, y_pred)

average_precision_score(y_test, y_pred)

def test_model_pred(model, threshold=0.475, samples=25):
    """Get random sample of rolls from X_test and make predictions.
    
    Compare prediction precision with probability > 0.5 positive label with
    positive label at other thresholds by adjusting threshold.
    
    Print number of samples.
    
    :var threshold: float
    :var samples: int
    """
    for test in X_test.sample(samples).values:
        
        print(test)
        
        # Create ground truth label.
        true = make_labels(test).astype(int)
        print(true)
        
        # Raw probability predictions.
        pred_proba = np.array([round(y[0][1], 3) for y in model.predict_proba([list(test)])])
        print(pred_proba)
        
        # Predict 1 if probability > 0.5.
        pred = (pred_proba > 0.5).astype(int)
        print(pred)
        
        # Predict 1 if probability > threshold.
        pred_thresh = (pred_proba > threshold).astype(int)
        print(pred_thresh)
        
        result = 'Nailed it' if list(true) == list(pred) else 'Nuts'
        print(result)
        result = 'Nailed it' if list(true) == list(pred_thresh) else 'Nuts'
        print(result)
        print()

test_model_pred(best, threshold=.475, samples=40)

def test_threshold_precision(model, thresholds):
    """Test array of threshold values and calculate precision metrics for each.
    
    Calculate each threshold on a random sample of test data.
    Store and return in a dict.
    """
    results = dict()
    # This is going to take a while...
    size = len(X_test.values) / 10
    for threshold in thresholds:
        
        # Get sample of dice throws.
        throws = X_test.sample(size).values
        
        # Make predictions.
        y_pred = np.array([best.predict([dice])[0] for dice in throws])
        
        # Ground truth labels.
        true = np.array([make_labels(dice) for dice in throws])
        
        # Calculate metrics.
        f_one = f1_score(true, y_pred, average='samples')
        label_ranking = label_ranking_average_precision_score(true, y_pred)
        average_precision = average_precision_score(true, y_pred)
        
        # Save result.
        results[threshold] = {'f1_score': f_one,
                              'Label ranking average precision': label_ranking,
                              'Average precision': average_precision}
        
    return results

thresholds = np.linspace(.47, .5, 10)

threshold_test = test_threshold_precision(best, thresholds)
threshold_test

thresholds = np.linspace(.476, .486, 10)

threshold_test_1 = test_threshold_precision(best, thresholds)
threshold_test_1

thresholds = np.linspace(.482, .485, 5)

threshold_test_2 = test_threshold_precision(best, thresholds)
threshold_test_2

threshold = test_threshold_precision(best, [.48333333333333334])
threshold