Algorithms From Scratch

How To Create an Algorithm Test Harness From Scratch With Python

Suresh Madhra·Feb 10, 2026·9 min read

You cannot know which algorithm is best

There is no algorithm that wins on every dataset. The only reliable way to choose is to try several under identical conditions and compare. A test harness is the code that guarantees those conditions are identical: same data, same splits, same metric, same baseline.

Build it once and every future experiment becomes a one-line call. Skip it and you will end up comparing an algorithm evaluated on a lucky split against one evaluated on an unlucky one.

Anatomy of a harness

A resampling strategy that produces train/test pairs.
An algorithm signature every model conforms to: algorithm(train, test, *args) -> predictions.
A metric function that turns actual and predicted into one number.
A baseline to beat, evaluated through the exact same path.
A fixed random seed so today's numbers reproduce tomorrow.

1. Train/test harness

The simplest harness: one split, one score. Fast and useful when the dataset is large.

train_test_harness.py

from random import seed, randrange

def train_test_split(dataset, split=0.60):
    train, copy = [], list(dataset)
    n_train = round(len(dataset) * split)
    while len(train) < n_train:
        train.append(copy.pop(randrange(len(copy))))
    return train, copy

def accuracy_metric(actual, predicted):
    correct = sum(1 for a, p in zip(actual, predicted) if a == p)
    return correct / float(len(actual)) * 100.0

def evaluate_algorithm(dataset, algorithm, split, *args):
    train, test = train_test_split(dataset, split)
    test_set = []
    for row in test:
        row_copy = list(row)
        row_copy[-1] = None          # hide the answer from the algorithm
        test_set.append(row_copy)
    predicted = algorithm(train, test_set, *args)
    actual = [row[-1] for row in test]
    return accuracy_metric(actual, predicted)

def zero_rule_classifier(train, test):
    outputs = [row[-1] for row in train]
    prediction = max(set(outputs), key=outputs.count)
    return [prediction for _ in test]

seed(1)
dataset = [[i, 0 if i % 3 else 1] for i in range(100)]
print("Baseline accuracy: %.3f%%" % evaluate_algorithm(dataset, zero_rule_classifier, 0.6))

from random import seed, randrange

def train_test_split(dataset, split=0.60):
    train, copy = [], list(dataset)
    n_train = round(len(dataset) * split)
    while len(train) < n_train:
        train.append(copy.pop(randrange(len(copy))))
    return train, copy

def accuracy_metric(actual, predicted):
    correct = sum(1 for a, p in zip(actual, predicted) if a == p)
    return correct / float(len(actual)) * 100.0

def evaluate_algorithm(dataset, algorithm, split, *args):
    train, test = train_test_split(dataset, split)
    test_set = []
    for row in test:
        row_copy = list(row)
        row_copy[-1] = None          # hide the answer from the algorithm
        test_set.append(row_copy)
    predicted = algorithm(train, test_set, *args)
    actual = [row[-1] for row in test]
    return accuracy_metric(actual, predicted)

def zero_rule_classifier(train, test):
    outputs = [row[-1] for row in train]
    prediction = max(set(outputs), key=outputs.count)
    return [prediction for _ in test]

seed(1)
dataset = [[i, 0 if i % 3 else 1] for i in range(100)]
print("Baseline accuracy: %.3f%%" % evaluate_algorithm(dataset, zero_rule_classifier, 0.6))

2. Cross-validation harness

For small or medium datasets, one split is too noisy. k-fold cross-validation trains k models and reports the mean and spread — the spread is often more informative than the mean.

cv_harness.py

from random import seed, randrange

def cross_validation_split(dataset, n_folds):
    folds, copy = [], list(dataset)
    fold_size = int(len(dataset) / n_folds)
    for _ in range(n_folds):
        fold = []
        while len(fold) < fold_size:
            fold.append(copy.pop(randrange(len(copy))))
        folds.append(fold)
    return folds

def accuracy_metric(actual, predicted):
    return sum(1 for a, p in zip(actual, predicted) if a == p) / float(len(actual)) * 100.0

def evaluate_algorithm(dataset, algorithm, n_folds, *args):
    folds = cross_validation_split(dataset, n_folds)
    scores = []
    for fold in folds:
        train = [row for f in folds if f is not fold for row in f]
        test = []
        for row in fold:
            row_copy = list(row)
            row_copy[-1] = None
            test.append(row_copy)
        predicted = algorithm(train, test, *args)
        actual = [row[-1] for row in fold]
        scores.append(accuracy_metric(actual, predicted))
    return scores

def zero_rule_classifier(train, test):
    outputs = [row[-1] for row in train]
    prediction = max(set(outputs), key=outputs.count)
    return [prediction for _ in test]

def random_classifier(train, test):
    values = list(set(row[-1] for row in train))
    return [values[randrange(len(values))] for _ in test]

seed(1)
dataset = [[i, 0 if i % 3 else 1] for i in range(100)]
for name, algo in [("Zero Rule", zero_rule_classifier), ("Random", random_classifier)]:
    scores = evaluate_algorithm(dataset, algo, 5)
    mean = sum(scores) / len(scores)
    print("%-10s scores=%s mean=%.3f%%" % (name, [round(s, 1) for s in scores], mean))

from random import seed, randrange

def cross_validation_split(dataset, n_folds):
    folds, copy = [], list(dataset)
    fold_size = int(len(dataset) / n_folds)
    for _ in range(n_folds):
        fold = []
        while len(fold) < fold_size:
            fold.append(copy.pop(randrange(len(copy))))
        folds.append(fold)
    return folds

def accuracy_metric(actual, predicted):
    return sum(1 for a, p in zip(actual, predicted) if a == p) / float(len(actual)) * 100.0

def evaluate_algorithm(dataset, algorithm, n_folds, *args):
    folds = cross_validation_split(dataset, n_folds)
    scores = []
    for fold in folds:
        train = [row for f in folds if f is not fold for row in f]
        test = []
        for row in fold:
            row_copy = list(row)
            row_copy[-1] = None
            test.append(row_copy)
        predicted = algorithm(train, test, *args)
        actual = [row[-1] for row in fold]
        scores.append(accuracy_metric(actual, predicted))
    return scores

def zero_rule_classifier(train, test):
    outputs = [row[-1] for row in train]
    prediction = max(set(outputs), key=outputs.count)
    return [prediction for _ in test]

def random_classifier(train, test):
    values = list(set(row[-1] for row in train))
    return [values[randrange(len(values))] for _ in test]

seed(1)
dataset = [[i, 0 if i % 3 else 1] for i in range(100)]
for name, algo in [("Zero Rule", zero_rule_classifier), ("Random", random_classifier)]:
    scores = evaluate_algorithm(dataset, algo, 5)
    mean = sum(scores) / len(scores)
    print("%-10s scores=%s mean=%.3f%%" % (name, [round(s, 1) for s in scores], mean))

3. Reading the results honestly

Report the mean and the standard deviation of the fold scores. If algorithm A scores 78.2% ± 4.1 and algorithm B scores 79.0% ± 3.8, you have not found a better algorithm — you have found noise. Prefer the simpler model until the gap is several times the spread.

Always run the baseline through the same harness — a number without a baseline means nothing.
Fix the seed while developing; vary it deliberately when you want to measure variance.
Hide the target column in the test set so a bug cannot leak the answer.
Keep the harness in one module and import it. Copy-pasted harnesses drift apart.

← Back to blog