assert np.inf == Node(1, 0.9, [1,2,3]).split_score
test_x = np.array(
[[23.2, 44.4], #0
[ 2. , 2. ], #1
[34.3, 77.3], #2
[-1.5, -0.5], #3
[ 1.5, 1.5], #4
[ 1.5, 9.2], #5
[ 2. , -2. ]])#6
test_y = np.array([0.0, 1.1, 2.2, 3.3, 4.4, 5.5, 6.6])
test_data = DataWrapper.from_pandas(pd.DataFrame(test_x), pd.Series(test_y))
test_node = Node(0, 0, np.arange(7))
best_split_for_col(test_data, test_node, 0, 3)
le_split, gt_split = test_node.split_idxs
assert np.array_equal(test_y[le_split], [3.3, 4.4, 5.5])
assert np.array_equal(test_y[gt_split], [1.1, 6.6, 0. , 2.2])
test_node = Node(0, 0, np.arange(7))
best_split_for_col(test_data, test_node, 1)
le_split, gt_split = test_node.split_idxs
assert np.array_equal(test_y[le_split], [6.6])
assert np.array_equal(test_y[gt_split], [3.3, 4.4, 1.1, 5.5, 0. , 2.2])
# not enough data to split with at least 4 values in each leaf
test_node = Node(0, 0, np.arange(7))
best_split_for_col(test_data, test_node, 1, 4)
assert test_node.split_score == np.inf
test_node = Node(0, 0, np.arange(7)[4:])
test_split = best_split_for_col(test_data, test_node, 0, 1)
le_split, gt_split = test_node.split_idxs
assert np.array_equal(test_y[le_split], [4.4, 5.5])
assert np.array_equal(test_y[gt_split], [6.6])
assert test_node.split_values == (1.5, 2.0)
test_node = Node(0, 0, np.arange(7))
best_split(test_data, test_node, [0,1]); test_split
assert test_node.split_col_idx == 1
assert test_node.split_preds == (6.6, 2.75)
print(test_y)
print_tree(DecisionTree(test_data, min_leaf_samples=2).fit())
print_tree(DecisionTree(test_data, min_leaf_samples=1).fit())
TODO: xxxx clean-up
Might we be able to generalize better by ;
- adding a little randomness by using a split value that lies somewhere between the lower and upper boundary of the split. See np.random.uniform ...
- use the average of the lower and upper boundary values
both of these could be done at prediction time
When we make a prediction for a row that was used in training and we have only 1 sample in each leaf, the tree should predict exactly the right answer
- grab a single row of data
- make a prediction for this row
- assert that the prediction we made matches the actual for this row
test_tree = DecisionTree(test_data, min_leaf_samples=1).fit()
for i in range(test_data.x_rows):
test_sample = test_data.get_sample(i)
assert predict_row(test_sample[0], test_tree.node) == test_sample[1]
Set-up some data for testing. This data is copied from the final model used in https://github.com/fastai/fastai/tree/master/courses/ml1/lesson2-rf_interpretation.ipynb
bulldozers_data = np.load('test/data/bulldozers.npy', allow_pickle=True)
train_data = DataWrapper(*bulldozers_data[:4])
valid_data = DataWrapper(*bulldozers_data[4:])
train_data, valid_data
Use a very small amount of data to train a decision tree, then print the root node so we can see how the data has been split.
It's interesting that the depth of this tree is greater than the expected np.log2(test_tree.data.x_rows) - because it's unbalanced.
test_tree = DecisionTree(train_data.tail(10), min_leaf_samples=1).fit()
print_tree(test_tree)
Get predictions for all of the data we trained on.
Although we set min_leaf_samples=1, not every sample has it's own leaf. If 2 or more samples have;
- the same values for all independent variables and
- different values for the dependent variable,
- they will end up in the same leaf (because we can't find a value to split on) that will predict the mean of the dependent variables for all samples in the leaf
So we expect preds to be nearly 100% correct;
- loss to be nearly zero
- predictions vs actual plots a strait line with just a little variation
test_tree = DecisionTree(train_data.tail(2000), min_leaf_samples=1).fit()
test_preds = test_tree.predict(test_tree.data.x)
loss = rmse(test_preds, test_tree.data.y); print('loss', loss)
import matplotlib.pyplot as plt
plt.scatter(test_preds, test_tree.data.y, alpha=.1);
Get predictions for all of the data we trained on again - but allow a minimum of 5 items in each leaf. So we see;
- a non-zero loss
- some variance in the predictions vs actual plot
test_tree = DecisionTree(train_data.tail(2000), min_leaf_samples=5).fit()
test_preds = test_tree.predict(test_tree.data.x)
loss = rmse(test_preds, test_tree.data.y); print('loss', loss)
plt.scatter(test_preds, test_tree.data.y, alpha=.1);
Get predictions for the validation data - we don't expect a single tree to be very good
test_preds = test_tree.predict(valid_data.x)
loss = rmse(test_preds, valid_data.y); print('loss', loss)
plt.scatter(test_preds, valid_data.y, alpha=.1);
Create a tree ensemble and check that it has initialized correctly
test_ensemble = TreeEnsemble(train_data.tail(2000), sample_size=750, min_leaf_samples=5)
assert test_ensemble.sample_size == 750 == test_ensemble.trees[0].data.x_rows == len(test_ensemble.trees[0].data.y)
assert len(test_ensemble.col_idxs_fn(test_ensemble.data.all_x_col_idxs)) == 9
assert len(test_ensemble.trees) == 10
Fit the ensemble and get predictions for all of the data we trained on - TODO: I'd expect this to be better than a single tree but the loss is the same.
test_ensemble.fit()
test_preds = test_ensemble.predict(test_ensemble.data.x)
loss = rmse(test_preds, test_ensemble.data.y); print('loss', loss)
plt.scatter(test_preds, test_ensemble.data.y, alpha=.1);
Get predictions for the validation data - expect this to be better than a single tree.
test_preds = test_ensemble.predict(valid_data.x)
loss = rmse(test_preds, valid_data.y); print('loss', loss)
plt.scatter(test_preds, valid_data.y, alpha=.1);
TODO
- add classification capability
- confidence based on tree pred variance
- feature importance
- jumble single column -> create preds - which column makes preds the worst when jumbled
- WHAT are you forgetting?
- is it which split/feature contributes the biggest change from "bias" to "pred"
- avg depth of feature in tree <- i just made this up
- show dendogram of rank correlation
- partial dependence
- ggplot if monotonic relationship
- do the "what if" preds - i.e. change year of sale to 1960 and see what things would have sold for
- pdp plot
- tree interpret
- for single row pred: print contribution of each split (feature) to final result
- waterfall chart