If you really must use the label color visual indication, then you could try something like:
PublicClass Form1
Dim digits AsChar() = {"0", "1", "2", "3", "4", "5", "6", "7", "8", "9"}
PrivateSub Button1_Click(ByVal sender As System.Object, ByVal e As System.EventArgs) Handles...