This is a follow up question from my latest post: Put input in a tensorflow neural network

I precoded a neural network using tensorflow with the MNIST dataset, and with the help of @FinnE was able to change a bit of my code, the two files are listed below:

main.py:

import tensorflow as tf
import matplotlib.pyplot as plt
import numpy as npmnist=tf.keras.datasets.mnist(x_train, y_train), (x_test, y_test) = mnist.load_data()x_train=tf.keras.utils.normalize(x_train, axis=1)
x_test=tf.keras.utils.normalize(x_test, axis=1)model=tf.keras.models.Sequential()
model.add(tf.keras.layers.Flatten())
model.add(tf.keras.layers.Dense(128, activation=tf.nn.relu))
model.add(tf.keras.layers.Dense(128, activation=tf.nn.relu))
model.add(tf.keras.layers.Dense(10, activation=tf.nn.softmax))model.compile(optimizer='adam',loss='sparse_categorical_crossentropy',metrics=['accuracy'])
model.fit(x_train, y_train, epochs=3)val_loss, val_acc = model.evaluate(x_test, y_test)
print(val_loss, val_acc)model.save("num_reader.model")
new_model=tf.keras.models.load_model('num_reader.model')
predictions=new_model.predict([x_test])
print(predictions)

screen.py:

import tensorflow as tf
import pygame
import sys
import numpy as np
from main import *
import main as nnclass Screen:def __init__(self):pygame.init()#self.screen=pygame.display.set_mode((28,28),pygame.FULLSCREEN)self.screen=pygame.display.set_mode((280,280))self.array=[]self.setArr()self.bg_color=(250, 250,250)self.ok=Falseself.full=[]self.done=Falseprint(new_model)self.result=0def runGame(self):self.screen.fill(self.bg_color)while True:pygame.display.flip()self._check_events()self.draw()if self.full != []:breakdef _check_events(self):for event in pygame.event.get():if event.type==pygame.QUIT:sys.exit()if event.type==pygame.KEYDOWN:if event.key==pygame.K_ESCAPE:sys.exit()if event.key==pygame.K_d:self.done=Trueself.decode()print(len(self.full))if event.key==pygame.K_c:self.done=Falseself.setArr()self.screen.fill(self.bg_color)if event.type==pygame.MOUSEBUTTONDOWN:#print("mouseDown")self.ok=Trueelif event.type==pygame.MOUSEBUTTONUP:self.ok=Falsedef setArr(self):self.shortArr=[]for y in range(28):self.shortArr.append(0)for x in range(28):self.array.append(self.shortArr)def draw(self):if self.ok==True:x,y=pygame.mouse.get_pos()x=round(x/10)*10y=round(y/10)*10#print(x,y)#print(self.array[int(x)//10][int(y)//10])self.array[int(x)//10][int(y)//10]=1pygame.draw.rect(self.screen, (0,0,0), pygame.Rect(x, y, 10, 10))#print("draw")def decode(self):self.full=[]for x in range(28):for y in range(28):self.full.append(self.array[x][y])if __name__ == '__main__':Sc=Screen()Sc.runGame()result = nn.new_model.predict(tf.keras.utils.normalize(np.array(Sc.full), axis=1))print(result)

however I get the following error when the code is run.

Traceback (most recent call last):File "C:\Users\user\Documents\Jake\machine learning\MNIST dataset SOLVED\screen.py", line 81, in <module>result = nn.new_model.predict(tf.keras.utils.normalize(np.array(Sc.full), axis=1))File "C:\Users\user\AppData\Local\Programs\Python\Python310\lib\site-packages\keras\utils\np_utils.py", line 89, in normalizel2 = np.atleast_1d(np.linalg.norm(x, order, axis))File "<__array_function__ internals>", line 180, in normFile "C:\Users\user\AppData\Local\Programs\Python\Python310\lib\site-packages\numpy\linalg\linalg.py", line 2547, in normreturn sqrt(add.reduce(s, axis=axis, keepdims=keepdims))
numpy.AxisError: axis 1 is out of bounds for array of dimension 1

There is few problems.

1. you return flat array with 786 values but it needs 2D array 28x28 - and this makes problem with axis=1 in normalize - you have to use self.array instead of self.full

2. you create array 28x28 but predict needs 1x28x28 - so it needs [self.array] instead of `self.array

result = nn.new_model.predict(tf.keras.utils.normalize(np.array([Sc.array]), axis=1))    

3. it returns probability for all digits and you have to use np.argmax() to get digit with the biggest probability. Because predict() can get list with many images so it returns list with many results - even if you check single image - and it needs [0] to check result for first image.

digit = np.argmax(result[0])

4. I think there is problem with data in array. You keep it as x,y but numpy uses y,x (row, column) (and the same is with matrix in math). It needs self.array[y][x] = 1 instea self.array[x][y] = 1. EDIT: I added plt.imshow(), cv2.imshow() to confirm it. Using x,y it gives rotated image.

5. you import main and this runs all code in main and it trains model again at every start. You don't have to import main but only use load_model().

6. on my computer sometimes mouse.get_pos() gives value 280 and this gives array[...][28], and this raises error because array has only [27]. It needs to check x, y and convert 280 to 279 (or later convert 28 to 27)

7. you setArr() is wrong. You append the same list self.shortArr to self.array but (this is popular problem in Python) it doesn't put copy of array but it puts reference to the same array - and finally when you change one value in row then it change them in all rows.

EDIT:

8. mnist has images which use values 0...255 so maybe you should also use self.array[y][x] = 255 instead of self.array[y][x] = 1

My full working code:

screen.py

EDIT: I added right mouse to clear single pixel(s)

EDIT: I added plt.imshow(), cv2.imshow()

import pygame
import tensorflow as tf
import numpy as npclass Screen:def __init__(self):pygame.init()#self.screen = pygame.display.set_mode((28,28),pygame.FULLSCREEN)self.screen = pygame.display.set_mode((280,280))self.bg_color = (250, 250,250)self.array = []self.set_arr()self.ok = False    # exit with predictionself.done = False#self.button = Falsedef run(self):self.screen.fill(self.bg_color)while not self.done:pygame.display.flip()self._check_events()self.draw()pygame.quit()  # some systems need it to close windowreturn self.okdef _check_events(self):for event in pygame.event.get():if event.type == pygame.QUIT:# end program without OKself.done = Trueif event.type == pygame.KEYDOWN:if event.key == pygame.K_ESCAPE:# end program without OKself.done = Trueif event.key == pygame.K_d:# end program with OKself.ok = Trueself.done = Trueif event.key == pygame.K_c:# clear array and screenself.set_arr()self.screen.fill(self.bg_color)#if event.type == pygame.MOUSEBUTTONDOWN:#    #print("mouseDown")#    self.button = True#elif event.type == pygame.MOUSEBUTTONUP:#    self.button = Falsedef set_arr(self):"""Create empty array 2D for image."""self.array = [[0 for x in range(28)] for y in range(28)]def draw(self):buttons = pygame.mouse.get_pressed()if buttons[0] or buttons[2]:  # left or right button#if self.button:x0, y0 = pygame.mouse.get_pos()if x0 >= 280:x0 = 279if y0 >= 280:y0 = 279x = int(round(x0/10)*10)y = int(round(y0/10)*10)try:#self.array[int(x)//10][int(y)//10] = 255if buttons[0]:  # left drawself.array[int(y)//10][int(x)//10] = 255pygame.draw.rect(self.screen, (0,0,0), pygame.Rect(x, y, 10, 10))if buttons[2]:  # right clearself.array[int(y)//10][int(x)//10] = 0                pygame.draw.rect(self.screen, (255,255,255), pygame.Rect(x, y, 10, 10))except Exception as ex:print('Exception:', ex)print('Debug:', x0, y0, x, y, int(x)//10, int(y)//10)#print("draw")if __name__ == '__main__':print('loading model ...')model = tf.keras.models.load_model('num_reader.model')print('starting screen')screen = Screen()ok = screen.run()if not ok:print('finish without prediction')else:image = np.array(screen.array, dtype='uint8')print('image.shape:', image.shape)print('--- show image ---')import matplotlib.pyplot as pltplt.imshow(image, cmap=plt.get_cmap('gray'))        plt.show()#import cv2#cv2.imshow('image', image)        #cv2.waitKey(0)print('--- predict as is ---')arr = np.array([image])print('arr.shape:', arr.shape)#print(arr[:,:2])results = model.predict(arr)print('results:', results)digit = np.argmax(results[0])print('digit:', digit)print('--- predict normalized ---')arr = tf.keras.utils.normalize(np.array([image]), axis=1)print('arr.shape:', arr.shape)#print(arr[:,:2])results = model.predict(arr)print('results:', results)digit = np.argmax(results[0])print('digit:', digit)

train_model.py

import tensorflow as tfmnist = tf.keras.datasets.mnist(x_train, y_train), (x_test, y_test) = mnist.load_data()x_train = tf.keras.utils.normalize(x_train, axis=1)
x_test = tf.keras.utils.normalize(x_test, axis=1)model = tf.keras.models.Sequential()
model.add(tf.keras.layers.Flatten())
model.add(tf.keras.layers.Dense(128, activation=tf.nn.relu))
model.add(tf.keras.layers.Dense(128, activation=tf.nn.relu))
model.add(tf.keras.layers.Dense(10, activation=tf.nn.softmax))model.compile(optimizer='adam',loss='sparse_categorical_crossentropy',metrics=['accuracy'])model.fit(x_train, y_train, epochs=3)val_loss, val_acc = model.evaluate(x_test, y_test)
print('val_loss:', val_loss)
print('val_acc :', val_acc)#predictions = model.predict([x_test])
#print(predictions)model.save("num_reader.model")