Arabic Numbers Handwriting Recognition

In this post we will see simple approach to recognize Arabic Numbers Handwriting..
This code is part of interactive calculator where we use speech and handwriting as input for the calculator.



Full code is hosted in SourceForgue in the following location:

http://sourceforge.net/projects/interactive-cal/

This open source calculator uses Hand writing (Arabic/English) , Buttons , and Voice recognition (Arabic/English).

It uses HTK for recognition (open source and license included within).


1) 1st add the jPanel2 to your Application:



2) Then we need to capture the moving mouse position:

private void jPanel2MouseDragged(java.awt.event.MouseEvent evt) {
counter = 0;
if (to_be_clear == true) {
clearArea();
}
start++;
Graphics g = jPanel2.getGraphics();
//drawing color is identified by jLabel4 background.
g.setColor(jLabel4.getBackground());
//Only consider major changes
if (linePoints.size() == 0 || Math.abs(linePoints.lastElement().getX() - evt.getX()) > 8 || Math.abs(linePoints.lastElement().getY() - evt.getY()) > 8
|| (Math.abs(linePoints.lastElement().getX() - evt.getX()) > 5 && Math.abs(linePoints.lastElement().getY() - evt.getY()) > 5)) {
g.drawOval(evt.getX(), evt.getY(), 3, 3);
linePoints.add(new Point(evt.getX(), evt.getY(), false));
}
g.drawRect(evt.getX(), evt.getY(), 1, 1);
if (basal_line < evt.getY()) {
basal_line = evt.getY();
}
if (top_line > evt.getY()) {
top_line = evt.getY();
}
}
//clear drawing panel
private void clearArea() {
to_be_clear = false;
top_line = 200;
basal_line = 0;
start = 0;
linePoints = new Vector();
totalResult = "";
buffer = "";
counter = 0;
Graphics g=jPanel2.getGraphics();
g.setColor(Color.white);
g.fillRect(3, 13, jPanel2.getWidth()-6,jPanel2.getHeight()-16);
if(showSplitPanels){
drawSplitPanels(g);
}
}
//draw split pannel to separate numbers from operations , it is used also to separate letters from numbers, capital from small letters.
private void drawSplitPanels(Graphics g){
g.setColor(Color.BLUE);
g.drawLine(5, 85, jPanel2.getWidth() - 10, 85);
g.drawString("123",jPanel2.getWidth() - 30,80);
g.drawString("+-x",jPanel2.getWidth() - 30,95);
}

//run method to capture input
public void run() {
while (true) {
try {
counter++;
// no movement for 10 times call of run method each wait for 100 milliseconds, so total is 1 seconds of idle movement..
if (counter >= 10) {
counter = 0;
if (!totalResult.equals("")) {
String newEntry = processSingleLetter();
removeError();
if (newEntry.equals("=")) {
newEntry = MathProcess.getInstance().process(jTextField1.getText());
//jTextField1 capture the entered charachter
jTextField1.setText(newEntry);
//do some animation by playing the entered digits or charachter
doAnimation("=", false);
if (!jCheckBox1.isSelected()) {
//do some animation by playing the results of calculation
TextToSpeech.textToSpeach(jTextField1.getText());
}
} else if (newEntry.equals("")) {
//do nothing
} else {
addToResult(newEntry, false);
}
}
}
Thread.sleep(100);
} catch (InterruptedException ex) {
}
}

}
//method to add entered data into jTextField1
public void addToResult(String toAdd, boolean dots) {
removeError();
if (toAdd == null || toAdd.equals("")) {
return;
}
if (toAdd.equals("+") || toAdd.equals("-") || toAdd.equals("x") || toAdd.equals("÷")) {
if (jTextField1.getText().endsWith("+") || jTextField1.getText().endsWith("-")
|| jTextField1.getText().endsWith("x")
|| jTextField1.getText().endsWith("÷")) {
showError(false);
return;
}
}
jTextField1.setText(jTextField1.getText() + toAdd);
//play entered charachter
doAnimation(toAdd, dots);
}

//some utility methods:
private void showError(boolean mayBe) {
linePoints = new Vector();
totalResult = "";
Graphics g = jPanel2.getGraphics();
if (!jTextField2.getText().equals("")) {
if (mayBe) {
playSound("MayBe");
waitFor(1000);
playSound(jTextField2.getText());
g.drawImage(mayBeImage, 20, 35, null);
} else {
g.drawImage(errorImage, 15, 35, null);
playSound("wrong");
}
} else {
g.drawImage(errorImage, 15, 35, null);
playSound("wrong");
}
to_be_clear = true;
}

private void waitFor(int i) {
try {
Thread.sleep(i);
} catch (InterruptedException ex) {
}

}


3) We need to format the input data once the movement stop for certain period of time:
We capture direction using freeman's directions, the following chart describes these directions:


private String parseCurrent() {
String result = "";
if (linePoints.size() == 1) {
return "P";
} else if (linePoints.size() > 1) {
Point oldP = linePoints.get(0);
for (int i = 0; i < linePoints.size(); i++) {
Point newP = linePoints.get(i);
Graphics g = jPanel2.getGraphics();
g.setColor(jLabel4.getBackground());
g.drawOval(newP.getX(), newP.getY(), 3, 3);
if (i > 0) {
String newResult = Point.comparePoints(oldP, newP);
if (!result.endsWith(newResult)) {
result += newResult;
}
oldP = newP;
}
}
//consider 1st and last point if they are the same only
Point newP = linePoints.lastElement();
oldP = linePoints.firstElement();
String newResult = "";
if (Math.abs(newP.getX() - oldP.getX()) < 10 && Math.abs(newP.getY() - oldP.getY()) < 10) {
newResult = "C"; //C means joined 1st and last
} else {
if (newP.getY() >= (basal_line - 5)) {
newResult = "L"; //L means last point almost below basal line
} else if (newP.getY() < basal_line) {
newResult = "N"; //N means last ponint above basal line
} else {
newResult = "N"; //N means last ponint above basal line
}
}
result = result + newResult;
result = shortcutLine(result);
return result;
}
return "";
}



Removing extra information which are oblique lines connecting vertical vs horizontal lines.

private String shortcutLine(String result) {
if (result.length() <= 4) {
return result;
}
result = result.replaceAll("654", "64"); //1
result = result.replaceAll("456", "46");
result = result.replaceAll("432", "42"); //2
result = result.replaceAll("234", "24");
result = result.replaceAll("210", "20"); //3
result = result.replaceAll("012", "02");
result = result.replaceAll("076", "06"); //4
result = result.replaceAll("670", "60");
return result;
}



4)We need to identify the entry by direct lookup or by pattern recognition classifier:
We will use K-Nearest Neighbor KNN for that because we need simple classifier.

private String processSingleLetter() {
System.out.println("Letter Total=" + totalResult);
String value = lookupTable.get(totalResult);
if (value == null) {
//TODO : send type
value = KNN.getInstance().CalculateKNN(lookupTable, totalResult, jTextField2,updateMode(),sensitivity);
}
if (value != null && !value.equals("")) {
//doAnimation(value);
clearArea();
} else {
showError(true);
}
return value;
}

//KNN class code:
public static int MAX_NIGHBOURS=3;
public static int ALL=0;
public static int OPERATION=1;
public static int NUMBERS=2;
public static Hashtable operationsHashtable=new Hashtable();
public static Hashtable numbersHashtable=new Hashtable();
private static KNN kNNInstance;
private KNN(){
numbersHashtable.put("0",true);
numbersHashtable.put("1",true);
numbersHashtable.put("2",true);
numbersHashtable.put("3",true);
numbersHashtable.put("4",true);
numbersHashtable.put("5",true);
numbersHashtable.put("6",true);
numbersHashtable.put("7",true);
numbersHashtable.put("8",true);
numbersHashtable.put("9",true);
operationsHashtable.put("+", true);
operationsHashtable.put("-", true);
operationsHashtable.put("÷", true);
operationsHashtable.put("x", true);
operationsHashtable.put("=", true);
}
public static KNN getInstance(){
kNNInstance=new KNN();
return kNNInstance;
}
public String CalculateKNN(Hashtable lookupTable,String value,JTextField jTextField,int mode,int sensitivity){
Hashtable distanceTable=new Hashtable();
Enumeration keys=lookupTable.keys();
while(keys.hasMoreElements()){
String current=keys.nextElement();
int distance=calculateDistance(current,value);
if(distanceTable.get(lookupTable.get(current))==null){
distanceTable.put(lookupTable.get(current), new KNNBean());
}
distanceTable.get(lookupTable.get(current)).addDistance(distance);
}
int lessValue=100;
String lessValueKey="";
keys=distanceTable.keys();
while(keys.hasMoreElements()){
String current=keys.nextElement();
if(mode==OPERATION){
if(operationsHashtable.get(current)==null){
System.out.println("Not an operation,skip it");
continue;
}
}
if(mode==NUMBERS){
if(numbersHashtable.get(current)==null){
System.out.println("Not a number,skip it");
continue;
}
}
KNNBean bean=distanceTable.get(current);
if(lessValue>bean.getTotalSum()){
lessValue=bean.getTotalSum();
lessValueKey=current;
}
}
if(lessValue>1+sensitivity && lessValue<5+sensitivity){
jTextField.setText(lessValueKey);
return "";
}else if(lessValue>=10+sensitivity){
jTextField.setText("");
return "";
}
return lessValueKey;
}

private int calculateDistance(String current, String value) {
int match=0;
if(current.indexOf(value.substring(0,value.length()-1))==-1){
match+=2;
}
for(int i=0;i if(current.indexOf(value.charAt(i))==-1){
match++;
}
}
match+=Math.abs(current.length()-value.length());
return match;
}



The methodology is to capture directions without wights and remove extra information but not all directions are captured only significant directions, also the above picture show how different shapes with different scaling are translated to the same directions, this is simple and accurate methodology when we deal with unique shapes (need some concerns to deal with Arabic handwriting).