source: other-projects/is-sheet-music-encore/trunk/image-identification-development/src/Main.java@ 33310

Last change on this file since 33310 was 33310, checked in by cpb16, 5 years ago

developing line clustering. Have completed line cluster algorithm. need to check it is calculating correctly
File size: 21.4 KB
Line 
1import org.opencv.core.*;
2import org.opencv.core.Point;
3import org.opencv.highgui.HighGui;
4import org.opencv.imgcodecs.Imgcodecs;
5import org.opencv.imgproc.Imgproc;
6import org.opencv.photo.Photo;
7import static org.opencv.imgcodecs.Imgcodecs.imwrite;
8import java.awt.image.BufferedImage;
9import java.awt.image.DataBufferByte;
10import java.io.File;
11import java.util.ArrayList;
12import java.util.Collection;
13import java.util.Collections;
14import javax.imageio.ImageIO;
15
16//REFERENCES:
17//https://docs.opencv.org/3.4.3/d9/db0/tutorial_hough_lines.
18//https://stackoverflow.com/questions/43443309/count-red-pixel-in-a-given-image
19//https://www.wikihow.com/Calculate-Percentage-in-Java
20//https://riptutorial.com/opencv/example/21963/converting-an-mat-object-to-an-bufferedimage-object
21//https://beginnersbook.com/2013/12/java-arraylist-of-object-sort-example-comparable-and-comparator/
22//https://www.programiz.com/java-programming/examples/standard-deviation
23//https://www.geeksforgeeks.org/how-to-remove-duplicates-from-arraylist-in-java/
24
25
26
27
28//GOAL for 21st
29
30
31//Classifier 01
32//Have args so can call "java image-identification-classifier01 XX XX"
33//args can be parameters in algorthim such as threshold or theta?
34//Run on 5000 images.
35//Record success rates
36//All done with makefile
37
38
39//But first understand houghline transform
40//Know what the algorithm being used is doing.
41//MAke constants for this classifier
42//Make java be able to run on CMD line
43
44public class Main {
45
46 //DEPENDENT FUNCTIONS AND CLASSES
47 static class StartAndEndPoint {
48 //PRIVATES
49 private Point _p1;
50 private Point _p2;
51 //CONSTRUCTOR
52 public StartAndEndPoint(Point p1, Point p2){
53 _p1 = p1;
54 _p2 = p2;
55 }
56 //GETTERS
57 public Point getP1(){
58 return _p1;
59 }
60 public Point getP2(){
61 return _p2;
62 }
63 //SETTERS
64 public void setP1(Point p1){
65 _p1 = p1;
66 }
67 public void setP2(Point p2){
68 _p2 = p2;
69 }
70
71 //ToString
72 public String toString(){
73 return "Start: " + _p1 + " End: " + _p2;
74 }
75 /*
76 //CompareToOverride
77 //Compares start point y co ordinates of input PointArray
78 //With this. start point y co ordinate
79 @Override
80 public double compareTo(StartAndEndPoint comparePointArray){
81 Point comparePoint = (comparePointArray.getP1());
82 return (this.getP1().y) - (comparePoint.y);
83 }
84 */
85 }
86 public static <T> ArrayList<T> removeDuplicates(ArrayList<T> list) {
87 //DIRECTLY COPIED//DIRECTLY COPIED//DIRECTLY COPIED//DIRECTLY COPIED//DIRECTLY COPIED//DIRECTLY COPIED
88 // Function to remove duplicates from an ArrayList
89 // Create a new ArrayList
90 ArrayList<T> newList = new ArrayList();
91 // Traverse through the first list
92 for (T element : list) {
93 // If this element is not present in newList
94 // then add it
95 if (!newList.contains(element)) {
96 newList.add(element);
97 }
98 }
99 // return the new list
100 return newList;
101 //DIRECTLY COPIED//DIRECTLY COPIED//DIRECTLY COPIED//DIRECTLY COPIED//DIRECTLY COPIED//DIRECTLY COPIED
102 }
103 public static double StandardDeviation(double parseArray[])
104 {
105
106 double mean;
107 double sum =0.0;
108 double standardDeviation = 0.0;
109 //calculate sum of array
110 for(int i =0; i < parseArray.length; i++){
111 sum += parseArray[i];
112 }
113 //calculate mean of array
114 mean = sum/parseArray.length;
115 //calculate SD of array
116 for(int j =0; j < parseArray.length; j++){
117 standardDeviation += Math.pow(parseArray[j]-mean, 2);
118 }
119 return Math.sqrt(standardDeviation/parseArray.length);
120
121
122 }
123
124 //GLOBAL_CONSTANTS
125 static int CLASSIFIER_HOUGHLINESP_MIN = 10;
126 static int CLASSIFIER_HOUGHLINESP_MAX = 65;
127 static int HOUGHLINEP_THRESHOLD = 10;
128 static int STANDARD_DEVIATION_THRESHOLD = 6;
129 static int MINLINECOUNT = 40;
130 static double MAXLINEGAP = 1; //4
131 static double SLOPEGRADIENT = 0.02;
132 //SHOULD TURN INTO ARGS
133
134 //CLASSIFYING FUNCTIONS
135 private static BufferedImage toBufferedImage(Mat mat){
136 //MOSTLY COPY PASTE!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
137 //MOSTLY COPY PASTE!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
138 //https://riptutorial.com/opencv/example/21963/converting-an-mat-object-to-an-bufferedimage-object
139 try{
140 int type = BufferedImage.TYPE_3BYTE_BGR;
141 int bufferSize = mat.channels() * mat.cols() * mat.rows();
142 byte[] b = new byte[bufferSize];
143 //get all the pixels
144 mat.get(0, 0, b);
145 BufferedImage image = new BufferedImage(mat.cols(), mat.rows(), type);
146 final byte[] targetPixels = ((DataBufferByte) image.getRaster().getDataBuffer()).getData();
147 System.arraycopy(b, 0, targetPixels, 0, b.length);
148 return image;
149 }
150 catch(Exception e){
151 System.err.println(e);
152 }
153 return null;
154 }
155 private static boolean ClassifierPixelCount(BufferedImage img){
156 try {
157 //Read file
158 //BufferedImage img = ImageIO.read(new File(processedFile));
159 int x = img.getWidth();
160 int y = img.getHeight();
161 int pixelCount = 0;
162 int redCount = 0;
163 float percentage = 0;
164
165 //Go Thru every pixel
166 for(int i=0; i < y; i++){
167 for(int j=0;j < x; j++){
168 //Get value for current pixels RGB value
169 int currPixelRGB = img.getRGB(j, i);
170 //Check if pixel is red (hex value of red)
171 if(currPixelRGB == 0xFFFF0000){
172 redCount++;
173 }
174 pixelCount++;
175 }
176 }
177 //Calculate percentage of Red in image
178 percentage = ((float)redCount/(float)pixelCount)*(float)100;
179 //If more than %10 and less than %50 then its sheet music!
180 if(percentage > CLASSIFIER_HOUGHLINESP_MIN && percentage < CLASSIFIER_HOUGHLINESP_MAX){ //MAKE THESE CONSTANTS!!
181 return true;}
182 }
183 catch (Exception e) {
184 System.err.println(e);
185 }
186 return false;
187 }
188 private static boolean ClassifierLineCount(int lineCount){
189
190 if(lineCount>MINLINECOUNT){
191 return true;
192 }
193 else{
194 return false;
195 }
196 }
197 private static ArrayList ClassifierLineClusterOLD(BufferedImage img){
198 ArrayList returnArray = new ArrayList();
199 try {
200
201 //IF THIS WORKS THEN IMPLEMENT A VERSION THAT USES POINTS from the draw line code.
202 //ALSO CHECK OUT K NEAREST NEIGHBOR?
203 //0xFFFF0000 = RED
204
205 //go thru every pixel until find red pixel
206 //get y pos of red pixel
207 //continue with loop until find another red pixel
208 //get y pos of red pixel
209 //compare y pos (if close together then continue loop) else break
210
211 int x = img.getWidth();
212 int y = img.getHeight();
213 int closeLineCount = 0;
214 ArrayList<Integer> redPixelYpos = new ArrayList<Integer>();
215
216 //Go Thru every pixel
217 for(int i=0; i < y; i++){
218 for(int j=0;j < x; j++){
219 //Get value for current pixels RGB value
220 int currPixelRGB = img.getRGB(j, i);
221 //Check if pixel is red (hex value of red)
222 if(currPixelRGB == 0xFFFF0000) {
223
224 //Store y pos of red pixel if there is no duplicate
225 if(!redPixelYpos.contains(i)){
226 redPixelYpos.add(i);
227 //System.out.println(i );
228 }
229 }
230 }
231 }
232 //Check if any of the lines found are close together and that there has been more than one line found
233 if(redPixelYpos.size()>1){
234 //go through list and compare every value
235 for(int i =0; i< redPixelYpos.size(); i++){
236 //System.out.println("i: " +redPixelYpos.get(i));
237 for(int j=0; j< redPixelYpos.size(); j++){
238 //System.out.println("j: "+redPixelYpos.get(j));
239 //Check if difference is less than 4 and the values are not duplicates.
240 if(Math.abs(redPixelYpos.get(i) - redPixelYpos.get(j)) < 4 && !redPixelYpos.get(j).equals(redPixelYpos.get(i))){
241 closeLineCount++;
242 }
243 }
244 }
245 }
246 int clusterCount = closeLineCount/4;
247
248 if(closeLineCount >= 4){
249 returnArray.add(true);
250 returnArray.add(closeLineCount);
251 returnArray.add(clusterCount);
252 }
253 else{
254 returnArray.add(false);
255 returnArray.add(closeLineCount);
256 returnArray.add(clusterCount);
257 }
258 }
259 catch (Exception e) {
260 System.err.println(e);
261 }
262 return returnArray;
263 }
264 private static ArrayList ClassifierLineCluster(ArrayList<StartAndEndPoint> linePointsArray, Mat clustersFoundRGB){
265
266 /*
267 This will check for a cluster of lines that are close together.
268 1. Go through the list of Y positions(start point) in parsed array.
269 If, there is a small distance between them,
270 then, add to closeLineArray.
271
272 Have all Y positions that are close to each other now.
273 Need to find the lines that are clustered together.
274
275 Now check if there are four of these are close to each other.
276 2. Go through list of closeLine.
277 Get first four lines, traversing down a step each iteration {0,1,2,3} -> {1,2,3,4} -> {2,3,4,5}
278 If, those 4 lines are close together,
279 Then, add them to a new array that holds Line Cluster Values.
280 Go to line 4 positions down since, as do not want duplicates.
281
282 3.
283 */
284
285 ArrayList returnArray = new ArrayList();
286 ArrayList<Double> closeLineYPos = new ArrayList();
287 ArrayList<double[]> clusterArray = new ArrayList();
288 int clusterCount = 0;
289 try {
290 if(linePointsArray.size()> 1) {
291
292 /*
293 //Display input array TESTING PURPOSES
294 for (int i = 0; i < linePointsArray.size(); i++) {
295 System.out.println(linePointsArray.get(i).toString());
296 }
297 */
298
299 //1. Check if y points are close together
300 //go thru list and compare values against each other
301 for (int i = 0; i < linePointsArray.size(); i++){
302 //System.out.println("i: "+ linePointsArray.get(i).getP1().y);
303 for (int j = 0; j < linePointsArray.size(); j++) {
304 //System.out.println("j: "+ linePointsArray.get(j).getP1().y);
305 //Check if difference is less than 4 and the values are not duplicates.
306 if(Math.abs(linePointsArray.get(j).getP1().y - linePointsArray.get(i).getP1().y) < 5){
307 if(linePointsArray.get(j).getP1().y != linePointsArray.get(i).getP1().y){
308 closeLineYPos.add(linePointsArray.get(j).getP1().y);
309 }
310 }
311 }
312 }
313 /*for (double num : closeLineYPos){
314 System.out.println(num);
315 } */
316
317 //2. Now check if there are four of these are close to each other.
318 //Go through all of the items in this list and check if four of them are close together
319 //Check first four items, traverse down a step {0,1,2,3} -> {1,2,3,4} -> {2,3,4,5}
320 //If 4 items are close together,
321 //Then add them to a new array that holds Line Cluster Values.
322 //Go down 4 positions down since, as do not want duplicates.
323
324 //Now have an array of at least four lines that are close together.
325 //Sort array and remove duplicates
326 Collections.sort(closeLineYPos);
327 closeLineYPos = removeDuplicates(closeLineYPos);
328 if(closeLineYPos.size() >= 4) {
329 //FOR every item in array of CloseLines
330 for(int i= 0; i< closeLineYPos.size(); i++){
331 //If last comparator is at end of array.
332 if(i + 4 >= closeLineYPos.size()){
333 break;
334 }
335 else{
336 //Add 4 values of Close Line Array to a tempArray
337 double[] tempArray = new double[4];
338 tempArray[0] = closeLineYPos.get(i);
339 tempArray[1] = closeLineYPos.get(i + 1);
340 tempArray[2] = closeLineYPos.get(i + 2);
341 tempArray[3] = closeLineYPos.get(i + 3);
342
343 //Check standard deviation between these 4 values.
344 //If it SD is less than 5 then it is considered to be a cluster of lines.
345
346 if(StandardDeviation(tempArray) < STANDARD_DEVIATION_THRESHOLD){
347 System.out.println("tempArray PT: "+tempArray[0] + " , " + tempArray[1] + " , " + tempArray[2] + " , " + tempArray[3]);
348 System.out.println("tempArray SD: " + StandardDeviation(tempArray));
349 //Store array
350 clusterArray.add(tempArray);
351 //If I + 4 is less than the size of the array then increment by 4
352 //Go down +4 positions in closeLineYPos array
353 if((i + 4 < closeLineYPos.size())){
354 System.out.println("IF, i = " + i + " -> "+ (i+4) + ", CloseLineYpos size= " + closeLineYPos.size());
355 i = i+4;
356 }
357 else{
358 //break
359 System.out.println("ELSE, i = " + i+ " closeLineYpos size= " + closeLineYPos.size());
360 Thread.sleep(2000);
361 break;
362 }
363 }
364 }
365 }
366 }
367 /*
368 System.out.println("Cluster Coordinates: ");
369 for(double[] items : clusterArray){
370 for(int i = 0; i <items.length; i++){
371 System.out.println("ITEMS: "+ items[i]);
372 }
373 }
374 */
375 //Setup Drawing clusters found.
376 //For every pt given the input array
377 for(StartAndEndPoint pt : linePointsArray){
378 //Go through every the Arrays in the clusterArray(clustered lines)
379 for(int i =0; i < clusterArray.size(); i++){
380 //Go through every item in the array
381 for(double item : clusterArray.get(i)) {
382 //Check if the curr item is equal to current pt
383 if (item == pt.getP1().y){
384 //calculate a different colour for each line
385
386 //Draw a line
387 Imgproc.line(clustersFoundRGB, pt.getP1(), pt.getP2(), new Scalar(0, 255, 0), 1, Imgproc.LINE_4, 0);
388 }
389 }
390 }
391
392 }
393
394 clusterCount = clusterArray.size();
395 //SETUP RETURN ARRAY
396 if(clusterCount >= 1){
397 returnArray.add(true);
398 returnArray.add(closeLineYPos.size());
399 returnArray.add(clusterCount);
400 returnArray.add(clustersFoundRGB);
401 }
402 else{
403 returnArray.add(false);
404 returnArray.add(closeLineYPos.size());
405 returnArray.add(clusterCount);
406 }
407 }
408 }
409 catch (Exception e) {
410 System.err.println(e);
411 }
412 return returnArray;
413 }
414
415
416
417 public static void main(String[] args) {
418
419 System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
420
421 try {
422 ArrayList<StartAndEndPoint> pointArrayList = new ArrayList<>();
423
424 //Variables
425 Mat edgesDetected = new Mat();
426 Mat edgesDetectedRGB = new Mat();
427 Mat clustersFoundRGB = new Mat();
428 String directory = "/Scratch/cpb16/is-sheet-music-encore/download-images/MU/";
429 //!!!!!!!!!!!!!!!!!!!!!!!!!!!NOT!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
430 //mdp.39015097852365-2.png 176 lines Contents page.
431 //mdp.39015097852555-3.png 76 lines
432 //String default_file = directory+"SheetMusic/coo.31924062612282-9.png";
433 //String default_file ="TestImages/NotNot/mdp.39015080972303-3.png";
434
435
436 //System.out.println(default_file);
437 //String default_file = "TestImages/NotSheetMusic01.png";
438 //String default_file = "TestImages/NotSheetMusic02.png";
439 //String default_file = "TestImages/SheetMusic01.png";
440 String default_file = "TestImages/SheetMusic02.png";
441 //String default_file = "TestImages/vLine.png";
442 String filename = ((args.length > 0) ? args[0] : default_file);
443 File file = new File(filename);
444 if(!file.exists()){System.err.println("Image not found: "+ filename);}
445
446 int horizontalLineCount =0;
447
448 // Load an image
449 Mat original = Imgcodecs.imread(filename, Imgcodecs.IMREAD_GRAYSCALE);
450 // Edge detection
451 Imgproc.adaptiveThreshold(original, edgesDetected,255, Imgproc.ADAPTIVE_THRESH_GAUSSIAN_C,Imgproc.THRESH_BINARY_INV, 15, 4);
452
453 //Convert to RGB for future use
454 Imgproc.cvtColor(edgesDetected, edgesDetectedRGB, Imgproc.COLOR_GRAY2BGR);
455 clustersFoundRGB = edgesDetectedRGB.clone();
456
457 Mat linesP = new Mat(); // will hold the results of the detection
458 //(edgeDetectedImage, outputOfDetection(r,Ξ), resolution of rho, resolution of theta, threshold (minimum num of intersections)
459
460 double minLineLength = edgesDetectedRGB.size().width/8;
461
462 Imgproc.HoughLinesP(edgesDetected, linesP, 1, Math.PI / 720, HOUGHLINEP_THRESHOLD, minLineLength,MAXLINEGAP); // runs the actual detection
463 System.out.println("Before Gradient Filtering num lines: " + linesP.rows());
464
465 // Draw the lines
466 for (int x = 0; x < linesP.rows(); x++) {
467 double[] l = linesP.get(x, 0);
468 Point p1 = new Point(l[0], l[1]);
469 Point p2 = new Point(l[2], l[3]);
470 double m = Math.abs(p2.y - p1.y)/(p2.x - p1.x);
471
472 if(m<=SLOPEGRADIENT) {
473 Imgproc.line(edgesDetectedRGB, p1, p2, new Scalar(0, 0, 255), 1, Imgproc.LINE_4, 0);
474 horizontalLineCount++;
475 pointArrayList.add(new StartAndEndPoint(p1, p2));
476 }
477
478 }
479 //Point is a co ordinate (x, y)
480 //Prove by finding number of points from one end to other:
481 //Get width of image.
482 File filenameTest = new File("TestImages/NotSheetMusic02.png");
483 BufferedImage i = ImageIO.read(filenameTest);
484 BufferedImage toBeClassifiedImg = toBufferedImage(edgesDetectedRGB);
485
486
487
488 //Display Results
489 //HighGui.imshow("Source", original);
490 //HighGui.imshow("Just Edges", justEdges); //TESTING
491 HighGui.imshow("Detected Lines (in red) - positive", edgesDetectedRGB);
492 if(ClassifierLineCluster(pointArrayList, clustersFoundRGB).get(3) != null) {
493 HighGui.imshow("CLUSTERS FOUND", clustersFoundRGB);
494 }
495 //HighGui.imshow("Detected Lines (in red) - negative", edgesDetectedRGBProb);
496
497
498
499 //System.out.println("LINE COUNT RESULT: " + ClassifierLineCount(horizontalLineCount) + '\t' +"LineCount: " + horizontalLineCount); //COUNT OF LINES CLASSIFICATION
500 //System.out.println("LINE CLUSTER RESULT: " + ClassifierLineClusterOLD(toBeClassifiedImg).get(0) + '\t' + "LinesFound: " + ClassifierLineClusterOLD(toBeClassifiedImg).get(1) + '\t' + "ClustersFound: " + ClassifierLineClusterOLD(toBeClassifiedImg).get(2));
501 System.out.println("NEW CLUSTER RESULTS: " + ClassifierLineCluster(pointArrayList,clustersFoundRGB).get(0) + '\t' + "LinesFound: " + ClassifierLineCluster(pointArrayList,clustersFoundRGB).get(1) + '\t' + "ClustersFound: " + ClassifierLineCluster(pointArrayList,clustersFoundRGB).get(2));
502 //System.out.println(ClassifierLineCluster(pointArrayList, clustersFoundRGB));
503
504 // Wait and Exit
505 HighGui.waitKey();
506 System.exit(0);
507 }
508 catch(Exception e){
509 System.err.println(e);
510 }
511 }
512}
Note: See TracBrowser for help on using the repository browser.