source: other-projects/is-sheet-music-encore/trunk/image-identification-dev-02/image-identification-development/src/MainMorph.java@ 33418

Last change on this file since 33418 was 33418, checked in by cpb16, 5 years ago

made progress with morphology, based one image, need to refine further, then test on multiple images
File size: 18.3 KB
Line 
1/*
2 StartAndEndPoint l1 = parseArray[i];
3 StartAndEndPoint l2 = parseArray[i+ 1];
4 //CHECK WHICH line starts after the other
5 //If l1 is starting after, then comparisons are based around l1.s
6 //System.out.println("l1: " + l1.getP1().x);
7 //System.out.println("l2: " + l2.getP1().x);
8
9 System.out.println("1.0: L1S: " + l1.getP1().x + " larger than L2S: " + l2.getP1().x);
10 if(l1.getP1().x > l2.getP1().x) {
11 System.out.println("1.1: Comparing L1S: " + l1.getP1().x + " less than L2E: " + l2.getP2().x);
12 if (l1.getP1().x < l2.getP2().x) {
13 //AND
14 System.out.println("1.2: Comparing L1S: " + l1.getP1().x + " larger than L2S: " + l2.getP1().x);
15 if (l1.getP1().x > l2.getP1().x) {
16 System.out.println("1: Success. NEXT");
17 //IT IS INTERSECTED
18 continue;
19 }
20 else {
21 //FAILED SECOND COMPARISON
22 System.out.println("1: Fail");
23 }
24 }
25 else {
26 System.out.println("Checking other line");
27 }
28 System.out.println("2.0: L2S: " + l2.getP1().x + " larger than L1S: " + l1.getP1().x);
29 }
30 //If l2 is starting after, then comparisons are based around l2.s
31 else if(l2.getP1().x > l1.getP1().x) {
32 System.out.println("2.1: Comparing L2S: " + l1.getP1().x + " less than L1E: " + l2.getP2().x);
33 if (l2.getP1().x < l1.getP2().x) {
34 //AND
35 System.out.println("2.2: Comparing L2S: " + l2.getP1().x + " larger than L1S: " + l1.getP1().x);
36 if (l2.getP1().x > l1.getP1().x) {
37 System.out.println("2: Success");
38 //IT IS INTERSECTED
39 //continue;
40 }
41 else {
42 //FAILED SECOND COMPARISON
43 System.out.println("2: Fail");
44 //return false;
45 }
46 }
47 else {
48 System.out.println("Failed second comparison RETURN FALSE");
49 return false;
50 }
51 //return false;
52 }
53 else{
54 System.out.println("NEITHER RETURN FALSE");
55 return false;
56 }
57 */
58
59import org.opencv.core.*;
60import org.opencv.core.Point;
61import org.opencv.highgui.HighGui;
62import org.opencv.imgcodecs.Imgcodecs;
63import org.opencv.imgproc.Imgproc;
64import org.opencv.photo.Photo;
65import static org.opencv.imgcodecs.Imgcodecs.imwrite;
66import java.awt.image.BufferedImage;
67import java.awt.image.DataBufferByte;
68import java.io.File;
69import java.util.ArrayList;
70import java.util.Collection;
71import java.util.Collections;
72import java.util.Comparator;
73import javax.imageio.ImageIO;
74
75//REFERENCES:
76//https://docs.opencv.org/3.4.3/d9/db0/tutorial_hough_lines.
77//https://stackoverflow.com/questions/43443309/count-red-pixel-in-a-given-image
78//https://www.wikihow.com/Calculate-Percentage-in-Java
79//https://riptutorial.com/opencv/example/21963/converting-an-mat-object-to-an-bufferedimage-object
80//https://beginnersbook.com/2013/12/java-arraylist-of-object-sort-example-comparable-and-comparator/
81//https://www.programiz.com/java-programming/examples/standard-deviation
82//https://www.geeksforgeeks.org/how-to-remove-duplicates-from-arraylist-in-java/
83//https://stackoverflow.com/questions/7988486/how-do-you-calculate-the-variance-median-and-standard-deviation-in-c-or-java/7988556
84//https://stackoverflow.com/questions/10396970/sort-a-list-that-contains-a-custom-class
85//https://stackoverflow.com/questions/37946482/crop-images-area-with-opencv-java
86//https://docs.opencv.org/3.4/dd/dd7/tutorial_morph_lines_detection.html
87
88
89
90//GOAL for 21st
91
92
93//Classifier 01
94//Have args so can call "java image-identification-classifier01 XX XX"
95//args can be parameters in algorthim such as threshold or theta?
96//Run on 5000 images.
97//Record success rates
98//All done with makefile
99
100
101//But first understand houghline transform
102//Know what the algorithm being used is doing.
103//MAke constants for this classifier
104//Make java be able to run on CMD line
105
106public class MainMorph {
107 //GLOBAL_CONSTANTS
108
109 static double THRESHOLD_C = 4;
110
111
112 //
113 static class StartAndEndPoint {
114 //PRIVATES
115 private Point _p1;
116 private Point _p2;
117 //CONSTRUCTOR
118 public StartAndEndPoint(Point p1, Point p2){
119 _p1 = p1;
120 _p2 = p2;
121 }
122 //GETTERS
123 public Point getP1(){
124 return _p1;
125 }
126 public Point getP2(){
127 return _p2;
128 }
129 //SETTERS
130 public void setP1(Point p1){
131 _p1 = p1;
132 }
133 public void setP2(Point p2){
134 _p2 = p2;
135 }
136
137 //ToString
138 public String toString(){
139 return "Start: " + _p1 + " End: " + _p2;
140 }
141
142 }
143 private static BufferedImage toBufferedImage(Mat mat){
144 //MOSTLY COPY PASTE!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
145 //MOSTLY COPY PASTE!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
146 //https://riptutorial.com/opencv/example/21963/converting-an-mat-object-to-an-bufferedimage-object
147 try{
148 int type = BufferedImage.TYPE_3BYTE_BGR;
149 int bufferSize = mat.channels() * mat.cols() * mat.rows();
150 byte[] b = new byte[bufferSize];
151 //get all the pixels
152 mat.get(0, 0, b);
153 BufferedImage image = new BufferedImage(mat.cols(), mat.rows(), type);
154 final byte[] targetPixels = ((DataBufferByte) image.getRaster().getDataBuffer()).getData();
155 System.arraycopy(b, 0, targetPixels, 0, b.length);
156 return image;
157 }
158 catch(Exception e){
159 System.err.println(e);
160 }
161 return null;
162 }
163
164 private static void showWaitDestroy(String winname, Mat img) {
165 HighGui.imshow(winname, img);
166 HighGui.moveWindow(winname, 500, 0);
167 HighGui.waitKey(0);
168 HighGui.destroyWindow(winname);
169 }
170 //MAIN
171 public static void main(String[] args) {
172
173 System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
174
175 try {
176 ArrayList<StartAndEndPoint> pointArrayList = new ArrayList<>();
177
178 //Variables
179 int codeVersion = 2;
180 Mat edgesDetected = new Mat();
181 Mat mid = new Mat();
182 Mat edgesDetectedRGB = new Mat();
183 Mat clustersFoundRGB = new Mat();
184 String directory = "/Scratch/cpb16/is-sheet-music-encore/download-images/MU/";
185 String hiresDirectory = "/Scratch/cpb16/is-sheet-music-encore/hires-download-images/";
186
187 //!!!!!!!!!!!!!!!!!!!!!!!!!!!NOT!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
188 //mdp.39015097852365-2.png 176 lines Contents page.
189 //mdp.39015097852555-3.png 76 lines
190 //!!!!!!!!!!!!!!!!!!!!!!!!!!!NOTNOT!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
191 //coo.31924062612282-9.png 8 lines
192 //String default_file = directory+"NotSheetMusic/coo.31924062612282-9.png";
193 //String default_file = directory+"NotSheetMusic/mdp.39015097852365-2.png";
194 //String default_file ="TestImages/NotNot/mdp.39015080972303-3.png";
195 //String default_file =hiresDirectory+"BK/NotSheetMusic/aeu.ark+=13960=t2q53nq6w-6.png";
196 String default_file = "/Scratch/cpb16/is-sheet-music-encore/image-identification-terminal/TestImages/test-coo.31924062612282-9.png";
197 //String default_file =hiresDirectory+"BK/NotSheetMusic/aeu.ark+=13960=t9z03w65z-4.png";
198
199 //System.out.println(default_file);
200 //String default_file = "TestImages/NotSheetMusic01.png";
201 //String default_file = "TestImages/NotSheetMusic02.png";
202 //String default_file = "TestImages/SheetMusic01.png";
203 //String default_file = "TestImages/SheetMusic02.png";
204 //String default_file = "TestImages/vLine.png";
205 String filename = ((args.length > 0) ? args[0] : default_file);
206 File file = new File(filename);
207 if(!file.exists()){System.err.println("Image not found: "+ filename);}
208
209 int horizontalLineCount =0;
210
211 // Load an image
212 Mat original = Imgcodecs.imread(filename, Imgcodecs.IMREAD_GRAYSCALE);
213 // Edge detection
214
215 Imgproc.adaptiveThreshold(original, edgesDetected,255, Imgproc.ADAPTIVE_THRESH_GAUSSIAN_C,Imgproc.THRESH_BINARY_INV, 15, THRESHOLD_C);
216 //TEST PARAMETERSImgproc.adaptiveThreshold(original, edgesDetected,255, Imgproc.ADAPTIVE_THRESH_GAUSSIAN_C,Imgproc.THRESH_BINARY_INV, 531,1);
217
218
219
220 //****************MORPHOLOGY****************************************************************************************
221 //ADDIOTIONAL FILTERING TO STOP STREAKS
222 //LOOK INTO STREAKS MORPHOGOLY.
223 //****************MORPHOLOGY****************************************************************************************
224
225 // Create the images that will use to extract the horizontal and vertical lines
226
227 //dynamic morphology??
228 if(codeVersion == 1) {
229 int hori = original.width();
230 int vert = original.height();
231 //Find ratio between 100 and width and 100 and height
232 int divX = hori/10;
233 int divY = vert/10;
234 int sizeX = (hori/divX) * 10;
235 int sizeY = (vert/divY) * 10;
236
237 Mat test = original.clone();
238 showWaitDestroy("Original", test);
239
240 System.out.println("hori: " + hori + '\t' + "vert: " + vert);
241 System.out.println("sizeX: " + sizeX + '\t' + "sizeY: " + sizeY);
242
243 Mat kernelErode = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(sizeX, (sizeY/100)));
244 Imgproc.erode(test,test,kernelErode);
245 showWaitDestroy("01 Erode", test);
246
247 Mat kernelDialate = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(sizeX,(sizeY/10)));
248 Imgproc.dilate(test, test, kernelDialate);
249 showWaitDestroy("02 Dialate", test);
250
251 Mat kernelErodeAgain = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size((sizeX/10),(sizeY/5)));
252 Imgproc.erode(test,test,kernelErodeAgain);
253 showWaitDestroy(" 03 Erode Again", test);
254
255 Mat kernelClose = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size((sizeX/10)*3,(sizeY/10)*3));
256 Imgproc.morphologyEx(test,test,Imgproc.MORPH_CLOSE, kernelClose);
257 showWaitDestroy("04 Close", test);
258
259 Imgproc.adaptiveThreshold(test, test,255, Imgproc.ADAPTIVE_THRESH_GAUSSIAN_C,Imgproc.THRESH_BINARY_INV, 15, THRESHOLD_C);
260 showWaitDestroy("05 Binarized", test);
261
262 Mat kernelOpen = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size((sizeX/10),(sizeY/20)));
263 Imgproc.morphologyEx(test,test,Imgproc.MORPH_OPEN, kernelOpen);
264 showWaitDestroy(" 06 Open", test);
265
266 Mat kernelDialateAgain = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size((sizeX/5),(sizeY/100)));
267 Imgproc.dilate(test, test, kernelDialateAgain);
268 showWaitDestroy("07 Dialate", test);
269
270
271 Mat kernelCloseAgain = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size((sizeX/10),(sizeY/2)));
272 Imgproc.morphologyEx(test,test,Imgproc.MORPH_CLOSE, kernelCloseAgain);
273 showWaitDestroy(" 08 Close Again (Final)", test);
274 }
275 //Successful hardcode for morhpology
276 if (codeVersion == 2) {
277 Mat test = original.clone();
278 showWaitDestroy("00 Original", test);
279
280 Mat kernelErode = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(100,1));
281 Imgproc.erode(test,test,kernelErode);
282 showWaitDestroy("01 Erode", test);
283
284 Mat kernelDialate = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(100,10));
285 Imgproc.dilate(test, test, kernelDialate);
286 showWaitDestroy("02 Dialate", test);
287
288 Mat kernelErodeAgain = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(10,20));
289 Imgproc.erode(test,test,kernelErodeAgain);
290 showWaitDestroy(" 03 Erode Again", test);
291
292 Mat kernelClose = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(50,30));
293 Imgproc.morphologyEx(test,test,Imgproc.MORPH_CLOSE, kernelClose);
294 showWaitDestroy("04 Close", test);
295
296 Imgproc.adaptiveThreshold(test, test,255, Imgproc.ADAPTIVE_THRESH_GAUSSIAN_C,Imgproc.THRESH_BINARY_INV, 15, THRESHOLD_C);
297 showWaitDestroy("05 Binarized", test);
298
299 Mat kernelOpen = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(4,4));
300 Imgproc.morphologyEx(test,test,Imgproc.MORPH_OPEN, kernelOpen);
301 showWaitDestroy(" 06 Open", test);
302
303// Mat kernelDialateAgain = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(1,10));
304// Imgproc.dilate(test, test, kernelDialateAgain);
305// showWaitDestroy("07 Dialate", test);
306
307 //FIGURE OUT FLOOD FILL!!
308 Imgproc.floodFill(test,test, new Point(1,1), new Scalar(2));
309
310
311 Mat kernelCloseAgain = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(10,50));
312 Imgproc.morphologyEx(test,test,Imgproc.MORPH_CLOSE, kernelCloseAgain);
313 showWaitDestroy(" 08 Close Again (Final)", test);
314
315 }
316 //Tutorial/Demo Code
317 if (codeVersion == 3) {
318 Mat horizontal = original.clone();
319 Mat vertical = original.clone();
320 // Specify size on horizontal axis
321 int horizontal_size = horizontal.cols() / 50;
322 // Create structure element for extracting horizontal lines through morphology operations
323 Mat horizontalStructure = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(horizontal_size, 2));
324 // Apply morphology operations
325 Imgproc.erode(horizontal, horizontal, horizontalStructure);
326 Imgproc.dilate(horizontal, horizontal, horizontalStructure);
327 // Show extracted horizontal lines
328 showWaitDestroy("horizontal", horizontal);
329 // Specify size on vertical axis
330 int vertical_size = vertical.rows() / 30;
331 // Create structure element for extracting vertical lines through morphology operations
332 Mat verticalStructure = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(1, vertical_size));
333 // Apply morphology operations
334 Imgproc.erode(vertical, vertical, verticalStructure);
335 Imgproc.dilate(vertical, vertical, verticalStructure);
336 // Show extracted vertical lines
337 showWaitDestroy("vertical", vertical);
338 // Inverse vertical image
339 Core.bitwise_not(vertical, vertical);
340 showWaitDestroy("vertical_bit", vertical);
341 // Extract edges and smooth image according to the logic
342 // 1. extract edges
343 // 2. dilate(edges)
344 // 3. src.copyTo(smooth)
345 // 4. blur smooth img
346 // 5. smooth.copyTo(src, edges)
347 // Step 1
348 Mat edges = new Mat();
349 Imgproc.adaptiveThreshold(vertical, edges, 255, Imgproc.ADAPTIVE_THRESH_MEAN_C, Imgproc.THRESH_BINARY, 3, -2);
350 showWaitDestroy("edges", edges);
351 // Step 2
352 Mat kernel = Mat.ones(2, 2, CvType.CV_8UC1);
353 Imgproc.dilate(edges, edges, kernel);
354 showWaitDestroy("dilate", edges);
355 // Step 3
356 Mat smooth = new Mat();
357 vertical.copyTo(smooth);
358 // Step 4
359 Imgproc.blur(smooth, smooth, new Size(2, 2));
360 // Step 5
361 smooth.copyTo(vertical, edges);
362 // Show final result
363 showWaitDestroy("smooth - final", vertical);
364 System.exit(0);
365 }
366
367
368
369
370 //****************MORPHOLOGY****************************************************************************************
371
372 BufferedImage toBeClassifiedImg = toBufferedImage(edgesDetectedRGB);
373
374
375 //Display Results
376 //HighGui.imshow("Source", original);
377 //HighGui.imshow("Just Edges", justEdges); //TESTING
378
379
380 HighGui.imshow("LINESFOUND", edgesDetectedRGB);
381 HighGui.resizeWindow("LINESFOUND", 1000,1000);
382
383 //HighGui.imshow("CLUSTERS FOUND", clustersFoundRGB);
384 //HighGui.imshow("Detected Lines (in red) - negative", edgesDetectedRGBProb);
385
386 //COUNT OF LINES CLASSIFICATION
387 //System.out.println("LINE CLUSTER RESULT: " + ClassifierLineClusterOLD(toBeClassifiedImg).get(0) + '\t' + "LinesFound: " + ClassifierLineClusterOLD(toBeClassifiedImg).get(1) + '\t' + "ClustersFound: " + ClassifierLineClusterOLD(toBeClassifiedImg).get(2));
388 //System.out.println("NEW CLUSTER RESULTS: " + ClassifierLineClusterPt(pointArrayList,clustersFoundRGB).get(0) + '\t' + "LinesFound: " + horizontalLineCount + '\t' + "ClustersFound: " + ClassifierLineClusterPt(pointArrayList,clustersFoundRGB).get(1));
389 //System.out.println(ClassifierLineClusterPt(pointArrayList, clustersFoundRGB));
390
391 //System.out.println("TEST: " + LineCountOrCluster(horizontalLineCount, pointArrayList, clustersFoundRGB));
392
393 // Wait and Exit
394 HighGui.waitKey();
395 System.exit(0);
396 }
397 catch(Exception e){
398 System.err.println(e);
399 }
400 }
401}
Note: See TracBrowser for help on using the repository browser.