Index: /other-projects/is-sheet-music-encore/trunk/COMPX520-RUN-META.sh
===================================================================
--- /other-projects/is-sheet-music-encore/trunk/COMPX520-RUN-META.sh	(revision 33043)
+++ /other-projects/is-sheet-music-encore/trunk/COMPX520-RUN-META.sh	(revision 33044)
@@ -7,6 +7,7 @@
 
 doc_id=$1
+doc_id_file=`echo $doc_id | sed 's/:/+/' | sed 's/\//=/g'`
 
-output_file="java-gen-corpus/download-meta/$doc_id-META.txt"
+output_file="java-gen-corpus/download-meta/$doc_id_file-META.txt"
 echo "Retrieving doc-id-page: $doc_id -> $output_file"
 echo ""
Index: /other-projects/is-sheet-music-encore/trunk/COMPX520-RUN-PNG.sh
===================================================================
--- /other-projects/is-sheet-music-encore/trunk/COMPX520-RUN-PNG.sh	(revision 33043)
+++ /other-projects/is-sheet-music-encore/trunk/COMPX520-RUN-PNG.sh	(revision 33044)
@@ -9,5 +9,8 @@
 page_num=$2
 
-output_file="download-images/$doc_id-$page_num.png"
+doc_id_file=`echo $doc_id | sed 's/:/+/' | sed 's/\//=/g'`
+
+
+output_file="download-images/$doc_id_file-$page_num.png"
 echo "Retrieving doc-id-page: $doc_id-$page_num -> $output_file"
 echo ""
Index: /other-projects/is-sheet-music-encore/trunk/Makefile
===================================================================
--- /other-projects/is-sheet-music-encore/trunk/Makefile	(revision 33043)
+++ /other-projects/is-sheet-music-encore/trunk/Makefile	(revision 33044)
@@ -3,10 +3,16 @@
 
 filter-full-50-MU:
-	#cd java-gen-corpus; java TabProcTextGen hathiFull.txt hathiFullIDList-MU.txt MU
-	#./COMPX520-DOWNLOADER-META.sh java-gen-corpus/hathiFullIDList-MU.txt
+	#source SETUP.bash MUST TO THIS MANUALLY
+	cd java-gen-corpus; java TabProcTextGen hathiFull.txt hathiFullIDList-MU.txt MU
+	./COMPX520-DOWNLOADER-META.sh java-gen-corpus/hathiFullIDList-MU.txt
 	cd java-gen-corpus; ./metadata-formater.sh
 	cd java-gen-corpus; ./ValidIDListGen.sh MU
 	cd java-gen-corpus; java TabRndListGen hathiValidIDList-MU.txt 50 hathiValidRnd50List-MU.txt
 
+test:
+	cd java-gen-corpus; javac *.java
+	cd java-gen-corpus; java TabProcTextGen hathiFull.txt _test_hathiFullIDList-MU.txt MU
+	cd java-gen-corpus; java TabRndListGen _test_hathiFullIDList-MU.txt 50 _test_hathiValidIDList-MU-50.txt
 
 
+
Index: /other-projects/is-sheet-music-encore/trunk/dapiclient2-extended-META.pl
===================================================================
--- /other-projects/is-sheet-music-encore/trunk/dapiclient2-extended-META.pl	(revision 33043)
+++ /other-projects/is-sheet-music-encore/trunk/dapiclient2-extended-META.pl	(revision 33044)
@@ -153,6 +153,13 @@
     print $response->content;
 }
-exit $success;
+
+if ($success) {
+    exit 0;
+}
+else {
+    exit 1;
+}
 
 
 
+
Index: /other-projects/is-sheet-music-encore/trunk/java-gen-corpus/TabProcTextGen.java
===================================================================
--- /other-projects/is-sheet-music-encore/trunk/java-gen-corpus/TabProcTextGen.java	(revision 33043)
+++ /other-projects/is-sheet-music-encore/trunk/java-gen-corpus/TabProcTextGen.java	(revision 33044)
@@ -47,6 +47,9 @@
 					item[19].equals(inputType) && 
 					(item[24].equals("open")||item[24].equals("page"))){
+						//Encode id
+						String idEncoded = item[0];
+						//idEncoded = idEncoded.replaceAll(":", "+").replaceAll("/", "=");
 						//Write item to file
-						fw.write(item[0] + "\n");
+						fw.write(idEncoded + "\n");
 					}
 				}		
@@ -63,2 +66,3 @@
 //https://www.javatpoint.com/java-filewriter-class
 //https://docs.oracle.com/javase/8/docs/api/index.html?java/io/FileWriter.html
+//Email from supervisor (Compx520 ID Error Apr 24)
Index: /other-projects/is-sheet-music-encore/trunk/java-gen-corpus/TabRndListGen.java
===================================================================
--- /other-projects/is-sheet-music-encore/trunk/java-gen-corpus/TabRndListGen.java	(revision 33043)
+++ /other-projects/is-sheet-music-encore/trunk/java-gen-corpus/TabRndListGen.java	(revision 33044)
@@ -6,4 +6,5 @@
 import java.io.FileReader;
 import java.io.FileWriter;
+import java.io.*;
 import java.util.*;
 
@@ -25,5 +26,5 @@
 
 		ArrayList<String> list = new ArrayList<String>();
-		ArrayList<String> subList;
+	
 		String line = null;
 		String[] item;
@@ -38,7 +39,39 @@
 		//randomize list
 		Collections.shuffle(list);
-
-		//Take <outputCount>
-		subList = new ArrayList<String>(list.subList(0,outputCount));
+		
+		int countMatchingCrit = 0;
+		int j = 0;
+	    ArrayList<String> subList = new ArrayList<String>();
+		//Download meta record, check if numpages >=10 
+		while(countMatchingCrit < outputCount){
+			System.out.println("Processing item: " + j);
+			String idCurr = list.get(j);
+			//run download script
+			String cmd = "./download_metadata_temp.sh " + idCurr;
+			Process p = Runtime.getRuntime().exec(cmd);
+			/*BufferedReader br = new BufferedReader(new InputStreamReader(p.getInputStream()));
+                while(br.ready())
+                {
+                        System.out.println(br.readLine());
+                }*/
+			p.waitFor();
+			int exitStatus = p.exitValue();
+			if(exitStatus == 0){
+				//Extract numpages 
+				int numpages = getNumPages(idCurr);
+				if(numpages >= 10){
+					System.out.println("Successful items: " + countMatchingCrit);
+					countMatchingCrit++;
+					subList.add(idCurr);						
+				}
+			}
+			else{
+				System.err.println("failed to run:" + cmd);
+				System.exit(exitStatus);
+			}
+			j++;
+			if(j >= list.size()){break;}
+		}
+		
 
 		//Write these to new file
@@ -54,12 +87,48 @@
 	}
     }
+    private static int getNumPages(String id){
+    	String numpages = null;
+		try{
+				//Variables
+				String inputFilename = "metadata_temp.xml";
+				FileReader fileReader = new FileReader(inputFilename);
+				BufferedReader buf = new BufferedReader(fileReader);
+				String line = null;
+				String[] item;
+
+				//Splits into each record
+				while ((line = buf.readLine()) != null) {					
+					/*
+					if(line.contains("<id>")){
+						//Isoclate and store the id from the line
+						idLine = line.substring(line.indexOf(">")+1, line.indexOf("</"));
+						id  = idLine.substring(idLine.lastIndexOf("meta/")+5);
+						
+					}
+					*/	
+					if(line.contains("<htd:numpages>")){
+						//Iscolate and store the page number
+						numpages = line.substring(line.indexOf(">")+1, line.lastIndexOf("<"));
+										
+					}											
+				}
+				buf.close();
+				//Check if there are more than 10 pages
+	
+		}catch(Exception e){
+			e.printStackTrace();
+		}
+		return Integer.parseInt(numpages);
+    }
 }
 // Returns 42474
 
 //REFERNECES
+//https://stackoverflow.com/questions/12892665/how-to-capture-the-exit-status-of-a-shell-command-in-java
 //https://www.javatpoint.com/java-filewriter-class
 //https://docs.oracle.com/javase/8/docs/api/index.html?java/io/FileWriter.html
 //https://www.geeksforgeeks.org/randomly-select-items-from-a-list-in-java/
 //https://codereview.stackexchange.com/questions/146551/picking-10-distinct-words-randomly-from-list-of-unique-words
+//http://www.linuxforums.org/forum/programming-scripting/65117-c-c-system-function-analog-java.html
 
 //USE RUN-LIST.txt (modify file it reads using $1 (terminal entry variable (filename))
Index: /other-projects/is-sheet-music-encore/trunk/java-gen-corpus/download_metadata_temp.sh
===================================================================
--- /other-projects/is-sheet-music-encore/trunk/java-gen-corpus/download_metadata_temp.sh	(revision 33044)
+++ /other-projects/is-sheet-music-encore/trunk/java-gen-corpus/download_metadata_temp.sh	(revision 33044)
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+if [ $# != 1 ] ; then 
+  echo "Usage: ./download_metadata_temp.sh doc_id" 1>&2
+  exit 1
+fi
+
+doc_id=$1
+
+output_file="metadata_temp.xml"
+output_cmp_file="compact_metadata_temp.xml"
+echo "Retrieving metadata: $doc_id -> $output_file"
+echo ""
+
+../dapiclient2-extended-META.pl "$doc_id" > "$output_cmp_file"
+xmllint --format $output_cmp_file > $output_file
+
