source: other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PagedJSON.java@ 30933

package org.hathitrust;

import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URI;
import java.util.ArrayList;
import java.util.Iterator;

import org.apache.commons.compress.compressors.CompressorException;
import org.apache.commons.compress.compressors.CompressorInputStream;
import org.apache.commons.compress.compressors.CompressorStreamFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.json.JSONArray;
import org.json.JSONObject;

/*
class PagedJSON implements Function<String, Boolean> {

    private static final long serialVersionUID = 1L;

    public Boolean call(String s) { return s.contains("a"); }
}
 */


class PagedJSON implements FlatMapFunction<String, String> 
{
    private static final long serialVersionUID = 1L;

    protected String _input_dir;
    
    public PagedJSON(String input_dir)
    {
        _input_dir = input_dir;
    }
    
    protected static BufferedInputStream getBufferedInputStream(String fileIn) 
            throws IOException 
    {
        BufferedInputStream bis = null;
        
        if (fileIn.startsWith("hdfs://")) {
            URI uri = URI.create (fileIn);
            Configuration conf = new Configuration();
            FileSystem file = FileSystem.get(uri, conf);
            FSDataInputStream fin = file.open(new Path(uri));

            bis = new BufferedInputStream(fin);
        }
        else {
            // Trim 'file://' off the front
            
            String local_file_in = fileIn;
            if (local_file_in.startsWith("file://")) {
                local_file_in = fileIn.substring("file://".length());
            }
            FileInputStream fin = new FileInputStream(local_file_in);
            bis = new BufferedInputStream(fin);
        }

        return bis;
        
    }
    protected static BufferedReader getBufferedReaderForCompressedFile(String fileIn) 
            throws IOException, CompressorException 
    {
        BufferedInputStream bis = getBufferedInputStream(fileIn);
        CompressorInputStream comp_input = new CompressorStreamFactory().createCompressorInputStream(bis);
        BufferedReader br = new BufferedReader(new InputStreamReader(comp_input,"UTF8"));
        return br;
    }
    
    protected JSONObject readJSONFile(String filename)
    {
        //Path path = Paths.get(filename);

        StringBuilder sb = new StringBuilder();

        try {

            String str;
            BufferedReader br = getBufferedReaderForCompressedFile(_input_dir + "/" + filename);
            while ((str = br.readLine()) != null) {
                sb.append(str);
                //System.out.println(str);
            }

            br.close();
            
            //System.err.println("*****" + sb.toString());
                   
            /*
            List<String> lines = Files.readAllLines(path,StandardCharsets.UTF_8);


            for (String line : lines) {
                sb.append(line);

            }
            */
            
        } 
        catch (Exception e) {
            e.printStackTrace();
        }

        JSONObject json_obj = new JSONObject(sb.toString());
        
        
        return json_obj;
        
        //return sb.toString();
    }

    public Iterator<String> call(String s) 
    { 
        JSONObject extracted_feature_record = readJSONFile(s);
        
        String id = extracted_feature_record.getString("id");
        
        JSONObject ef_metadata = extracted_feature_record.getJSONObject("metadata");
        JSONObject ef_features = extracted_feature_record.getJSONObject("features");
        
        
        int ef_page_count = ef_features.getInt("pageCount");
        
        JSONArray ef_pages = ef_features.getJSONArray("pages");
        int ef_num_pages = ef_pages.length();
        
        ArrayList<String> ids = new ArrayList<String>(ef_num_pages);
        for (int i = 0; i < ef_page_count; i++) {
            ids.add(id + "." + i);
        }
        
        /*
        for (int i = 0; i < ef_num_pages; i++)
        {
            //String post_id = ef_pages.getJSONObject(i).getString("post_id");
            //......
        }
        */
        //String pageName = json_obj.getJSONObject("pageInfo").getString("pageName");
/*
        JSONArray arr = obj.getJSONArray("posts");
        for (int i = 0; i < arr.length(); i++)
        {
            String post_id = arr.getJSONObject(i).getString("post_id");
            ......
        }
*/
        
        
        ids.add(id);
        
        return ids.iterator();
    }
}