source: other-projects/maori-lang-detection/src/org/greenstone/atea/CountryCodeCountsMapData.java@ 33959

Last change on this file since 33959 was 33959, checked in by ak19, 4 years ago

URIEncoding the mapData makes it unparseable by geojson.io
File size: 29.4 KB
Line 
1package org.greenstone.atea;
2
3import java.io.BufferedReader;
4import java.io.BufferedWriter;
5import java.io.File;
6import java.io.FileReader;
7import java.io.FileWriter;
8import java.io.Writer;
9
10import java.net.URLEncoder;
11
12import java.util.HashMap;
13import java.util.LinkedList;
14import java.util.List;
15import java.util.Map;
16
17//import java.lang.Math; //automatically imported apparently
18
19import org.apache.commons.csv.*;
20import org.apache.log4j.Logger;
21
22// Google's gson imports for parsing any kind of json
23import com.google.gson.JsonArray;
24import com.google.gson.JsonElement;
25import com.google.gson.JsonObject;
26import com.google.gson.JsonParser;
27
28// For working with GeoJSON's Simple Features in Java
29import mil.nga.sf.geojson.Feature;
30import mil.nga.sf.geojson.FeatureCollection;
31import mil.nga.sf.geojson.FeatureConverter;
32import mil.nga.sf.geojson.Geometry;
33import mil.nga.sf.geojson.MultiPoint;
34import mil.nga.sf.geojson.Polygon;
35import mil.nga.sf.geojson.Position;
36
37
38import org.greenstone.util.SafeProcess;
39
40/**
41 * Run a mongodb query that produces counts per countrycode like in the following 2 examples:
42 *
43 * 1. count of country codes for all sites
44 * db.Websites.aggregate([
45 *
46 * { $unwind: "$geoLocationCountryCode" },
47 * {
48 * $group: {
49 * _id: "$geoLocationCountryCode",
50 * count: { $sum: 1 }
51 * }
52 * },
53 * { $sort : { count : -1} }
54 * ]);
55 *
56 * Then store the mongodb query result's JSON format output in a file called "counts.json".
57 * Then run this program with counts.json as parameter
58 * Copy the geojson output into http://geojson.tools/
59 *
60 * 2. count of country codes for sites that have at least one page detected as MRI
61 *
62 * db.Websites.aggregate([
63 * {
64 * $match: {
65 * numPagesInMRI: {$gt: 0}
66 * }
67 * },
68 * { $unwind: "$geoLocationCountryCode" },
69 * {
70 * $group: {
71 * _id: {$toLower: '$geoLocationCountryCode'},
72 * count: { $sum: 1 }
73 * }
74 * },
75 * { $sort : { count : -1} }
76 * ]);
77 *
78 * Store the mongodb query result's JSON format output in a file called "counts_sitesWithPagesInMRI.json".
79 * Then run this program with counts_sitesWithPagesInMRI.json as parameter.
80 * Copy the geojson output into http://geojson.tools/
81 *
82 * ##################
83 * TO COMPILE:
84 * maori-lang-detection/src$
85 * javac -cp ".:../conf:../lib/*" org/greenstone/atea/CountryCodeCountsMapData.java
86 *
87 * TO RUN:
88 * maori-lang-detection/src$
89 * java -cp ".:../conf:../lib/*" org/greenstone/atea/CountryCodeCountsMapData ../mongodb-data/counts.json
90 *###################
91 *
92 * This class needs the gson library, and now the sf-geojson(-2.02).jar and
93 * helper jars sf(-2.02).jar and 3 jackson jars too,
94 * to create and store Simple Features geo json objects with Java.
95 * I copied the gson jar file from GS3.
96 *
97 * Simple Features GeoJSON Java
98 * https://ngageoint.github.io/simple-features-geojson-java/ - liks to API and more
99 *
100 * https://mvnrepository.com/artifact/mil.nga.sf/sf-geojson (https://github.com/ngageoint/simple-features-geojson-java/)
101 *
102 * Also need the basic data types used by the Geometry objects above:
103 * https://mvnrepository.com/artifact/mil.nga/sf (https://github.com/ngageoint/simple-features-java)
104 *
105 * Further helper jars needed (because of encountering the exception documented at
106 * stackoverflow.com/questions/36278293/java-lang-classnotfoundexception-com-fasterxml-jackson-core-jsonprocessingexcep/36279872)
107 * https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-core/2.10.0
108 * https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-databind
109 * https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-annotations/2.10.0
110 */
111public class CountryCodeCountsMapData {
112
113 static Logger logger = Logger.getLogger(org.greenstone.atea.CountryCodeCountsMapData.class.getName());
114
115 static public final String GEOJSON_MAP_TOOL_URL = "http://geojson.io/"; //"http://geojson.tools/";
116 static private final String DATA_STR = "#data=data:application/json,";
117
118 /* "http://geojson.io" has a URL API to programmatically get it to generate
119 a map from a geojson URL parameter.
120
121 See http://geojson.io/ -> Help
122
123 "I'm a coder
124
125 geojson.io has an array of cli tools that make it easy to go from a GeoJSON file on your computer to geojson.io."
126
127 http://geojson.io/#geojson-io-api
128 "Geojson.io API
129
130 You can interact with geojson.io programmatically in two ways:
131
132 => URL parameters
133 Browser console"
134
135 http://geojson.io/#url-api
136 "data=data:application/json,
137
138 Open the map and load a chunk of GeoJSON data from a URL segment directly onto the map.
139 The GeoJSON data should be encoded as per encodeURIComponent(JSON.stringify(geojson_data)).
140 Example:
141
142 http://geojson.io/#data=data:application/json,%7B%22type%22%3A%22LineString%22%2C%22coordinates%22%3A%5B%5B0%2C0%5D%2C%5B10%2C10%5D%5D%7D
143
144 The above uses JavaScript's encodeURIComponent(). Although partly different from
145 URL.encode() in Java, it's not different in our case.
146 However, using URL.encode() does end up producing a string that is not recognised by
147 geojson.io's parser. Unencoded, the string is recognised fine.
148 */
149
150 public static final int SUPPRESS_MAPDATA_DISPLAY = 0;
151 public static final int PRINT_MAPDATA_TO_SCREEN = 1;
152
153 //Map<String, JsonObject> countryToJsonMap;
154 JsonArray countryCodesJsonArray;
155 JsonArray countryCountsJsonArray;
156
157 // North-central Antarctica coords
158 private final double ANTARCTICA_LNG = 57.0d;
159 private final double ANTARCTICA_LAT = -70.0d;
160 // For EU coords, spot in Atlantic Ocean close to western European coast.
161 private final double EU_LNG = -20.0d;
162 private final double EU_LAT = 50.0d;
163
164 private final String geoJsonFilenameWithSuffix;
165 private final File outputFolder;
166
167 /** The string version of the Feature geojson generated */
168 private String featuresGeojsonString;
169
170 // Used to create screenshots
171 private final String MONITOR_RESOLUTION = "1920,1080"; // format: "x,y"
172
173 public CountryCodeCountsMapData(String countryCountsJSONFilename) throws Exception {
174
175 // work out the unique filename we're going to save the geojson files under
176 // and the folder we're going to save them into
177 File countryCountsJSONFile = new File(countryCountsJSONFilename);
178 String tailname = countryCountsJSONFile.getName();
179 this.geoJsonFilenameWithSuffix = (tailname.startsWith("counts_")) ? tailname.substring("counts_".length()) : tailname;
180 this.outputFolder = countryCountsJSONFile.getParentFile().getCanonicalFile(); // canonical resolves any .. and . in path
181
182 // locate the countrycodes.json file
183 File countryCoordsJSONFile = new File(this.getClass().getClassLoader().getResource("countrycodes.json").getFile());
184
185 // Create a map of ALL country code names to ALL the country code json objects
186 // that contain the location (lat, lng) info for each country code
187 Map<String, JsonObject> countryToJsonMap = new HashMap<String, JsonObject>();
188
189 // Parse json file of country codes and put into a JsonArray.
190 // then put into map of each country code to its JsonObject.
191 countryCodesJsonArray = parseJSONFile(countryCoordsJSONFile);
192 for(JsonElement obj : countryCodesJsonArray) {
193 JsonObject countryCodeJson = obj.getAsJsonObject();
194 countryToJsonMap.put(countryCodeJson.get("country").getAsString(), countryCodeJson);
195 }
196
197 // Parse json file of country code counts
198 // Then for each JsonObject in this file,
199 // find a match on its country code in the map created above to get a country code JsonObject
200 // Get the longitude and latitude of the JsonObject that matched that country code.
201 // Add this lng,lat location information to the current JsonObject from the counts file.
202 countryCountsJsonArray = parseJSONFile(countryCountsJSONFile);
203
204 for(JsonElement obj : countryCountsJsonArray) {
205 JsonObject json = obj.getAsJsonObject();
206 String countryCode = json.get("_id").getAsString().toUpperCase();
207 // set the property back as uppercase and with property name "countrycode" instead of "_id"
208 json.remove("_id");
209 json.addProperty("countrycode", countryCode);
210
211 int count = (int)json.get("count").getAsDouble();
212
213 //logger.info("Got country code: " + countryCode);
214 //logger.info(" count: " + count);
215
216 // locate in countryCode map
217 JsonObject countryCodeJson = countryToJsonMap.get(countryCode);
218
219 if(countryCodeJson != null) {
220 //logger.info("Found in map: " + countryCodeJson.toString());
221
222 // for geojson, want longitude then latitude
223 Double lng = countryCodeJson.get("longitude").getAsDouble();
224 Double lat = countryCodeJson.get("latitude").getAsDouble();
225 //logger.info("long: " + Double.toString(lng) + ", lat: " + Double.toString(lat));
226 String countryName = countryCodeJson.get("name").getAsString();
227
228 // let's add lat and lng fields to countryCounts object
229 json.addProperty("lng", lng); // adds Number: https://javadoc.io/static/com.google.code.gson/gson/2.8.5/com/google/gson/JsonObject.html
230 json.addProperty("lat", lat);
231 json.addProperty("region", countryName);
232
233 } else {
234 logger.info("No geolocation info found for country code " + countryCode);
235 if(countryCode.equals("EU")) {
236 logger.info(" Adding lat,lng for somewhere around Europe");
237 //logger.info("Unlisted country code: EU");
238 // add lat and lng for Europe
239 json.addProperty("lng", EU_LNG);
240 json.addProperty("lat", EU_LAT);
241 json.addProperty("region", "Europe");
242 }
243 else if(countryCode.equals("UNKNOWN")) {
244 logger.info(" Adding lat,lng for somewhere in Antarctica");
245 //logger.info("Unlisted country code: UNKNOWN");
246 // add lat and lng for Antarctica
247 json.addProperty("lng", ANTARCTICA_LNG);
248 json.addProperty("lat", ANTARCTICA_LAT);
249 json.addProperty("region", "UNKNOWN");
250 } else {
251 logger.error("ERROR: entirely unknown country code: " + countryCode);
252 }
253 }
254 }
255
256 }
257
258 /** Convert mongodb tabular output of json records stored in the given file
259 * into a JsonArray.
260 */
261 public JsonArray parseJSONFile(File file) throws Exception {
262 JsonArray jsonArray = null;
263 // read into string
264 try (
265 BufferedReader reader = new BufferedReader(new FileReader(file));
266 ) {
267 StringBuilder str = //new StringBuilder();
268 new StringBuilder("[");
269 String line;
270
271 boolean multi_line_comment = false;
272
273 while((line = reader.readLine()) != null) {
274 line = line.trim();
275
276 // ignore any single line comments nested in multi-line symbols
277 if(line.startsWith("/*") && line.endsWith("*/")) {
278 continue; // skip line
279 }
280
281 // skip multi-line comments spread over multiple lines
282 // assumes this ends on a line containing */ without further content on the line.
283 if(line.startsWith("/*") && !line.endsWith("*/")) {
284 multi_line_comment = true;
285 continue; // skip line
286 }
287 if(multi_line_comment) {
288 if(line.contains("*/")) {
289 multi_line_comment = false;
290 }
291
292 continue; // we're in a comment or at end of comment, skip line
293 }
294
295 str.append(line);
296 if(line.endsWith("}")) {
297 str.append(",\n");
298 }
299 }
300 // replace last comma with closing bracket
301 String fileContents = str.substring(0, str.length()-2) + "]";
302
303 //logger.debug("Got file:\n" + fileContents);
304
305 // https://stackoverflow.com/questions/2591098/how-to-parse-json-in-java
306 jsonArray = new JsonParser().parse(fileContents).getAsJsonArray();
307
308 } catch(Exception e) {
309 throw e;
310 }
311
312
313 return jsonArray;
314 }
315
316 /**
317 * Reading
318 * https://www.here.xyz/api/concepts/geojsonbasics/
319 * https://ngageoint.github.io/simple-features-geojson-java/docs/api/
320 *
321 * https://stackoverflow.com/questions/55621480/cant-access-coordinates-member-of-geojson-feature-collection
322 *
323 * Downloaded geojson simple features' jar file from maven, but it didn't work:
324 * a more private version of MultiPoint.java is not included in the jar file (there's only
325 * mil.nga.sf.geojson.MultiPoint , whereas
326 * mil.nga.sf.MultiPoint is missing
327 *
328 * This seems to have gone wrong at
329 * https://github.com/ngageoint/simple-features-geojson-java/tree/master/src/main/java/mil/nga/sf
330 * but the one at
331 * https://github.com/ngageoint/simple-features-java/tree/master/src/main/java/mil/nga/sf
332 * has it. So I've been trying to build that, but don't have the correct version of maven.
333 */
334 public Geometry toMultiPointGeoJson() {
335
336 List<Position> points = new LinkedList<Position>();
337
338 for(JsonElement obj : this.countryCountsJsonArray) {
339 JsonObject json = obj.getAsJsonObject();
340 Double lng = json.get("lng").getAsDouble();
341 Double lat = json.get("lat").getAsDouble();
342
343 Position point = new Position(lng, lat);
344 points.add(point);
345 }
346
347 Geometry multiPoint = new MultiPoint(points);
348
349 return multiPoint;
350 }
351
352 // https://javadoc.io/static/com.google.code.gson/gson/2.8.5/index.html
353 public FeatureCollection toFeatureCollection() {
354 final int HISTOGRAM_WIDTH = 4;
355
356 FeatureCollection featureCollection = new FeatureCollection();
357
358 for(JsonElement obj : this.countryCountsJsonArray) {
359 JsonObject json = obj.getAsJsonObject();
360
361 String countryCode = json.get("countrycode").getAsString();
362 String region = json.get("region").getAsString();
363 int count = json.get("count").getAsInt();
364
365 // make a histogram for each country
366 Geometry rectangle = this.toPolygon(json, count, HISTOGRAM_WIDTH);
367
368 Feature countryFeature = new Feature(rectangle);
369 Map<String, Object> featureProperties = new HashMap<String, Object>();
370 featureProperties.put("count", new Integer(count));
371 featureProperties.put("code", countryCode);
372 featureProperties.put("region", region);
373 countryFeature.setProperties(featureProperties);
374
375 featureCollection.addFeature(countryFeature);
376 }
377
378 return featureCollection;
379 }
380
381 // create rectangular "histogram" for each country code
382 private Geometry toPolygon(JsonObject json, final int count, final int HISTOGRAM_WIDTH) {
383 int half_width = HISTOGRAM_WIDTH/2;
384 double vertical_factor = 1.0;
385
386 final Double lng = json.get("lng").getAsDouble();
387 final Double lat = json.get("lat").getAsDouble();
388
389 String countryCode = json.get("countrycode").getAsString();
390
391
392 //create the 4 corners of the rectangle
393 // West is negative, east is positive, south is negative, north is positive
394 // See http://www.learnz.org.nz/sites/learnz.org.nz/files/lat-long-geo-data-01_0.jpg
395 // But since the histograms grow vertically/northwards and we can't go past a latitude of 90,
396 // to compensate, we increase the width of the histograms by the same factor as our inability
397 // to grow northwards.
398 Double north = lat + (vertical_factor * count);
399
400 while (north > 90) {
401 // recalculate north after decreasing histogram's vertical growth
402 // by the same factor as we increase its width
403 vertical_factor = vertical_factor/2.0;
404 half_width = 2 * half_width;
405 north = lat + (vertical_factor * count);
406 }
407 Double east = lng + half_width;
408 Double west = lng - half_width;
409 Double south = lat;
410
411 List<Position> pts = recalculateAreaIfLarge(count, HISTOGRAM_WIDTH, countryCode, lat, lng, north, south, east, west);
412
413 /*
414 logger.debug("For country " + countryCode + ":");
415 logger.debug("north = " + north);
416 logger.debug("south = " + south);
417 logger.debug("east = " + east);
418 logger.debug("west = " + west + "\n");
419 logger.debug("-------------");
420 */
421
422 List<List<Position>> outerList = new LinkedList<List<Position>>();
423 if(pts != null) {
424 outerList.add(pts);
425 } else {
426
427
428 List<Position> points = new LinkedList<Position>();
429 outerList.add(points);
430
431 points.add(new Position(west, south)); // Position(lng, lat) not Position(lat, lng)
432 points.add(new Position(west, north));
433 points.add(new Position(east, north));
434 points.add(new Position(east, south));
435 }
436
437 Geometry rectangle = new Polygon(outerList);
438
439 // Coords: a List of List of Positions, see https://ngageoint.github.io/simple-features-geojson-java/docs/api/
440 // https://www.here.xyz/api/concepts/geojsonbasics/#polygon
441
442 return rectangle;
443 }
444
445 private List<Position> recalculateAreaIfLarge(final int count, final int HISTOGRAM_WIDTH, String countryCode,
446 final Double lat, final Double lng,
447 Double north, Double south, Double east, Double west) {
448 boolean recalculated = false;
449
450 // Check if we're dealing with very large numbers, in which case, we can have follow off the longitude edges
451 // Max longitude values are -180 to 180. So a max of 360 units between them. (Max latitude is -90 to 90)
452 // "Longitude is in the range -180 and +180 specifying coordinates west and east of the Prime Meridian, respectively.
453 // For reference, the Equator has a latitude of 0°, the North pole has a latitude of 90° north (written 90° N or +90°),
454 // and the South pole has a latitude of -90°."
455 if((east + Math.abs(west)) > 360 || east > 180 || west < -180) {
456 logger.debug("For country " + countryCode + ":");
457 logger.debug("north = " + north);
458 logger.debug("south = " + south);
459 logger.debug("east = " + east);
460 logger.debug("west = " + west + "\n");
461
462 int half_width = HISTOGRAM_WIDTH/2; // reset half_width
463
464 double v_tmp_count = Math.sqrt(count);
465 //double h_tmp_count = Math.floor(v_tmp_count);
466 //v_tmp_count = Math.ceil(v_tmp_count);
467 double h_tmp_count = v_tmp_count;
468 logger.debug("count = " + count);
469 logger.debug("v = " + v_tmp_count);
470 logger.debug("h = " + h_tmp_count);
471 logger.debug("lat = " + lat);
472 logger.debug("lng = " + lng + "\n");
473
474 if(h_tmp_count > 90) { // 360 max width, of which each longitude
475 // is 4 units (horizontal factor = 4, and half-width is half
476 // of that). So max width/h_tmp_count allowed 90 => 360
477 // longitude on map (-180 to 180).
478 // Put the excess h_tmp_count into v_tmp_count and ensure
479 // that does not go over 90+90 = 180 max. Vertical_factor is 1.
480
481 logger.debug("Out of longitude range. Attempting to compensate...");
482
483 double diff = h_tmp_count - 80.0; // actually 90 wraps on geojson tools, 80 doesn't
484 h_tmp_count -= diff;
485 v_tmp_count = (count/h_tmp_count);
486
487 if(v_tmp_count > 180 || h_tmp_count > 90) {
488 logger.warn("Warning: still exceeded max latitude and/or longitude range");
489 }
490
491 }
492
493 logger.debug("Recalculating polygon for country with high count: " + countryCode + ".");
494 logger.debug("count = " + count);
495 logger.debug("v = " + v_tmp_count);
496 logger.debug("h = " + h_tmp_count);
497 logger.debug("lat = " + lat);
498 logger.debug("lng = " + lng + "\n");
499
500
501 north = lat + v_tmp_count;
502 south = lat;
503 east = lng + (h_tmp_count * half_width); // a certain width, half_width, represents one unit in the x axis
504 west = lng - (h_tmp_count * half_width);
505
506 /*
507 logger.debug("north = " + north);
508 logger.debug("south = " + south);
509 logger.debug("east = " + east);
510 logger.debug("west = " + west + "\n");
511 */
512
513 if(north > 90) {
514 // centre vertically on lat
515 north = lat + (v_tmp_count/2);
516 south = lat - (v_tmp_count/2);
517 }
518
519 if(west < -180.0) {
520 double h_diff = -180.0 - west; // west is a larger negative value than -180, so subtracting west from -180 produces a positive h_diff value
521 west = -180.0; // set to extreme western edge
522 east = east + h_diff;
523 }
524 else if(east > 180.0) {
525 double h_diff = east - 180.0; // the country's longitude (lng) is h_diff from the eastern edge
526 east = 180.0; // maximise eastern edge
527 west = west - h_diff; // then grow the remainder of h_tmp_count in the opposite (western/negative) direction
528 }
529
530 // NOTE: Can't centre on country, (lat,lng), as we don't know whether either of lat or lng has gone past the edge
531
532 // Hopefully we don't exceed +90/-90 lat and +/-180 longitude
533
534 recalculated = true;
535
536
537 } else if(west < -140.0) {
538 // past -140 west, the edges don't wrap well in geotools, so shift any points more west/negative than -140:
539
540 double diff = -140.0 - west;
541 west = -140.0;
542 east += diff;
543
544 recalculated = true;
545 }
546
547 if(recalculated) {
548 logger.debug("\nnorth = " + north);
549 logger.debug("south = " + south);
550 logger.debug("east = " + east);
551 logger.debug("west = " + west);
552
553
554 List<Position> points = new LinkedList<Position>();
555
556 points.add(new Position(west, south)); // Position(lng, lat) not Position(lat, lng)
557 points.add(new Position(west, north));
558 points.add(new Position(east, north));
559 points.add(new Position(east, south));
560
561 return points;
562 }
563
564 return null;
565 }
566
567 // by default, display mapdata output on screen too
568 public String writeMultiPointGeoJsonToFile() {
569 return writeMultiPointGeoJsonToFile(PRINT_MAPDATA_TO_SCREEN);
570 }
571 public String writeMultiPointGeoJsonToFile(int displayMapData) {
572 final String filename = "multipoint_" + this.geoJsonFilenameWithSuffix;
573 File outFile = new File(this.outputFolder, filename);
574
575 Geometry geometry = this.toMultiPointGeoJson();
576 String multiPointGeojsonString = FeatureConverter.toStringValue(geometry);
577 if(displayMapData == PRINT_MAPDATA_TO_SCREEN) {
578 logger.info("\nMap data as MultiPoint geometry:\n" + multiPointGeojsonString + "\n");
579 }
580 try (
581 Writer writer = new BufferedWriter(new FileWriter(outFile));
582 ) {
583
584 // Some basic re-formatting for some immediate legibility
585 // But pasting the contents of the file (or the System.err/logger.info output above)
586 // directly into http://geojson.tools/ or http://geojson.io/
587 // will instantly reformat the json perfectly anyway.
588 multiPointGeojsonString = multiPointGeojsonString.replace("[[", "\n[\n\t[");
589 multiPointGeojsonString = multiPointGeojsonString.replace("],[", "],\n\t[");
590 multiPointGeojsonString = multiPointGeojsonString.replace("]]", "]\n]");
591
592 writer.write(multiPointGeojsonString + "\n");
593 } catch(Exception e) {
594 logger.error("Unable to write multipoint geojson:\n**********************");
595 logger.error(multiPointGeojsonString);
596 logger.error("**********************\ninto file " + outFile.getAbsolutePath());
597 logger.error(e.getMessage(), e);
598 }
599
600 return outFile.getAbsolutePath();
601
602 }
603
604 // by default, display mapdata output on screen too
605 public String writeFeaturesGeoJsonToFile() {
606 return writeFeaturesGeoJsonToFile(PRINT_MAPDATA_TO_SCREEN);
607 }
608 // write out geojson features to appropriately named file
609 // If displayMapData == PRINT_MAPDATA_TO_SCREEN, then it will also be printed to screen
610 public String writeFeaturesGeoJsonToFile(int displayMapData) {
611 final String filename = "geojson-features_" + this.geoJsonFilenameWithSuffix;
612 File outFile = new File(this.outputFolder, filename);
613
614 FeatureCollection featureColl = this.toFeatureCollection();
615 this.featuresGeojsonString = FeatureConverter.toStringValue(featureColl);
616 if(displayMapData == PRINT_MAPDATA_TO_SCREEN) {
617 logger.info("\nMap data as featurecollection:\n" + featuresGeojsonString + "\n");
618 }
619 try (
620 Writer writer = new BufferedWriter(new FileWriter(outFile));
621 ) {
622
623 writer.write(featuresGeojsonString + "\n");
624 } catch(Exception e) {
625 logger.error("Unable to write multipoint geojson:\n**********************");
626 logger.error(featuresGeojsonString);
627 logger.error("**********************\ninto file " + outFile.getAbsolutePath());
628 logger.error(e.getMessage(), e);
629 }
630
631 return outFile.getAbsolutePath();
632 }
633
634
635 public String getFeaturesGeoJsonString(boolean uriEncoded) {
636 if(this.featuresGeojsonString == null) {
637 this.featuresGeojsonString = FeatureConverter.toStringValue(this.toFeatureCollection());
638 }
639 if(uriEncoded) {
640 // Want to return encodeURIComponent(JSON.stringify(featuresGeojsonString));
641 // https://stackoverflow.com/questions/607176/java-equivalent-to-javascripts-encodeuricomponent-that-produces-identical-outpu
642 featuresGeojsonString = URLEncoder.encode(featuresGeojsonString);
643 }
644 return featuresGeojsonString;
645 }
646
647 public String getAsMapURL() {
648 // geojson.io advises to call JavaScript's encodeURIComponent() on the string
649 // which for the geojson strings we're dealing with is the same as Java's URI.encode().
650 // But uriEncoding our geojson produces a string unparseable by geojson.io,
651 // whereas the unencoded string is recognised. Not sure why. Turning off uriEncoding.
652 boolean uriEncoded = false;
653 String url = GEOJSON_MAP_TOOL_URL + DATA_STR + getFeaturesGeoJsonString(uriEncoded);
654
655 return url;
656 }
657
658 public String geoJsonMapScreenshot(File outputFolder, String fileNamePrefix) {
659 // https://stackoverflow.com/questions/49606051/how-to-take-a-screenshot-in-firefox-headless-selenium-in-java
660
661
662 String mapURL = this.getAsMapURL();
663 ///String mapURLescapedForBash = .replace("\"", "\\\"");//.replace("[", "\\[").replace("]", "\\]");
664
665 // Testing the geojson.io URL with a simpler polygon. Also doesn't work.
666 //String mapURL = GEOJSON_MAP_TOOL_URL + DATA_STR + URLEncoder.encode("{\"type\":\"Polygon\",\"coordinates\":[[[-77,37],[-77,38],[-76,38],[-76,37],[-77,37]]]}");
667
668
669 File outputFile = new File(outputFolder + File.separator + fileNamePrefix+".png");
670 String outputFilePath = Utility.getFilePath(outputFile);
671
672
673 // https://developer.mozilla.org/en-US/docs/Mozilla/Firefox/Headless_mode
674 // /path/to/firefox -P my-profile --screenshot test.jpg https://developer.mozilla.org --window-size=800,1000
675 // https://stackoverflow.com/questions/15783701/which-characters-need-to-be-escaped-when-using-bash
676 // FIREFOX doesn't work as expected: screenshot appears to happen too fast, before json completely loaded
677
678 /*
679 String[] cmdArgs = {
680 "firefox",
681 "--screenshot",
682 outputFilePath,
683 //"--delay=5", // only in firefox console?
684 "--window-size="+MONITOR_RESOLUTION,
685 mapURL //mapURLescapedForBash //"'" + mapURL + "'"
686 };
687 */
688
689 // Trying CHROME - background world map is faded at times or else blocky, page not loaded fully?
690 //https://www.bleepingcomputer.com/news/software/chrome-and-firefox-can-take-screenshots-of-sites-from-the-command-line/
691 // https://bitsofco.de/using-a-headless-browser-to-capture-page-screenshots/
692 // https://developers.google.com/web/updates/2017/04/headless-chrome
693
694
695 String[] cmdArgs = {
696 "/opt/google/chrome/chrome",
697 "--headless",
698 "--screenshot="+outputFilePath, //"--screenshot=\""+outputFilePath+"\"",
699 "--window-size="+MONITOR_RESOLUTION, //"--window-size=1024,1024",
700 mapURL
701 };
702
703
704 /*
705 //WORKS TO RUN FIREFOX TO OPEN A NEW TAB WITH MAPDATA:
706 // https://askubuntu.com/questions/57163/how-to-open-an-url-from-command-line-to-firefox-and-switch-to-tab
707 String[] cmdArgs = {
708 "firefox",
709 "-new-tab",
710 mapURL
711 };
712 */
713
714 /* Can try CutyCapt, http://cutycapt.sourceforge.net/
715 mentioned at https://bugzilla.mozilla.org/show_bug.cgi?id=1412061
716 It requires Qt 4.4.0+.
717 It's installed, https://askubuntu.com/questions/435564/qt5-installation-and-path-configuration
718 Versions 4 and 5 are installed, seen if I run
719 qtchooser -list-versions
720 And when I run: whereis qt5
721 qt5: /usr/share/qt5
722 https://unix.stackexchange.com/questions/116254/how-do-i-change-which-version-of-qt-is-used-for-qmake
723
724 No, doesn't appear to be installed.
725 */
726
727 System.err.println();
728 System.err.print("Running:");
729 for(String arg : cmdArgs) {
730 System.err.print(" " + arg);
731 }
732 System.err.println();
733
734
735 //String cmdArgs = "firefox --screenshot " + outputFilePath + " " + GEOJSON_MAP_TOOL_URL + DATA_STR;
736 //String cmdArgs = "firefox --screenshot " + outputFilePath + " " + "'" + mapURL + "'";
737 //System.err.println("Running: " + cmdArgs);
738
739 System.err.println();
740
741 SafeProcess proc = new SafeProcess(cmdArgs);
742
743 int retVal = proc.runProcess();
744
745 logger.info("Process out: " + proc.getStdOutput());
746 logger.info("Process err: " + proc.getStdError());
747 logger.info("Screenshot process returned with: " + retVal);
748
749 return outputFilePath;
750
751 }
752
753 public int getTotalCount() {
754 int total = 0;
755 for(JsonElement obj : this.countryCountsJsonArray) {
756 JsonObject json = obj.getAsJsonObject();
757 int count = json.get("count").getAsInt();
758 total += count;
759 }
760 return total;
761 }
762
763
764 // Unfinished and unused
765 public void parseCSVFile(String filename) throws Exception {
766 File csvData = new File(filename);
767 CSVParser parser = CSVParser.parse(csvData, java.nio.charset.Charset.forName("US-ASCII"), CSVFormat.RFC4180);
768 for (CSVRecord csvRecord : parser) {
769 logger.info("Got record: " + csvRecord.toString());
770 }
771 }
772
773 public static void printUsage() {
774 System.err.println("CountryCodeCountsMapData <counts-by-countrycode-file>.json");
775 }
776
777 public static void main(String args[]) {
778 if(args.length != 1) {
779 printUsage();
780 System.exit(-1);
781 }
782
783 try {
784 File countsFile = new File(args[0]);
785 if(!countsFile.exists()) {
786 logger.error("File " + countsFile + " does not exist");
787 System.exit(-1);
788 }
789
790 CountryCodeCountsMapData mapData = new CountryCodeCountsMapData(args[0]);
791
792 String multipointOutFileName = mapData.writeMultiPointGeoJsonToFile();
793 String featuresOutFileName = mapData.writeFeaturesGeoJsonToFile();
794
795 logger.info("***********\nWrote mapdata to files " + multipointOutFileName
796 + " and " + featuresOutFileName);
797 logger.info("You can paste the geojson contents of either of these files into "
798 + " the editor at " + GEOJSON_MAP_TOOL_URL
799 + " to see the data arranged on a world map");
800
801 logger.info("Total count for query: " + mapData.getTotalCount());
802
803 } catch(Exception e) {
804 logger.error(e.getMessage(), e);
805 }
806 }
807}
Note: See TracBrowser for help on using the repository browser.