source: other-projects/maori-lang-detection/src/org/greenstone/atea/CountryCodeCountsMapData.java@ 34000

Last change on this file since 34000 was 34000, checked in by ak19, 4 years ago

Some debugging and other minor changes
File size: 32.5 KB
Line 
1package org.greenstone.atea;
2
3import java.io.BufferedReader;
4import java.io.BufferedWriter;
5import java.io.File;
6import java.io.FileReader;
7import java.io.FileWriter;
8import java.io.Writer;
9
10import java.net.URLEncoder;
11
12import java.util.HashMap;
13import java.util.LinkedList;
14import java.util.List;
15import java.util.Map;
16
17//import java.lang.Math; //automatically imported apparently
18
19import org.apache.commons.csv.*;
20import org.apache.log4j.Logger;
21
22// Google's gson imports for parsing any kind of json
23import com.google.gson.JsonArray;
24import com.google.gson.JsonElement;
25import com.google.gson.JsonObject;
26import com.google.gson.JsonParser;
27
28// For working with GeoJSON's Simple Features in Java
29import mil.nga.sf.geojson.Feature;
30import mil.nga.sf.geojson.FeatureCollection;
31import mil.nga.sf.geojson.FeatureConverter;
32import mil.nga.sf.geojson.Geometry;
33import mil.nga.sf.geojson.MultiPoint;
34import mil.nga.sf.geojson.Polygon;
35import mil.nga.sf.geojson.Position;
36
37
38import org.greenstone.util.SafeProcess;
39
40// copying to clipboard
41import java.awt.Toolkit;
42import java.awt.datatransfer.StringSelection;
43import java.awt.datatransfer.Clipboard;
44
45
46/**
47 * Run a mongodb query that produces counts per countrycode like in the following 2 examples:
48 *
49 * 1. count of country codes for all sites
50 * db.Websites.aggregate([
51 *
52 * { $unwind: "$geoLocationCountryCode" },
53 * {
54 * $group: {
55 * _id: "$geoLocationCountryCode",
56 * count: { $sum: 1 }
57 * }
58 * },
59 * { $sort : { count : -1} }
60 * ]);
61 *
62 * Then store the mongodb query result's JSON format output in a file called "counts.json".
63 * Then run this program with counts.json as parameter
64 * Copy the geojson output into http://geojson.tools/
65 *
66 * 2. count of country codes for sites that have at least one page detected as MRI
67 *
68 * db.Websites.aggregate([
69 * {
70 * $match: {
71 * numPagesInMRI: {$gt: 0}
72 * }
73 * },
74 * { $unwind: "$geoLocationCountryCode" },
75 * {
76 * $group: {
77 * _id: {$toLower: '$geoLocationCountryCode'},
78 * count: { $sum: 1 }
79 * }
80 * },
81 * { $sort : { count : -1} }
82 * ]);
83 *
84 * Store the mongodb query result's JSON format output in a file called "counts_sitesWithPagesInMRI.json".
85 * Then run this program with counts_sitesWithPagesInMRI.json as parameter.
86 * Copy the geojson output into http://geojson.tools/
87 *
88 * ##################
89 * TO COMPILE:
90 * maori-lang-detection/src$
91 * javac -cp ".:../conf:../lib/*" org/greenstone/atea/CountryCodeCountsMapData.java
92 *
93 * TO RUN:
94 * maori-lang-detection/src$
95 * java -cp ".:../conf:../lib/*" org/greenstone/atea/CountryCodeCountsMapData ../mongodb-data/counts.json
96 *###################
97 *
98 * This class needs the gson library, and now the sf-geojson(-2.02).jar and
99 * helper jars sf(-2.02).jar and 3 jackson jars too,
100 * to create and store Simple Features geo json objects with Java.
101 * I copied the gson jar file from GS3.
102 *
103 * Simple Features GeoJSON Java
104 * https://ngageoint.github.io/simple-features-geojson-java/ - liks to API and more
105 *
106 * https://mvnrepository.com/artifact/mil.nga.sf/sf-geojson (https://github.com/ngageoint/simple-features-geojson-java/)
107 *
108 * Also need the basic data types used by the Geometry objects above:
109 * https://mvnrepository.com/artifact/mil.nga/sf (https://github.com/ngageoint/simple-features-java)
110 *
111 * Further helper jars needed (because of encountering the exception documented at
112 * stackoverflow.com/questions/36278293/java-lang-classnotfoundexception-com-fasterxml-jackson-core-jsonprocessingexcep/36279872)
113 * https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-core/2.10.0
114 * https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-databind
115 * https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-annotations/2.10.0
116 */
117public class CountryCodeCountsMapData {
118
119 static Logger logger = Logger.getLogger(org.greenstone.atea.CountryCodeCountsMapData.class.getName());
120
121 static public final String GEOJSON_MAP_TOOL_URL = "http://geojson.io/"; //"http://geojson.tools/";
122 static private final String DATA_STR = "#data=data:application/json,";
123
124 /* "http://geojson.io" has a URL API to programmatically get it to generate
125 a map from a geojson URL parameter.
126
127 See http://geojson.io/ -> Help
128
129 "I'm a coder
130
131 geojson.io has an array of cli tools that make it easy to go from a GeoJSON file on your computer to geojson.io."
132
133 http://geojson.io/#geojson-io-api
134 "Geojson.io API
135
136 You can interact with geojson.io programmatically in two ways:
137
138 => URL parameters
139 Browser console"
140
141 http://geojson.io/#url-api
142 "data=data:application/json,
143
144 Open the map and load a chunk of GeoJSON data from a URL segment directly onto the map.
145 The GeoJSON data should be encoded as per encodeURIComponent(JSON.stringify(geojson_data)).
146 Example:
147
148 http://geojson.io/#data=data:application/json,%7B%22type%22%3A%22LineString%22%2C%22coordinates%22%3A%5B%5B0%2C0%5D%2C%5B10%2C10%5D%5D%7D
149
150 The above uses JavaScript's encodeURIComponent(). Although partly different from
151 URL.encode() in Java, it's not different in our case.
152 However, using URL.encode() does end up producing a string that is not recognised by
153 geojson.io's parser. Unencoded, the string is recognised fine.
154 */
155
156 public static final int SUPPRESS_MAPDATA_DISPLAY = 0;
157 public static final int PRINT_MAPDATA_TO_SCREEN = 1;
158
159 //Map<String, JsonObject> countryToJsonMap;
160 JsonArray countryCodesJsonArray;
161 JsonArray countryCountsJsonArray;
162
163 // North-central Antarctica coords
164 private final double ANTARCTICA_LNG = 57.0d;
165 private final double ANTARCTICA_LAT = -70.0d;
166 // For EU coords, spot in Atlantic Ocean close to western European coast.
167 private final double EU_LNG = -20.0d;
168 private final double EU_LAT = 50.0d;
169
170 private final String geoJsonFilenameWithSuffix;
171 private final File outputFolder;
172
173 /** The string version of the Feature geojson generated */
174 private String featuresGeojsonString;
175
176 // Used to create screenshots
177 private final String MONITOR_RESOLUTION = "1920,1080"; // format: "x,y"
178
179 public CountryCodeCountsMapData(String countryCountsJSONFilename) throws Exception {
180
181 // work out the unique filename we're going to save the geojson files under
182 // and the folder we're going to save them into
183 File countryCountsJSONFile = new File(countryCountsJSONFilename);
184 String tailname = countryCountsJSONFile.getName();
185 this.geoJsonFilenameWithSuffix = (tailname.startsWith("counts_")) ? tailname.substring("counts_".length()) : tailname;
186 this.outputFolder = countryCountsJSONFile.getParentFile().getCanonicalFile(); // canonical resolves any .. and . in path
187
188 // locate the countrycodes.json file
189 File countryCoordsJSONFile = new File(this.getClass().getClassLoader().getResource("countrycodes.json").getFile());
190
191 // Create a map of ALL country code names to ALL the country code json objects
192 // that contain the location (lat, lng) info for each country code
193 Map<String, JsonObject> countryToJsonMap = new HashMap<String, JsonObject>();
194
195 // Parse json file of country codes and put into a JsonArray.
196 // then put into map of each country code to its JsonObject.
197 countryCodesJsonArray = parseJSONFile(countryCoordsJSONFile);
198 for(JsonElement obj : countryCodesJsonArray) {
199 JsonObject countryCodeJson = obj.getAsJsonObject();
200 countryToJsonMap.put(countryCodeJson.get("country").getAsString(), countryCodeJson);
201 }
202
203 // Parse json file of country code counts
204 // Then for each JsonObject in this file,
205 // find a match on its country code in the map created above to get a country code JsonObject
206 // Get the longitude and latitude of the JsonObject that matched that country code.
207 // Add this lng,lat location information to the current JsonObject from the counts file.
208 countryCountsJsonArray = parseJSONFile(countryCountsJSONFile);
209
210 for(JsonElement obj : countryCountsJsonArray) {
211 JsonObject json = obj.getAsJsonObject();
212 String countryCode = json.get("_id").getAsString().toUpperCase();
213 // set the property back as uppercase and with property name "countrycode" instead of "_id"
214 json.remove("_id");
215 json.addProperty("countrycode", countryCode);
216
217 int count = (int)json.get("count").getAsDouble();
218
219 //logger.info("Got country code: " + countryCode);
220 //logger.info(" count: " + count);
221
222 // locate in countryCode map
223 JsonObject countryCodeJson = countryToJsonMap.get(countryCode);
224
225 if(countryCodeJson != null) {
226 //logger.info("Found in map: " + countryCodeJson.toString());
227
228 // for geojson, want longitude then latitude
229 Double lng = countryCodeJson.get("longitude").getAsDouble();
230 Double lat = countryCodeJson.get("latitude").getAsDouble();
231 //logger.info("long: " + Double.toString(lng) + ", lat: " + Double.toString(lat));
232 String countryName = countryCodeJson.get("name").getAsString();
233
234 // let's add lat and lng fields to countryCounts object
235 json.addProperty("lng", lng); // adds Number: https://javadoc.io/static/com.google.code.gson/gson/2.8.5/com/google/gson/JsonObject.html
236 json.addProperty("lat", lat);
237 json.addProperty("region", countryName);
238
239 } else {
240 logger.info("No geolocation info found for country code " + countryCode);
241 if(countryCode.equals("EU")) {
242 logger.info(" Adding lat,lng for somewhere around Europe");
243 //logger.info("Unlisted country code: EU");
244 // add lat and lng for Europe
245 json.addProperty("lng", EU_LNG);
246 json.addProperty("lat", EU_LAT);
247 json.addProperty("region", "Europe");
248 }
249 else if(countryCode.equals("UNKNOWN")) {
250 logger.info(" Adding lat,lng for somewhere in Antarctica");
251 //logger.info("Unlisted country code: UNKNOWN");
252 // add lat and lng for Antarctica
253 json.addProperty("lng", ANTARCTICA_LNG);
254 json.addProperty("lat", ANTARCTICA_LAT);
255 json.addProperty("region", "UNKNOWN");
256 } else {
257 logger.error("ERROR: entirely unknown country code: " + countryCode);
258 }
259 }
260 }
261
262 }
263
264 /** Convert mongodb tabular output of json records stored in the given file
265 * into a JsonArray.
266 */
267 public JsonArray parseJSONFile(File file) throws Exception {
268 JsonArray jsonArray = null;
269 // read into string
270 try (
271 BufferedReader reader = new BufferedReader(new FileReader(file));
272 ) {
273 StringBuilder str = //new StringBuilder();
274 new StringBuilder("[");
275 String line;
276
277 boolean multi_line_comment = false;
278
279 while((line = reader.readLine()) != null) {
280 line = line.trim();
281
282 // ignore any single line comments nested in multi-line symbols
283 if(line.startsWith("/*") && line.endsWith("*/")) {
284 continue; // skip line
285 }
286
287 // skip multi-line comments spread over multiple lines
288 // assumes this ends on a line containing */ without further content on the line.
289 if(line.startsWith("/*") && !line.endsWith("*/")) {
290 multi_line_comment = true;
291 continue; // skip line
292 }
293 if(multi_line_comment) {
294 if(line.contains("*/")) {
295 multi_line_comment = false;
296 }
297
298 continue; // we're in a comment or at end of comment, skip line
299 }
300
301 str.append(line);
302 if(line.endsWith("}")) {
303 str.append(",\n");
304 }
305 }
306 // replace last comma with closing bracket
307 String fileContents = str.substring(0, str.length()-2) + "]";
308
309 //logger.debug("Got file:\n" + fileContents);
310
311 // https://stackoverflow.com/questions/2591098/how-to-parse-json-in-java
312 jsonArray = new JsonParser().parse(fileContents).getAsJsonArray();
313
314 } catch(Exception e) {
315 throw e;
316 }
317
318
319 return jsonArray;
320 }
321
322 /**
323 * Reading
324 * https://www.here.xyz/api/concepts/geojsonbasics/
325 * https://ngageoint.github.io/simple-features-geojson-java/docs/api/
326 *
327 * https://stackoverflow.com/questions/55621480/cant-access-coordinates-member-of-geojson-feature-collection
328 *
329 * Downloaded geojson simple features' jar file from maven, but it didn't work:
330 * a more private version of MultiPoint.java is not included in the jar file (there's only
331 * mil.nga.sf.geojson.MultiPoint , whereas
332 * mil.nga.sf.MultiPoint is missing
333 *
334 * This seems to have gone wrong at
335 * https://github.com/ngageoint/simple-features-geojson-java/tree/master/src/main/java/mil/nga/sf
336 * but the one at
337 * https://github.com/ngageoint/simple-features-java/tree/master/src/main/java/mil/nga/sf
338 * has it. So I've been trying to build that, but don't have the correct version of maven.
339 */
340 public Geometry toMultiPointGeoJson() {
341
342 List<Position> points = new LinkedList<Position>();
343
344 for(JsonElement obj : this.countryCountsJsonArray) {
345 JsonObject json = obj.getAsJsonObject();
346 Double lng = json.get("lng").getAsDouble();
347 Double lat = json.get("lat").getAsDouble();
348
349 Position point = new Position(lng, lat);
350 points.add(point);
351 }
352
353 Geometry multiPoint = new MultiPoint(points);
354
355 return multiPoint;
356 }
357
358 // https://javadoc.io/static/com.google.code.gson/gson/2.8.5/index.html
359 public FeatureCollection toFeatureCollection() {
360 final int HISTOGRAM_WIDTH = 4;
361
362 FeatureCollection featureCollection = new FeatureCollection();
363
364 for(JsonElement obj : this.countryCountsJsonArray) {
365 JsonObject json = obj.getAsJsonObject();
366
367 String countryCode = json.get("countrycode").getAsString();
368 String region = json.get("region").getAsString();
369 int count = json.get("count").getAsInt();
370
371 // make a histogram for each country
372 Geometry rectangle = this.toPolygon(json, count, HISTOGRAM_WIDTH);
373
374 Feature countryFeature = new Feature(rectangle);
375 Map<String, Object> featureProperties = new HashMap<String, Object>();
376 featureProperties.put("count", new Integer(count));
377 featureProperties.put("code", countryCode);
378 featureProperties.put("region", region);
379 countryFeature.setProperties(featureProperties);
380
381 featureCollection.addFeature(countryFeature);
382 }
383
384 return featureCollection;
385 }
386
387 // create rectangular "histogram" for each country code
388 private Geometry toPolygon(JsonObject json, final int count, final int HISTOGRAM_WIDTH) {
389 int half_width = HISTOGRAM_WIDTH/2;
390 double vertical_factor = 1.0;
391
392 final Double lng = json.get("lng").getAsDouble();
393 final Double lat = json.get("lat").getAsDouble();
394
395 String countryCode = json.get("countrycode").getAsString();
396
397
398 //create the 4 corners of the rectangle
399 // West is negative, east is positive, south is negative, north is positive
400 // See http://www.learnz.org.nz/sites/learnz.org.nz/files/lat-long-geo-data-01_0.jpg
401 // But since the histograms grow vertically/northwards and we can't go past a latitude of 90,
402 // to compensate, we increase the width of the histograms by the same factor as our inability
403 // to grow northwards.
404 Double north = lat + (vertical_factor * count);
405
406 while (north > 90) {
407 // recalculate north after decreasing histogram's vertical growth
408 // by the same factor as we increase its width
409 vertical_factor = vertical_factor/2.0;
410 half_width = 2 * half_width;
411 north = lat + (vertical_factor * count);
412 }
413 Double east = lng + half_width;
414 Double west = lng - half_width;
415 Double south = lat;
416
417 List<Position> pts = recalculateAreaIfLarge(count, HISTOGRAM_WIDTH, countryCode, lat, lng, north, south, east, west);
418
419 /*
420 logger.debug("For country " + countryCode + ":");
421 logger.debug("north = " + north);
422 logger.debug("south = " + south);
423 logger.debug("east = " + east);
424 logger.debug("west = " + west + "\n");
425 logger.debug("-------------");
426 */
427
428 List<List<Position>> outerList = new LinkedList<List<Position>>();
429 if(pts != null) {
430 outerList.add(pts);
431 } else {
432
433
434 List<Position> points = new LinkedList<Position>();
435 outerList.add(points);
436
437 points.add(new Position(west, south)); // Position(lng, lat) not Position(lat, lng)
438 points.add(new Position(west, north));
439 points.add(new Position(east, north));
440 points.add(new Position(east, south));
441 }
442
443 Geometry rectangle = new Polygon(outerList);
444
445 // Coords: a List of List of Positions, see https://ngageoint.github.io/simple-features-geojson-java/docs/api/
446 // https://www.here.xyz/api/concepts/geojsonbasics/#polygon
447
448 return rectangle;
449 }
450
451 private List<Position> recalculateAreaIfLarge(final int count, final int HISTOGRAM_WIDTH, String countryCode,
452 final Double lat, final Double lng,
453 Double north, Double south, Double east, Double west) {
454 boolean recalculated = false;
455
456 // Check if we're dealing with very large numbers, in which case, we can have follow off the longitude edges
457 // Max longitude values are -180 to 180. So a max of 360 units between them. (Max latitude is -90 to 90)
458 // "Longitude is in the range -180 and +180 specifying coordinates west and east of the Prime Meridian, respectively.
459 // For reference, the Equator has a latitude of 0°, the North pole has a latitude of 90° north (written 90° N or +90°),
460 // and the South pole has a latitude of -90°."
461 if((east + Math.abs(west)) > 360 || east > 180 || west < -180) {
462 logger.debug("For country " + countryCode + ":");
463 logger.debug("north = " + north);
464 logger.debug("south = " + south);
465 logger.debug("east = " + east);
466 logger.debug("west = " + west + "\n");
467
468 int half_width = HISTOGRAM_WIDTH/2; // reset half_width
469
470 double v_tmp_count = Math.sqrt(count);
471 //double h_tmp_count = Math.floor(v_tmp_count);
472 //v_tmp_count = Math.ceil(v_tmp_count);
473 double h_tmp_count = v_tmp_count;
474 logger.debug("count = " + count);
475 logger.debug("v = " + v_tmp_count);
476 logger.debug("h = " + h_tmp_count);
477 logger.debug("lat = " + lat);
478 logger.debug("lng = " + lng + "\n");
479
480 if(h_tmp_count > 90) { // 360 max width, of which each longitude
481 // is 4 units (horizontal factor = 4, and half-width is half
482 // of that). So max width/h_tmp_count allowed 90 => 360
483 // longitude on map (-180 to 180).
484 // Put the excess h_tmp_count into v_tmp_count and ensure
485 // that does not go over 90+90 = 180 max. Vertical_factor is 1.
486
487 logger.debug("Out of longitude range. Attempting to compensate...");
488
489 double diff = h_tmp_count - 80.0; // actually 90 wraps on geojson tools, 80 doesn't
490 h_tmp_count -= diff;
491 v_tmp_count = (count/h_tmp_count);
492
493 if(v_tmp_count > 180 || h_tmp_count > 90) {
494 logger.warn("Warning: still exceeded max latitude and/or longitude range");
495 }
496
497 }
498
499 logger.debug("Recalculating polygon for country with high count: " + countryCode + ".");
500 logger.debug("count = " + count);
501 logger.debug("v = " + v_tmp_count);
502 logger.debug("h = " + h_tmp_count);
503 logger.debug("lat = " + lat);
504 logger.debug("lng = " + lng + "\n");
505
506
507 north = lat + v_tmp_count;
508 south = lat;
509 east = lng + (h_tmp_count * half_width); // a certain width, half_width, represents one unit in the x axis
510 west = lng - (h_tmp_count * half_width);
511
512 /*
513 logger.debug("north = " + north);
514 logger.debug("south = " + south);
515 logger.debug("east = " + east);
516 logger.debug("west = " + west + "\n");
517 */
518
519 if(north > 90) {
520 // centre vertically on lat
521 north = lat + (v_tmp_count/2);
522 south = lat - (v_tmp_count/2);
523 }
524
525 if(west < -180.0) {
526 double h_diff = -180.0 - west; // west is a larger negative value than -180, so subtracting west from -180 produces a positive h_diff value
527 west = -180.0; // set to extreme western edge
528 east = east + h_diff;
529 }
530 else if(east > 180.0) {
531 double h_diff = east - 180.0; // the country's longitude (lng) is h_diff from the eastern edge
532 east = 180.0; // maximise eastern edge
533 west = west - h_diff; // then grow the remainder of h_tmp_count in the opposite (western/negative) direction
534 }
535
536 // NOTE: Can't centre on country, (lat,lng), as we don't know whether either of lat or lng has gone past the edge
537
538 // Hopefully we don't exceed +90/-90 lat and +/-180 longitude
539
540 recalculated = true;
541
542
543 } else if(west < -140.0) {
544 // past -140 west, the edges don't wrap well in geotools, so shift any points more west/negative than -140:
545
546 double diff = -140.0 - west;
547 west = -140.0;
548 east += diff;
549
550 recalculated = true;
551 }
552
553 if(recalculated) {
554 logger.debug("\nnorth = " + north);
555 logger.debug("south = " + south);
556 logger.debug("east = " + east);
557 logger.debug("west = " + west);
558
559
560 List<Position> points = new LinkedList<Position>();
561
562 points.add(new Position(west, south)); // Position(lng, lat) not Position(lat, lng)
563 points.add(new Position(west, north));
564 points.add(new Position(east, north));
565 points.add(new Position(east, south));
566
567 return points;
568 }
569
570 return null;
571 }
572
573 // by default, display mapdata output on screen too
574 public String writeMultiPointGeoJsonToFile() {
575 return writeMultiPointGeoJsonToFile(PRINT_MAPDATA_TO_SCREEN);
576 }
577 public String writeMultiPointGeoJsonToFile(int displayMapData) {
578 final String filename = "multipoint_" + this.geoJsonFilenameWithSuffix;
579 File outFile = new File(this.outputFolder, filename);
580
581 Geometry geometry = this.toMultiPointGeoJson();
582 String multiPointGeojsonString = FeatureConverter.toStringValue(geometry);
583 if(displayMapData == PRINT_MAPDATA_TO_SCREEN) {
584 logger.info("\nMap data as MultiPoint geometry:\n" + multiPointGeojsonString + "\n");
585 }
586 try (
587 Writer writer = new BufferedWriter(new FileWriter(outFile));
588 ) {
589
590 // Some basic re-formatting for some immediate legibility
591 // But pasting the contents of the file (or the System.err/logger.info output above)
592 // directly into http://geojson.tools/ or http://geojson.io/
593 // will instantly reformat the json perfectly anyway.
594 multiPointGeojsonString = multiPointGeojsonString.replace("[[", "\n[\n\t[");
595 multiPointGeojsonString = multiPointGeojsonString.replace("],[", "],\n\t[");
596 multiPointGeojsonString = multiPointGeojsonString.replace("]]", "]\n]");
597
598 writer.write(multiPointGeojsonString + "\n");
599 } catch(Exception e) {
600 logger.error("Unable to write multipoint geojson:\n**********************");
601 logger.error(multiPointGeojsonString);
602 logger.error("**********************\ninto file " + outFile.getAbsolutePath());
603 logger.error(e.getMessage(), e);
604 }
605
606 return outFile.getAbsolutePath();
607
608 }
609
610 // by default, display mapdata output on screen too
611 public String writeFeaturesGeoJsonToFile() {
612 return writeFeaturesGeoJsonToFile(PRINT_MAPDATA_TO_SCREEN);
613 }
614 // write out geojson features to appropriately named file
615 // If displayMapData == PRINT_MAPDATA_TO_SCREEN, then it will also be printed to screen
616 public String writeFeaturesGeoJsonToFile(int displayMapData) {
617 final String filename = "geojson-features_" + this.geoJsonFilenameWithSuffix;
618 File outFile = new File(this.outputFolder, filename);
619
620 FeatureCollection featureColl = this.toFeatureCollection();
621 this.featuresGeojsonString = FeatureConverter.toStringValue(featureColl);
622 if(displayMapData == PRINT_MAPDATA_TO_SCREEN) {
623 logger.info("\nMap data as featurecollection:\n" + featuresGeojsonString + "\n");
624 }
625 try (
626 Writer writer = new BufferedWriter(new FileWriter(outFile));
627 ) {
628
629 writer.write(featuresGeojsonString + "\n");
630 } catch(Exception e) {
631 logger.error("Unable to write multipoint geojson:\n**********************");
632 logger.error(featuresGeojsonString);
633 logger.error("**********************\ninto file " + outFile.getAbsolutePath());
634 logger.error(e.getMessage(), e);
635 }
636
637 return outFile.getAbsolutePath();
638 }
639
640
641 public String getFeaturesGeoJsonString(boolean uriEncoded) {
642 if(this.featuresGeojsonString == null) {
643 this.featuresGeojsonString = FeatureConverter.toStringValue(this.toFeatureCollection());
644 }
645 if(uriEncoded) {
646 // Want to return encodeURIComponent(JSON.stringify(featuresGeojsonString));
647 // https://stackoverflow.com/questions/607176/java-equivalent-to-javascripts-encodeuricomponent-that-produces-identical-outpu
648 featuresGeojsonString = URLEncoder.encode(featuresGeojsonString);
649 }
650 return featuresGeojsonString;
651 }
652
653 public String getAsMapURL() {
654 // geojson.io advises to call JavaScript's encodeURIComponent() on the string
655 // which for the geojson strings we're dealing with is the same as Java's URI.encode().
656 // But uriEncoding our geojson produces a string unparseable by geojson.io,
657 // whereas the unencoded string is recognised. Not sure why. Turning off uriEncoding.
658 boolean uriEncoded = false;
659 String url = GEOJSON_MAP_TOOL_URL + DATA_STR + getFeaturesGeoJsonString(uriEncoded);
660
661 return url;
662 }
663
664 public static void openFirefox() {
665 // https://www.cyberciti.biz/faq/howto-run-firefox-from-the-command-line/
666 String[] cmdArgs = {
667 "firefox",
668 "-P",
669 "screenshot", //profile //"-profileManager=\"screenshot\"",
670 //"--browser",
671 "--devtools"
672 };
673
674 SafeProcess proc = new SafeProcess(cmdArgs);
675 int retVal = proc.runProcess();
676
677 if(retVal != 0) {
678 logger.info("Process out: " + proc.getStdOutput());
679 logger.info("Process err: " + proc.getStdError());
680 logger.info("Screenshot process returned with: " + retVal);
681 }
682 proc = null;
683
684 /*
685 String[] tab_cmdArgs = {
686 "firefox",
687 "-new-tab",
688 "-jsconsole"
689 };
690 proc = new SafeProcess(tab_cmdArgs);
691 retVal = proc.runProcess();
692 */
693 }
694
695 public String getGeoJSONMapScreenshotCommand(File outputFolder, String fileNamePrefix) {
696 // return the firefox :screenshot WEB CONSOLE command
697 // Only works from Firefox version 62 onwards
698 // https://developer.mozilla.org/en-US/docs/Tools/Taking_screenshots
699
700 String mapURL = this.getAsMapURL();
701
702 File outputFile = new File(outputFolder + File.separator + fileNamePrefix+".png");
703 String outputFilePath = Utility.getFilePath(outputFile);
704
705
706 // Run in web console:
707 // :screenshot --selector ".map" --file --filename "<outputPath/outputImageFile.ext>"
708 // https://developer.mozilla.org/en-US/docs/Tools/Browser_Console
709 String webConsoleCommand = ":screenshot --selector \".map\" --file --filename \"" + outputFilePath + "\"";
710 System.out.println("The following command should have been copied to your clipboard. Else copy it.");
711 System.out.println("Run it in the opened Firefox tab's Web Console command line (press F12 and undock web console first):\n");
712 System.out.println(webConsoleCommand);
713 copyStringToCopyBuffer(webConsoleCommand);
714
715 // Launch firefox in a new tab
716 // https://askubuntu.com/questions/57163/how-to-open-an-url-from-command-line-to-firefox-and-switch-to-tab
717 //https://developer.mozilla.org/en-US/docs/Mozilla/Command_Line_Options
718 String[] cmdArgs = {
719 "firefox",
720 //"-P",
721 //"screenshot", // which does --profileManager "screenshot"
722 //"--devtools", //"-jsconsole" opens browser console instead of web console
723 "-new-tab",
724 mapURL
725 };
726
727
728 System.err.println();
729 SafeProcess proc = new SafeProcess(cmdArgs);
730
731 int retVal = proc.runProcess();
732
733 if(retVal != 0) {
734 logger.info("Process out: " + proc.getStdOutput());
735 logger.info("Process err: " + proc.getStdError());
736 logger.info("Screenshot process returned with: " + retVal);
737 }
738
739 return webConsoleCommand;
740 }
741
742 /** Copies string to copy buffer. Works for me in Firefox on Ubuntu.
743 * Code from https://stackoverflow.com/questions/3591945/copying-to-the-clipboard-in-java
744 */
745 public void copyStringToCopyBuffer(String str) {
746 StringSelection selection = new StringSelection(str);
747 Clipboard clipboard = Toolkit.getDefaultToolkit().getSystemClipboard();
748 clipboard.setContents(selection, selection);
749 }
750
751 // Doesn't work
752 public String geoJsonMapScreenshot(File outputFolder, String fileNamePrefix) {
753 // https://stackoverflow.com/questions/49606051/how-to-take-a-screenshot-in-firefox-headless-selenium-in-java
754
755
756 String mapURL = this.getAsMapURL();
757 ///String mapURLescapedForBash = .replace("\"", "\\\"");//.replace("[", "\\[").replace("]", "\\]");
758
759 // Testing the geojson.io URL with a simpler polygon. Also doesn't work.
760 //String mapURL = GEOJSON_MAP_TOOL_URL + DATA_STR + URLEncoder.encode("{\"type\":\"Polygon\",\"coordinates\":[[[-77,37],[-77,38],[-76,38],[-76,37],[-77,37]]]}");
761
762
763 File outputFile = new File(outputFolder + File.separator + fileNamePrefix+".png");
764 String outputFilePath = Utility.getFilePath(outputFile);
765
766
767 // https://developer.mozilla.org/en-US/docs/Mozilla/Firefox/Headless_mode
768 // /path/to/firefox -P my-profile --screenshot test.jpg https://developer.mozilla.org --window-size=800,1000
769 // https://stackoverflow.com/questions/15783701/which-characters-need-to-be-escaped-when-using-bash
770 // FIREFOX doesn't work as expected: screenshot appears to happen too fast, before json completely loaded
771
772 /*
773 String[] cmdArgs = {
774 "firefox",
775 "--screenshot",
776 outputFilePath,
777 //"--delay=5", // only in firefox console?
778 "--window-size="+MONITOR_RESOLUTION,
779 mapURL //mapURLescapedForBash //"'" + mapURL + "'"
780 };
781 */
782
783 // Trying CHROME - background world map is faded at times or else blocky, page not loaded fully?
784 //https://www.bleepingcomputer.com/news/software/chrome-and-firefox-can-take-screenshots-of-sites-from-the-command-line/
785 // https://bitsofco.de/using-a-headless-browser-to-capture-page-screenshots/
786 // https://developers.google.com/web/updates/2017/04/headless-chrome
787
788
789 String[] cmdArgs = {
790 "/opt/google/chrome/chrome",
791 "--headless",
792 "--screenshot="+outputFilePath, //"--screenshot=\""+outputFilePath+"\"",
793 "--window-size="+MONITOR_RESOLUTION, //"--window-size=1024,1024",
794 mapURL
795 };
796
797
798 /*
799 //WORKS TO RUN FIREFOX TO OPEN A NEW TAB WITH MAPDATA:
800 // https://askubuntu.com/questions/57163/how-to-open-an-url-from-command-line-to-firefox-and-switch-to-tab
801 String[] cmdArgs = {
802 "firefox",
803 "-new-tab",
804 mapURL
805 };
806 */
807
808 /* Can try CutyCapt, http://cutycapt.sourceforge.net/
809 mentioned at https://bugzilla.mozilla.org/show_bug.cgi?id=1412061
810 It requires Qt 4.4.0+.
811 It's installed, https://askubuntu.com/questions/435564/qt5-installation-and-path-configuration
812 Versions 4 and 5 are installed, seen if I run
813 qtchooser -list-versions
814 And when I run: whereis qt5
815 qt5: /usr/share/qt5
816 https://unix.stackexchange.com/questions/116254/how-do-i-change-which-version-of-qt-is-used-for-qmake
817
818 No, doesn't appear to be installed.
819 */
820
821 System.err.println();
822 System.err.print("Running:");
823 for(String arg : cmdArgs) {
824 System.err.print(" " + arg);
825 }
826 System.err.println();
827
828
829 //String cmdArgs = "firefox --screenshot " + outputFilePath + " " + GEOJSON_MAP_TOOL_URL + DATA_STR;
830 //String cmdArgs = "firefox --screenshot " + outputFilePath + " " + "'" + mapURL + "'";
831 //System.err.println("Running: " + cmdArgs);
832
833 System.err.println();
834
835 SafeProcess proc = new SafeProcess(cmdArgs);
836
837 int retVal = proc.runProcess();
838
839 logger.info("Process out: " + proc.getStdOutput());
840 logger.info("Process err: " + proc.getStdError());
841 logger.info("Screenshot process returned with: " + retVal);
842
843 return outputFilePath;
844
845 }
846
847 public int getTotalCount() {
848 int total = 0;
849 for(JsonElement obj : this.countryCountsJsonArray) {
850 JsonObject json = obj.getAsJsonObject();
851 int count = json.get("count").getAsInt();
852 total += count;
853 }
854 return total;
855 }
856
857
858 // Unfinished and unused
859 public void parseCSVFile(String filename) throws Exception {
860 File csvData = new File(filename);
861 CSVParser parser = CSVParser.parse(csvData, java.nio.charset.Charset.forName("US-ASCII"), CSVFormat.RFC4180);
862 for (CSVRecord csvRecord : parser) {
863 logger.info("Got record: " + csvRecord.toString());
864 }
865 }
866
867 public static void printUsage() {
868 System.err.println("CountryCodeCountsMapData <counts-by-countrycode-file>.json");
869 }
870
871 public static void main(String args[]) {
872 if(args.length != 1) {
873 printUsage();
874 System.exit(-1);
875 }
876
877 try {
878 File countsFile = new File(args[0]);
879 if(!countsFile.exists()) {
880 logger.error("File " + countsFile + " does not exist");
881 System.exit(-1);
882 }
883
884 CountryCodeCountsMapData mapData = new CountryCodeCountsMapData(args[0]);
885
886 String multipointOutFileName = mapData.writeMultiPointGeoJsonToFile();
887 String featuresOutFileName = mapData.writeFeaturesGeoJsonToFile();
888
889 logger.info("***********\nWrote mapdata to files " + multipointOutFileName
890 + " and " + featuresOutFileName);
891 logger.info("You can paste the geojson contents of either of these files into "
892 + " the editor at " + GEOJSON_MAP_TOOL_URL
893 + " to see the data arranged on a world map");
894
895 logger.info("Total count for query: " + mapData.getTotalCount());
896
897 } catch(Exception e) {
898 logger.error(e.getMessage(), e);
899 }
900 }
901}
Note: See TracBrowser for help on using the repository browser.