Changeset 34979 for main

Timestamp:

2021-03-28T00:14:39+13:00 (3 years ago)

Author:

davidb

Message:

Initial version; code tidy up in other areas

Location:

main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/errata-categories

Files:

• esc-wikipedia-download-and-detect-missing-cat-entries.py (added)
• esc-wikipedia-download-and-process-votes.py (modified) (2 diffs)
• escwikipedia.py (modified) (5 diffs)

main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/errata-categories/esc-wikipedia-download-and-process-votes.py

@@ -121 +121 @@
     parser = argparse.ArgumentParser()
     parser.add_argument('--startyear', type=int, default=1956)
-    parser.add_argument('--endyear',   type=int, default=1999)
+    parser.add_argument('--endyear',   type=int, default=2019)
     parser.add_argument('output-file.json', nargs="?", default="metadata-esc-year/metadata-esc.json");
     
@@ -135 +135 @@
         
     for year in range(start_year, end_year+1):
-        country_year_recs = escwikipedia.process_category_page(year)
+        country_year_recs = escwikipedia.process_esc_article_page(year)
 
         all_country_year_recs.append(country_year_recs)

main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/errata-categories/escwikipedia.py

@@ +1 @@
+
+# TODO
+# Check to see if Song/Title coming back with quotes around it
+# In addition to Languages->Language, split of "," ??
+#   (e.g., 2005 has "Language" but entries like English, Spanish)
+
+
 
 from __future__ import print_function
@@ -5 +12 @@
 import os
 import re
+import requests
 
 import argparse
 import bs4
 import wikipedia
+
+import wikipediaapi
 
 DEBUG=False
@@ -218 +228 @@
             print("  " + metadata_key + " = " + repr(metadata_val))
         
-def process_category_page(year):
+def process_esc_article_page(year):
     esc_wiki_page = "Eurovision_Song_Contest_" + str(year)
     esc_wiki_page_file = esc_wiki_page.replace(" ","_") + ".html"
@@ -263 +273 @@
     country_year_recs = convert_cols_to_country_year_recs(header_to_vals,year)
 
-    print()
     print("==========")
 
@@ -271 +280 @@
 
 
-
-
+def process_category_page(year):
+
+    category_countries = {}
+    
+    esc_wiki_page = "Category:Countries_in_the_Eurovision_Song_Contest_" + str(year)
+    esc_wiki_page_file = esc_wiki_page.replace(" ","_") + ".html"
+    esc_wiki_page_filename = os.path.join(cache_dir,esc_wiki_page_file)
+
+    esc_cat_year_html = ""
+    if not(os.path.exists(esc_wiki_page_filename)):
+        print("Retrieving Wikipedia page '" + esc_wiki_page + "'")
+        ##esc_cat_year_wp = wikipedia.page(esc_wiki_page,auto_suggest=False,preload=True,redirect=True)
+        #esc_cat_year_wp = wikipedia.page(esc_wiki_page)
+        #esc_cat_year_html = esc_cat_year_wp.html()
+
+        response = requests.get("https://en.wikipedia.org/wiki/"+esc_wiki_page)
+        esc_cat_year_html = response.text
+        
+        print("  Saving page to cache")
+        write_text_file(esc_wiki_page_filename,esc_cat_year_html)
+    else:
+        print("Retrieving " + esc_wiki_page_file + " from cache")
+        esc_cat_year_html = read_text_file(esc_wiki_page_filename)
+
+        
+    esc_cat_year_soup = bs4.BeautifulSoup(esc_cat_year_html, 'html.parser')
+    # print(repr(esc_cat_year_soup.body))
+          
+    a_href_re = re.compile(r"^/wiki/([^/:]+)_in_the_Eurovision_Song_Contest_" + str(year) + r"$")
+
+    esc_cat_a_tags = esc_cat_year_soup.find_all("a",href=a_href_re)
+    
+    for a_tag in esc_cat_a_tags:
+        href = a_tag.get("href")
+        country = re.search(a_href_re,href).group(1)
+        country = country.replace("_"," ")
+        
+        category_countries[country] = 1
+            
+    return category_countries
+
+
+
+def cat_test():
+
+    wiki_wiki = wikipediaapi.Wikipedia("en",extract_format=wikipediaapi.ExtractFormat.HTML)
+
+    cat_title = "Category:Countries_in_the_Eurovision_Song_Contest_1956"
+    cat = wiki_wiki.page(cat_title)
+
+    print("**** html = " + cat.text)
+    
+    
+    print("Category members: " + cat_title)
+    for p in cat.categorymembers.values():
+        if p.namespace == wikipediaapi.Namespace.CATEGORY:
+            # it is category, so you have to make decision
+            # if you want to fetch also text from pages that belong
+            # to this category
+            print("CatNS")
+            print(p)
+        elif p.namespace == wikipediaapi.Namespace.MAIN:
+            # it is page => we can get text
+            print("MAIN")
+            print(p)
+            print(p.text)