Index: other-projects/mirex/grand-challenge/generate-jamendo-dataset/scripts-2015/generate-csv-musicinfo-tags.py
===================================================================
--- other-projects/mirex/grand-challenge/generate-jamendo-dataset/scripts-2015/generate-csv-musicinfo-tags.py	(revision 30005)
+++ other-projects/mirex/grand-challenge/generate-jamendo-dataset/scripts-2015/generate-csv-musicinfo-tags.py	(revision 30005)
@@ -0,0 +1,101 @@
+#!/usr/bin/python
+
+from os import listdir
+from os.path import isfile, join
+
+import json
+import sys
+import signal
+
+def tabulate_results():
+    print "===="
+    print "Number of records processed " + str(jid_count)
+    print "===="
+
+    ofd_freq = open(join(output_dir,"genre-tags-freq.csv"), "w")
+
+    print "Generating ..."
+    print "  Genre Tags Frequency Distribution"
+    print "  (one track can be given assigned multiple genres)"
+    
+    print >> ofd_freq, "Genre tag,Frequency count"
+
+    for gk in genre_table.keys():
+        print >> ofd_freq, "{0:20}".format(gk) + "," + str(genre_table[gk])
+
+    ofd_freq.close();
+
+    ofd_distrib = open(join(output_dir, "genre-tags-distribution-count.csv"), "w")
+
+    print >> ofd_distrib, "Number of Genre Tags Assigned per Track"
+    print >> ofd_distrib, "Num of tags,Frequency count"
+    for gk in sorted(genre_table_tagcount.keys()):
+        print >> ofd_distrib, gk + "," + str(genre_table_tagcount[gk])
+
+    ofd_distrib.close();
+
+
+def trap_error(signal, frame):
+    print >> sys.stderr, "Ctrl-C Caught!"
+    tabulate_results()
+    sys.exit(0)
+
+
+
+def processTagset(jid,tagrec):
+
+    # genres, instruments, vartags
+
+    genres = tagrec[u'genres']
+    num_genres = len(genres)
+
+    num_genres_str = str(num_genres)
+    if not genre_table_tagcount.has_key(num_genres_str):
+        genre_table_tagcount[num_genres_str] = 0
+    genre_table_tagcount[num_genres_str] += 1
+
+    for g in genres:
+        if not genre_table.has_key(g):
+            genre_table[g] = 0
+        genre_table[g] += 1
+
+argc = len(sys.argv)
+
+input_dir  = sys.argv[1] if (argc==2) or (argc==3)  else "download-json-all"
+output_dir = sys.argv[2] if argc==3 else "csv-data"
+
+json_files = [ jf for jf in listdir(input_dir) if isfile(join(input_dir,jf)) ]
+
+
+genre_table = {}
+genre_table_tagcount = {}
+
+jid_count = 0
+
+signal.signal(signal.SIGINT, trap_error)
+
+
+for jf in json_files:
+    print "Processing" + jf
+
+    json_filename = join(input_dir,jf)
+    ifd = open(json_filename)
+    json_data = ifd.read()
+    data = json.loads(json_data)
+
+#    if jid_count>100:
+#        break
+
+    for rec in data[u'results']:
+
+        jid = rec[u'id']
+        jid_count += 1
+ 
+        musicinfo = rec[u'musicinfo']
+        tags = musicinfo[u'tags']
+        processTagset(jid,tags)
+
+    ifd.close()
+
+
+tabulate_results()
Index: other-projects/mirex/grand-challenge/generate-jamendo-dataset/scripts-2015/generate-evened-out-10000-set.py
===================================================================
--- other-projects/mirex/grand-challenge/generate-jamendo-dataset/scripts-2015/generate-evened-out-10000-set.py	(revision 30005)
+++ other-projects/mirex/grand-challenge/generate-jamendo-dataset/scripts-2015/generate-evened-out-10000-set.py	(revision 30005)
@@ -0,0 +1,179 @@
+#!/usr/bin/python
+
+import json
+import os
+from os.path import isfile, join
+
+import sys
+import csv
+
+import jamendo
+
+verbosity = 1
+dataset_size = 10000
+artist_limit = 20
+
+track_id_to_artist_id = {}
+
+evened_out_by_genre_distrib = {}
+evened_out_by_id_distrib    = {}
+evened_out_by_rank_distrib  = {}
+
+evened_out_unique_tracks = []
+evened_out_chosen_track  = {}
+evened_out_artist_limit  = {}
+
+def genre_sorted_by_freq(input_file,order='ascending'):
+    with open(input_file) as csvfile:
+        freq_reader = csv.reader(csvfile, delimiter=',', quotechar='"')
+
+        # Remove first entry, which is text information
+        freq_reader.next()
+        
+        freq_data = []
+        for row in freq_reader:
+            genre=row[0].rstrip()
+            freq=row[1]
+            freq_data.append( (genre,freq))
+
+            if order == "descending":
+                freq_data_sorted = sorted(freq_data, key=lambda data: int(data[1]),reverse=True)
+            else:
+                   freq_data_sorted = sorted(freq_data, key=lambda data: int(data[1]),reverse=False)         
+    
+
+    return freq_data_sorted
+
+
+def process_boosted_genre(genre,rank_base,json_data):
+    results = json_data[u'results']
+
+    rank = 1 + rank_base
+    
+    for result in results:
+        id   = result[u'id'];
+        name = result[u'name'];
+        artist_id   = result[u'artist_id'];
+
+        track_id_to_artist_id[id] = artist_id
+
+        if (verbosity>=2):
+            print "genre: " + genre + ", id = " + id + ", name = " + result[u'name']
+
+        if evened_out_by_genre_distrib.get(id) == None:
+            evened_out_by_genre_distrib[genre] = {}
+        evened_out_by_genre_distrib[genre][id] = True;
+
+        if evened_out_by_id_distrib.get(id) == None:
+            evened_out_by_id_distrib[id] = []            
+        evened_out_by_id_distrib[id].append(genre)
+
+        if evened_out_by_rank_distrib.get(rank) == None:
+            evened_out_by_rank_distrib[rank] = []
+        evened_out_by_rank_distrib[rank].append(id)
+        
+        rank = rank + 1
+
+def evened_out_by_rank():
+    for rank_id in evened_out_by_rank_distrib:
+        rank_id_list = evened_out_by_rank_distrib[rank_id]
+        print "Rank: " + str(rank_id)
+        for track_id in rank_id_list:
+            artist_id = track_id_to_artist_id[track_id]
+            
+            if evened_out_artist_limit.get(artist_id) == None:
+                evened_out_artist_limit[artist_id] = 0
+                
+            evened_out_artist_limit[artist_id] = evened_out_artist_limit[artist_id] + 1
+            if (evened_out_chosen_track.get(track_id) == None):
+                            
+                if (evened_out_artist_limit[artist_id] < artist_limit):
+                    print "  Adding: " + track_id
+                    evened_out_unique_tracks.append(track_id)
+                    evened_out_chosen_track[track_id] = True
+                else:
+                    if verbosity >= 2:
+                        print "    Skipping track id due to artist limit: " + track_id
+            else:
+                if verbosity >= 2:
+                    print "    Skipping as track already chosen: " + track_id          
+
+            if len(evened_out_unique_tracks) == dataset_size:
+                print "Reached dataset size!"
+                break
+
+        if len(evened_out_unique_tracks) == dataset_size:
+            break
+
+def boosted_by_genre(genre,curr_freq_level):
+
+    for i in xrange(0, curr_freq_level, jamendo.chunk_size):
+        curr_chunk_size = curr_freq_level % jamendo.chunk_size if (i +jamendo.chunk_size)>curr_freq_level else jamendo.chunk_size
+
+        json_data = jamendo.download_from_jamendo(i,curr_chunk_size,
+                                                  "&tags="+genre+"&boost=popularity_total")
+        process_boosted_genre(genre,i,json_data)
+
+            
+
+def boosted_by_all_genres_descending(genre_freq_sorted,freq_levels):
+
+    curr_freq_level = freq_levels.pop(0);
+
+    break_count = 0
+    
+    for genrec in genre_freq_sorted:
+
+#        if break_count==2:
+#            break
+    
+        genre = genrec[0]
+        freq = int(genrec[1])
+
+##        while freq<2*curr_freq_level:
+        while freq<curr_freq_level:            
+            # under current limit
+            # => move on to next lower limit
+            # (but stop if freq_levels becomes empty)
+
+            if (len(freq_levels)==0):
+                break
+            
+            curr_freq_level = freq_levels.pop(0);
+
+        if freq>=curr_freq_level:
+                
+            # still in comfortable zone for selecting
+            print "boost by genre " + genre + ": get "+ str(curr_freq_level) + " from " + str(freq) + " entries"
+            boosted_by_genre(genre,curr_freq_level)
+
+        break_count = break_count + 1
+
+    print "*******"
+    print "* Number of tracks to choose from: " + str(len(evened_out_by_id_distrib))
+    print "*******"
+    
+        
+argc = len(sys.argv)
+
+input_file = sys.argv[1] if argc==2 else join("csv-data","genre-tags-freq.csv")
+
+genre_freq_sorted = genre_sorted_by_freq(input_file,order='descending')
+#genre_freq_sorted = genre_sorted_by_freq(input_file,order='ascending')
+
+#for row in genre_freq_sorted:
+#    print(','.join(row))
+
+
+freq_levels = [1000, 800, 600, 400, 300, 200, 100, 50]
+
+#freq_levels = [400, 200, 100, 10]
+#freq_levels = [10, 200, 400]
+
+boosted_by_all_genres_descending(genre_freq_sorted,freq_levels)
+
+
+evened_out_by_rank()
+
+with open("jamendo-evened-out-dataset-trackids.json", "w") as json_ofile:
+    json.dump(evened_out_unique_tracks,json_ofile)
Index: other-projects/mirex/grand-challenge/generate-jamendo-dataset/scripts-2015/generate-evened-out-genre-csv.py
===================================================================
--- other-projects/mirex/grand-challenge/generate-jamendo-dataset/scripts-2015/generate-evened-out-genre-csv.py	(revision 30005)
+++ other-projects/mirex/grand-challenge/generate-jamendo-dataset/scripts-2015/generate-evened-out-genre-csv.py	(revision 30005)
@@ -0,0 +1,124 @@
+#!/usr/bin/python
+
+from os import listdir
+from os.path import isfile, join
+
+import json
+import random
+import sys
+
+
+
+def tabulate_results():
+    print "===="
+    print "Number of records processed " + str(jid_count)
+    print "===="
+
+    ofd_freq = open(join(output_dir,"genre-tags-freq-10000.csv"), "w")
+
+    print "Generating ..."
+    print "  Genre Tags Frequency Distribution"
+    print "  (one track can be given assigned multiple genres)"
+    
+    print >> ofd_freq, "Genre tag,Frequency count"
+
+    for gk in genre_table.keys():
+        print >> ofd_freq, "{0:20}".format(gk) + "," + str(genre_table[gk])
+
+    ofd_freq.close();
+
+    ofd_distrib = open(join(output_dir, "genre-tags-distribution-count-10000.csv"), "w")
+
+    print "  Number of Genre Tags Assigned per Track"
+    print >> ofd_distrib, "Num of tags,Frequency count"
+    for gk in sorted(genre_table_tagcount.keys()):
+        print >> ofd_distrib, gk + "," + str(genre_table_tagcount[gk])
+
+    ofd_distrib.close();
+
+
+
+def processTagset(jid,tagrec):
+
+    # genres, instruments, vartags
+
+    genres = tagrec[u'genres']
+    num_genres = len(genres)
+
+    num_genres_str = str(num_genres)
+    if not genre_table_tagcount.has_key(num_genres_str):
+        genre_table_tagcount[num_genres_str] = 0
+    genre_table_tagcount[num_genres_str] += 1
+
+    for g in genres:
+        if not genre_table.has_key(g):
+            genre_table[g] = 0
+        genre_table[g] += 1
+
+        
+
+argc = len(sys.argv)
+
+input_dir  = sys.argv[1] if (argc==2) or (argc==3)  else "download-json-all"
+output_dir = sys.argv[2] if argc==3 else "csv-data"
+
+json_files = [ jf for jf in listdir(input_dir) if isfile(join(input_dir,jf)) ]
+
+jamendo_ids = [];
+
+genre_table = {}
+genre_table_tagcount = {}
+
+jid_count = 0
+
+evened_out_json_data=open('jamendo-evened-out-dataset-trackids.json').read()
+evened_out_jamendo_ids=json.loads(evened_out_json_data)
+
+evened_out_hashmap = {}
+for jid in evened_out_jamendo_ids:
+    evened_out_hashmap[jid] = True
+
+print "Evened out dataset size = " + str(len(evened_out_hashmap))
+
+all_hashmap = {}
+
+for jf in json_files:
+    print "Processing" + jf
+
+    json_filename = join(input_dir,jf)
+    ifd = open(json_filename)
+    json_data = ifd.read()
+    data = json.loads(json_data)
+
+#    if jid_count>100:
+#        break
+
+    for rec in data[u'results']:
+
+        jid = rec[u'id']
+        all_hashmap[jid] = True
+        
+        jid_count += 1
+ 
+        musicinfo = rec[u'musicinfo']
+        tags = musicinfo[u'tags']
+        if (evened_out_hashmap.get(jid) != None):
+            processTagset(jid,tags)
+        
+
+    ifd.close()
+
+
+tabulate_results()
+
+print "Checking for consitency!"
+
+for jid in evened_out_jamendo_ids:
+    if all_hashmap.get(jid) == None:
+        print "**** Not found in all jid map:" + str(jid)
+
+
+
+
+
+
Index: other-projects/mirex/grand-challenge/generate-jamendo-dataset/scripts-2015/get-audio-download-mp3.py
===================================================================
--- other-projects/mirex/grand-challenge/generate-jamendo-dataset/scripts-2015/get-audio-download-mp3.py	(revision 30005)
+++ other-projects/mirex/grand-challenge/generate-jamendo-dataset/scripts-2015/get-audio-download-mp3.py	(revision 30005)
@@ -0,0 +1,42 @@
+#!/usr/bin/python
+
+import os
+import os.path
+
+import json
+import sys
+
+#input_dir = sys.argv[1] if len(sys.argv)==2 else "download-json"
+input_dir = sys.argv[1] if len(sys.argv)==2 else "download-json-musicinfo"
+
+output_dir = "download-audio"
+
+json_files = [ jf for jf in os.listdir(input_dir) if os.path.isfile(os.path.join(input_dir,jf)) ]
+
+for jf in json_files:
+    json_filename = os.path.join(input_dir,jf)
+    json_data=open(json_filename).read()
+    data = json.loads(json_data)
+
+    for rec in data[u'results']:
+
+        id = int(rec[u'id'])
+        prefix = id//1000
+
+        output_prefix_dir = os.path.join(output_dir,str(prefix));
+        if not os.path.isdir(output_prefix_dir):
+            print "Creating " + output_prefix_dir
+            os.mkdir(output_prefix_dir)
+
+        output_filename = os.path.join(output_prefix_dir, str(id) + ".mp3")
+
+        if not os.path.isfile(output_filename):
+            audio_download_url = rec[u'audio']
+             
+            cmd = "wget -O " + output_filename +  " \"" + audio_download_url + "\""
+
+            print "Downloading:" +cmd
+            os.system(cmd)
+        else:
+            print "Already downloaded: " + output_filename + " => skipping"
+
Index: other-projects/mirex/grand-challenge/generate-jamendo-dataset/scripts-2015/get-nc-sa-all.py
===================================================================
--- other-projects/mirex/grand-challenge/generate-jamendo-dataset/scripts-2015/get-nc-sa-all.py	(revision 30005)
+++ other-projects/mirex/grand-challenge/generate-jamendo-dataset/scripts-2015/get-nc-sa-all.py	(revision 30005)
@@ -0,0 +1,92 @@
+#!/usr/bin/python
+
+import json
+import os
+import os.path
+import urllib
+import sys
+
+
+chunk_size=200
+
+base_url = "http://api.jamendo.com/v3.0/tracks/"
+base_url_args = base_url + "?client_id=54cc3f68&format=json&audioformat=mp32&audiodlformat=flac&ccnc=1&ccsa=1"
+
+# OK, there are also the include fields 'licenses' and 'lyrics' but for GCUX, there are currently considered unnecessary
+
+base_url_args = base_url_args + "&include=musicinfo+stats"
+
+chunk_url = base_url_args + "&limit=" + str(chunk_size)
+
+output_dir = "download-json-all"
+
+if not os.path.isdir(output_dir):
+    print "Creating directory: " + output_dir
+    os.mkdir(output_dir)
+    
+
+more_to_download = 1
+offset = 0
+chunk_count = 0
+
+error_count = 0
+
+#171800 (example of an argv[1] values?)
+#
+# but note, as currently written, the code makes use of 'offset' but not the derived 'chunk_offset'
+
+
+if (len(sys.argv)==2):
+    offset=argv[1]
+    chunk_offset=offset//chunk_size
+
+while (more_to_download) :
+
+    output_filename = os.path.join(output_dir ,"nc-sa-chunk-{0:03d}.json".format(chunk_count))
+
+    if os.path.isfile(output_filename):
+        print "Skipping Offset = " + str(offset) + " as downloaded file already exists"
+        offset += chunk_size;
+        chunk_count += 1;
+        continue
+
+    download_url = chunk_url + "&offset=" + str(offset)
+    print "Downloading: " + download_url
+    download_url_handle = urllib.urlopen(download_url)
+
+    json_data = download_url_handle.read()
+
+    with open(output_filename, "w") as json_ofile:
+        json_ofile.write(json_data)
+        json_ofile.close()
+
+    try:
+        data = json.loads(json_data)
+
+        headers = data[u'headers']
+        print "  Status: " + headers[u"status"]
+
+        results_count = headers[u'results_count']
+
+
+        offset += results_count
+
+        if results_count != chunk_size:
+            more_to_download = 0
+
+
+    except:
+        print "Warning: failed to process Offset = " + str(offset) + " (chunk = " + str(chunk_count) + ")"
+        print "Assuming failed block was standard size (" + str(chunk_size) + ")"
+        offset += chunk_size
+        error_count = error_count + 1
+
+        if error_count >= 10:
+            more_to_download = 0
+
+
+    chunk_count += 1
+
+print "====="
+print "Retrieved {0} Non-Copyright Share-Alike tracks from Jamendo".format(offset)
+print "====="
Index: other-projects/mirex/grand-challenge/generate-jamendo-dataset/scripts-2015/jamendo.py
===================================================================
--- other-projects/mirex/grand-challenge/generate-jamendo-dataset/scripts-2015/jamendo.py	(revision 30005)
+++ other-projects/mirex/grand-challenge/generate-jamendo-dataset/scripts-2015/jamendo.py	(revision 30005)
@@ -0,0 +1,33 @@
+
+import urllib
+import json
+
+chunk_size=200
+
+base_url = "http://api.jamendo.com/v3.0/tracks/"
+base_url_args = base_url + "?client_id=54cc3f68&format=json&audioformat=mp32&audiodlformat=flac&ccnc=1&ccsa=1"
+
+##base_url_args = base_url_args + "&include=musicinfo+stats"
+
+
+def download_from_jamendo(offset=0,currentChunkSize=chunk_size,extraArgs=''):
+
+    chunk_url = base_url_args + "&limit=" + str(currentChunkSize)
+
+    download_url = chunk_url + "&offset=" + str(offset)
+    download_url = download_url + extraArgs
+    
+    print "Downloading: " + download_url
+    download_url_handle = urllib.urlopen(download_url)
+
+    json_data = download_url_handle.read()
+
+    try:
+        data = json.loads(json_data)
+        print "  Status: " + data[u"headers"][u"status"]
+        
+    except:
+        print "Warning: failed to process Offset = " + str(offset) + " (chunk = " + str(chunk_count) + ")"
+        data = null
+                
+    return data
