source: other-projects/mirex/grand-challenge/generate-jamendo-dataset/scripts-2015/get-nc-sa-all.py@ 30006

#!/usr/bin/python

import json
import os
import os.path
import urllib
import sys


chunk_size=200

base_url = "http://api.jamendo.com/v3.0/tracks/"
base_url_args = base_url + "?client_id=54cc3f68&format=jsonpretty&audioformat=mp32&audiodlformat=flac&ccnc=true&ccsa=true"

# OK, there are also the include fields 'licenses' and 'lyrics' but for GCUX, there are currently considered unnecessary

base_url_args = base_url_args + "&include=musicinfo+stats"

chunk_url = base_url_args + "&limit=" + str(chunk_size)

output_dir = "download-json-all"

if not os.path.isdir(output_dir):
    print "Creating directory: " + output_dir
    os.mkdir(output_dir)
    

more_to_download = 1
offset = 0
chunk_count = 0

error_count = 0

#171800 (example of an argv[1] values?)
#
# but note, as currently written, the code makes use of 'offset' but not the derived 'chunk_offset'


if (len(sys.argv)==2):
    offset=argv[1]
    chunk_offset=offset//chunk_size

while (more_to_download) :

    output_filename = os.path.join(output_dir ,"nc-sa-chunk-{0:03d}.json".format(chunk_count))

    if os.path.isfile(output_filename):
        print "Skipping Offset = " + str(offset) + " as downloaded file already exists"
        offset += chunk_size;
        chunk_count += 1;
        continue

    download_url = chunk_url + "&offset=" + str(offset)
    print "Downloading: " + download_url
    download_url_handle = urllib.urlopen(download_url)

    json_data = download_url_handle.read()

    with open(output_filename, "w") as json_ofile:
        json_ofile.write(json_data)
        json_ofile.close()

    try:
        data = json.loads(json_data)

        headers = data[u'headers']
        print "  Status: " + headers[u"status"]

        results_count = headers[u'results_count']


        offset += results_count

        if results_count != chunk_size:
            more_to_download = 0


    except:
        print "Warning: failed to process Offset = " + str(offset) + " (chunk = " + str(chunk_count) + ")"
        print "Assuming failed block was standard size (" + str(chunk_size) + ")"
        offset += chunk_size
        error_count = error_count + 1

        if error_count >= 10:
            more_to_download = 0


    chunk_count += 1

print "====="
print "Retrieved {0} Non-Copyright Share-Alike tracks from Jamendo".format(offset)
print "====="