source: other-projects/metadata-encoding/py/using-mongodb/graphSampleError.py@ 38791

#!/usr/bin/env python3

# Script to generate a time series graph of percentage of & errors in CrossRef titles
# Does not generate a data file! Graphing is handled by matplotlib.

import pymongo
from matplotlib import pyplot as plt
from collections import defaultdict
import alive_progress

# Connect to MongoDB
client = pymongo.MongoClient("mongodb://localhost")
db = client["alpss"]
collection = db["crossref"]

# Query MongoDB to get the relevant data
cursor = collection.find({}, {"title": 1, "issued": 1})

# Process the data to calculate the percentage of titles containing "&amp" by year
year_count = defaultdict(int)
amp_count = defaultdict(int)
count = 0

for document in cursor:
    issued_year = document["issued"][0][0] if "issued" in document and document["issued"] else None
    title_contains_amp = "&amp" in document.get("title", "")
    #count = count + 1
    #print(str(count))
    if issued_year is not None:
        year_count[repr(issued_year)] += 1
        if title_contains_amp:
            amp_count[repr(issued_year)] += 1
    #if count == 500000:
        #break

# Calculate the percentage
percentage_data = {year: (amp_count[year] / year_count[year]) * 100 if year_count[year] > 0 else 0 for year in year_count}

#print(year_count)
#print(amp_count)

year_range_min = 1980
year_range_max = 2023

years = []
percentages = []

for yr in range(year_range_min, year_range_max + 1):
    years.append(yr)
    percentages.append(percentage_data[str(yr)])

# Create a time-series graph using matplotlib
#years = list(percentage_data.keys())
#percentages = list(percentage_data.values())

print(percentage_data)

plt.plot(years, percentages, marker='o')
plt.xlabel('Year')
plt.ylabel('Percentage of Titles with "&amp"')
plt.title('Time Series of Percentage of Titles with "&amp" by Year')
plt.grid(True)
plt.show()