get_inflection_scores_by_year.py

import numpy as np
import constants
import copy


def gini(populations, accuracy):
    assert len(populations) == len(accuracy)
    N = len(populations)
    sum_nom = 0
    sum_denom = 0
    for i in range(N):
        for j in range(N):
            sum_nom += (
                populations[i] * populations[j] * np.abs(accuracy[i] - accuracy[j])
            )
        sum_denom += populations[i] * accuracy[i]
    return sum_nom / (2 * np.sum(populations) * sum_denom)


def include_diversity(l, T=1):
    acc_arr = np.array(l)
    acc_arr = [f**T for f in acc_arr]
    N = sum(acc_arr)
    acc_arr = [f / N for f in acc_arr]
    return list(acc_arr)


TOTAL_POPULATION = constants.TOTAL_POPULATION / 1000000
all_populations = constants.read_sig_populations()
languages1 = constants.get_sig_languages()
languages2 = list(languages1)
languageso = list(languages1)
pop_denom = constants.TOTAL_POPULATION / 1000000


prev_all_bleus = {}
print(f"year\tx (tau=1)\ty (tau->0)")
for year in range(2014, 2022):
    all_bleus = constants.read_sig(year=str(year)[-2:])
    if prev_all_bleus:
        for key in prev_all_bleus:
            if key not in all_bleus:
                all_bleus[key] = prev_all_bleus[key]
            elif prev_all_bleus[key] > all_bleus[key]:
                all_bleus[key] = prev_all_bleus[key]
    prev_all_bleus = copy.deepcopy(all_bleus)

    languages1 = constants.get_sig_languages()
    languages2 = list(languages1)
    languageso = list(languages1)

    # populationso = [all_populations[l] for l in languages2 if l !='eng' ]
    populationso = [all_populations[l] for l in languages2]
    accuracyo = [all_bleus[l1] for l1 in languages2]
    languages = list(languageso)

    # print(f"Year: {year}")
    # print(f"Pairs: {len(accuracyo)}")
    langs_to_show = set()

    TOTAL_LANGS = 6500
    answers = []
    for temperature in [1, 0.01]:
        if temperature == 1:
            remaining = 1
        else:
            remaining = TOTAL_LANGS - len(languages)
        accuracy = accuracyo + [0] * remaining
        languages = languageso + ["rest"] * remaining
        if temperature == 1:
            tosplit = TOTAL_POPULATION - sum(populationso)
            populations = populationso + [tosplit]
        else:
            populations = [1 for l in languages]

        inds = np.flip(np.argsort(accuracy))
        N = np.sum(populations)
        populations = [populations[i] / N for i in inds]
        accuracy = [accuracy[i] / 100 for i in inds]
        languages = [languages[i] for i in inds]

        N = np.sum(populations)
        old_populations = [p / N for p in populations]

        populations = include_diversity(old_populations, T=temperature)
        gini_coeff = gini(np.array(populations) * N, accuracy)

        M_score = np.sum(np.array(populations) * np.array(accuracy))
        # print(f"temperature {temperature}: {M_score} ({len(languageso)} languages)")
        answers.append(M_score)

    print(f"{year}\t{answers[0]}\t{answers[1]}")