from tools.viz import *
gv()

{
    1997: ["Betreff 1", "Betreff 2", ...],
    ...
    2023: ["Betreff 14289", "Betreff 14290", ...]
}

import pickle

# import transformed headings
with open("subjects_1997-2023_clean.pkl", "rb") as f:
    subjects_dict = pickle.load(f)

subjects_dict[1997][:10]

['kehrer ueber scientology',
 'korrektion',
 'tagung in england',
 'beitraege',
 'adresse fuer beitraege',
 'mitteleuropaeische zeit',
 'virtueller ausflug',
 'scientology',
 'noch mal scientology',
 'scientology-artikel']

# create list with all subjects for later; you can ignore this right now
all_subjects = list()

for _,v in subjects_dict.items():
    all_subjects.extend(v)

filter_subjects_by_length(all_subjects, 200)

array(['konferenz: historische etappen der globalisierung in christentumsgeschichtlicher perspektive, 18.\x9620. februar 2010 (fünfte internationale münchen-freising konferenz zur außereuropäischen christentumsgeschichte)',
       '"discursive formation, ethnographic encounter, photographic evidence: the centenary of durkheim\'s basic forms of religious life and the anthropological study of australian aboriginal religion in his time',
       '5th international kraków study of religions symposium: understanding and explanation in the study of religions, in memory of w. burkert (1931-2015), jagiellonian university in kraków, 7th-9th november 2016',
       "ii congresso intern. studi ermeneutici su simbolo, mito e 'modernità dell’antico' nella letteratura italiana e nelle arti dal rinascimento ai giorni nostri, milano, università cattolica, 19-20 dic. 2016",
       "p. santangelo, l' uomo fra cosmo e società. il neoconfucianesimo e un millennio di storia cinese (= man between cosmos and society: neo-confucianism and a millennium of chinese history), milano-udine 2016",
       'call for papers iii intern. conf. on hermeneutics of symbol, myth and modernity of antiquity in italian literature and the arts from the renaissance up to the present day milan, università cattolica, 15-16 dec. 2017',
       'intern. conf. on hermeneutics of symbol, myth and ‘modernity of antiquity’ in italian literature and arts from the renaissance to the present day, milan, università cattolica del sacro cuore, 19-20, dec., 2016)',
       '[tonan] panel proposal contrasting religious multiple identitities: efforts to mark ...., congress of european society of history of religions - bern june 2018. corrige: association for the study of religions',
       'cfp: globale, nationale und lokale dynamiken religiöser pluralität und ihre regulierung (sektionsveranstaltung der sektion religionssoziologie beim 39. kongress der deutschen gesellschaft für soziologie in göttingen)',
       'call for papers: mothering(s) and religions: normative perspectives and individual appropriations. a cross-cultural and interdisciplinary approach from antiquity to the present, deadline dec. 21, 2018.',
       'einladung zum workshop „kommunikation mit dem jenseits: von der antiken literatur bis zu modernen kinder- und jugendliteratur / communication with the underworld: from ancient literature to modern children’s and young adult literature”',
       'workshop an der humboldt-universität zu berlin: kommunikation mit dem jenseits: von der antiken literatur bis zur modernen kinder- und jugendliteratur / communication with the underworld: from ancient literature to modern children’s and young adult literature',
       'assistenzprofessur für religion und öffentlichkeit (religion and the public) am zentrum für religion, wirtschaft und politik (zrwp) für sechs jahre (ohne tenure track), theologische fakultät der universität zürich',
       'call for papers: religion in modern education: conflict, policy and practices. hosted by the australian national university (anu), canberra, australia, 13-15 april 2023 - abstract deadline: 14 february 2023'],
      dtype='<U259')

show_most_frequent_topics(all_subjects)

SUCHWÖRTER = ["islam", "muslim"]

plot_topics_per_year(subjects_dict, SUCHWÖRTER)

[('islam bei wikipedia', 18), ('gedanken zur verantwortung der religionswissenschaft/islamwissenschaft', 16), ('aufruf islamischer gelehrter an den is', 12), ('neue schärfe in islamdebatte,drohungengegen einen kollegen (prof. rohe, nürnberg)', 7), ('ausschreibung juniorprofessuren für islamische theologie, tübingen', 7)]

with open("subjects_lemmatized.pkl", "rb") as f:
    lemmatized_subjects = pickle.load(f)
    lemmatized_subjects = [sub for sub in lemmatized_subjects if sub !=""]

bow(lemmatized_subjects, top=30)

{'Religionswissenschaft': 1301,
 'Ausschreibung': 321,
 'Stellenausschreibung': 319,
 'Neuerscheinung': 234,
 'religionswissenschaftlich': 186,
 'Einladung': 181,
 'Universität': 162,
 'Deutschland': 149,
 'Religionswissenschaftlerinn': 139,
 'Geschichte': 138,
 'Marburg': 117,
 'nochmals': 111,
 'Hinweis': 107,
 'Vorschlag': 106,
 'suchen': 103,
 'Religionswissenschaftler': 88,
 'Nachtrag': 84,
 'interreligiös': 82,
 'Leipzig': 80,
 'Erfurt': 77,
 'Heidelberg': 76,
 'Berlin': 74,
 'Vortrag': 72,
 'Fachverständnis': 72,
 'Schweiz': 67,
 'Frau': 65,
 'Begriff': 64,
 'Verantwortung': 64,
 'islamisch': 62,
 'Konferenz': 60}

# loading document vectors (see Code Appendix)
with open("doc_vecs.pkl", "rb") as f:
    vecs = pickle.load(f)

# vector of the first subject: "Kehrer über Religion"
vecs.iloc[0]

0      0.648040
1      0.448960
2     -1.376667
3     -0.711435
4     -0.765597
         ...   
295    0.497893
296    0.735443
297   -0.823900
298   -1.129867
299    1.687930
Name: Kehrer ueber Scientology, Length: 300, dtype: float32

# vectors of the first five subjects
vecs.iloc[:5]

reducer = umap.UMAP(metric="cosine", n_components=2, random_state=42)
embedding = reducer.fit_transform(vecs)

/mnt/d/environments/ds_lecture_311/lib/python3.11/site-packages/umap/umap_.py:1943: UserWarning: n_jobs value -1 overridden to 1 by setting random_state. Use no seed for parallelism.
  warn(f"n_jobs value {self.n_jobs} overridden to 1 by setting random_state. Use no seed for parallelism.")

embedding[0]

array([ 6.033508, -5.082357], dtype=float32)

plot_embeddings(embedding)

clusterer = hdbscan.HDBSCAN(min_cluster_size=40)
cluster_labels = clusterer.fit_predict(embedding)

vecs_labels = vecs.copy()
vecs_labels["cluster"] = cluster_labels

plot_cluster(embedding, cluster_labels)

vecs_labels.value_counts("cluster")

cluster
 4     2795
 15    2703
-1      700
 11     349
 14     326
 8      171
 12     147
 3      105
 0       96
 9       89
 10      86
 6       52
 1       47
 13      46
 5       45
 2       43
 7       43
Name: count, dtype: int64

vecs_labels[vecs_labels.cluster == 3]["cluster"][:25]

Iahr Congress Durban                  3
Mircea eliaden                        3
conference Detail                     3
Cambridge conference Programm         3
Easr Cambridge conference Programm    3
Easr conference Documents             3
Haeuptling Seattle                    3
conference                            3
Easr conference                       3
Programm                              3
Conference Paris update               3
Iahr Conference Delhi                 3
Vacancy Amsterdam                     3
Paris bergen                          3
charming conference                   3
Shinto wars                           3
Delhi                                 3
Easr congress Congrès                 3
Easr Conference Bergen                3
Easr congress Programm                3
Easr congress Deadline                3
Iahr Weltkongress Tokyo               3
Tokyo                                 3
Iahr Conference Indonesia             3
Sseasr conference                     3
Name: cluster, dtype: int64

vecs_labels[vecs_labels.cluster == 13]["cluster"][:25]

polylzentrisch Religionswissenschaft                           13
Religionswissenschaft                                          13
Sekt angewandt Religionswissenschaft                           13
Religionswissenschaft Magisternebenfach fwd                    13
infoheft Religionswissenschaft religionsgeschichte             13
infoheft Religionswissenschaft                                 13
angewandt Religionswissenschaft Kopftuchurteil                 13
angewandt Religionswissenschaft                                13
studierendenzahlen Religionswissenschaft zuwachsraten          13
praktisch Religionswissenschaft                                13
Verhältnis Religionswissenschaft Religionspädagogik            13
Positionspapier Missionswissenschaft Religionswissenschaft     13
Kopftuchurteil angewandte Religionswissenschaft                13
Doktorandenstelle Religionswissenschaft                        13
Wallraff Religionswissenschaft                                 13
Lektorenstelle Religionswissenschaft                           13
Studierendensymposium Religionswissenschaft                    13
Religionswissenschaft wikipedia                                13
Sozialwissenschaft                                             13
mitarbeiterstell Religionssoziologie                           13
Lectureship Islamwissenschaften                                13
studieneingangstests Religionswissenschaft                     13
Wikipedia Religionswissenschaft                                13
wiki Religionswissenschaft                                     13
Doktorandenstipendi Religionswissenschaft Islamwissenschaft    13
Name: cluster, dtype: int64

# imports
import pickle
import email
import re
from typing import List, Dict
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
from collections import Counter
import pandas as pd
import spacy
from langdetect import detect
import umap
import umap.plot
import hdbscan

# pandas options
pd.set_option('display.max_colwidth', None)

# sns options
sns.set_style("whitegrid")  # Set the background style
sns.set_palette("viridis")  # Set the color palette

# spacy
nlp = spacy.load("de_core_news_md")

/mnt/d/environments/ds_lecture_311/lib/python3.11/site-packages/umap/plot.py:203: NumbaDeprecationWarning: The keyword argument 'nopython=False' was supplied. From Numba 0.59.0 the default is being changed to True and use of 'nopython=False' will raise a warning as the argument will have no effect. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.
  @numba.jit(nopython=False)

with open("../Datasets/Yggdrasill_1997-2023/storage_all.pkl", "rb") as f:
    data = pickle.load(f)

### HELPER FUNCTION FOR get_subject_from_emails()
def clean_subject(subject: str) -> str:
    '''Function to clean an email-subject using RE.
    '''
    CLEANING_PATTERN = re.compile(r"^re:\s+\[ygg\]|^fwd:\s+\[ygg\]|^aw:\s+\[ygg\]|^re:\s+fwd:\s+\[ygg\]|^\[ygg\]|\baw:|\bfwd:|\bfw:|\bre:|\[dolmen\]|\[candide\]|\[ygg\]", re.IGNORECASE)
    subject = re.sub(CLEANING_PATTERN, " ", subject)
    subject = re.sub(r"\s{2,}", " ", subject)
    subject = subject.strip().lower()
    return subject

def get_subject_from_emails(email_dict: Dict[int, Dict[str, Dict[int, email.message.Message]]]) -> Dict:
    '''Extract SUBJECT field from emails retrieved via get_emails_from_folder().

        Parameters
        ----------
            email_dict: Dict[str, Dict[str, Dict[int, email.message.Message]]
                The dictionary with emails, retrieved using get_emails_from_folder().

        Returns
        -------
            (Dict) 
            
            {
                <YEAR: int>: List[str]
            }
    '''
    emails = dict()
    for year in email_dict:
        emails[year] = list()
        for month in email_dict[year]:
            for idx, mail in email_dict[year][month].items():
                subject = mail["subject"]
                if subject != None:
                    emails[year].append(clean_subject(subject))
    return emails

subject_dictionary = get_subject_from_emails(data)

with open("subjects_1997-2023_clean.pkl", "wb") as f:
    pickle.dump(subject_dictionary,f)

# get years/messages from subjects dictionary
years = [str(k) for k,_ in subjects_dict.items()]
messages = [len(v) for _,v in subjects_dict.items()]

def visualize_messages_per_year(years: List[str], messages: List[int]) -> None:
    '''Function to visualize messages distro per year with matplotlib.'''
    sns.lineplot(x=years, y=messages, marker='o', markersize=8, linestyle='-', linewidth=2)
    plt.title(f"Yggdrasill Beiträge/Jahr 1997-2023", fontsize=16)
    plt.xlabel("Jahr", fontsize=13)
    plt.xticks(rotation=45)
    plt.ylabel("Anz. Beiträge", fontsize=13)
    plt.ylim(0, max(messages)+200)
    plt.tight_layout()
    plt.savefig(f"yearly_stats.png")

visualize_messages_per_year(years, messages)

## arithm. Mittel
np.sum(messages)/len(years)

545.9629629629629

## Median
np.median(messages)

497.0

subject_len_dict = {year: [len(betreff) for betreff in messages] for year,messages in subject_dictionary.items()}

all_subject_len = list()

for _,v in subject_len_dict.items():
    all_subject_len.extend(v)

def create_boxplot(msg_len_list: List[int]) -> None:
    '''Function to visualize the distribution of Betreffzeilen-length using a boxplot.'''
    plt.figure(figsize=(8, 6))
    sns.boxplot(x=msg_len_list, color='skyblue')

    # labels and a title
    plt.ylabel("Beiträge", fontsize=14)
    plt.xlabel("Länge", fontsize=14)
    plt.title("Boxplot: Länge der Betreffzeilen", fontsize=16)

    plt.tight_layout()
    plt.savefig("boxplot.png")

def create_violinplot(msg_len_list: List[int]) -> None:
    '''Function to visualize the distribution of Betreffzeilen-length using a violin-plot.'''
    plt.figure(figsize=(8, 6))
    sns.violinplot(x=msg_len_list, color='orchid')

    # labels and a title
    plt.ylabel("Beiträge", fontsize=14)
    plt.xlabel("Länge", fontsize=14)
    plt.title("Violinplot: Länge der Betreffzeilen", fontsize=16)

    plt.tight_layout()
    plt.savefig("violinplot.png")

create_boxplot(all_subject_len)

create_violinplot(all_subject_len)

def filter_subjects_by_length(subjects: List[str], min_len: int) -> np.array:
    '''Function to filter list of subjects to keep only subjects of a certain lengths.'''
    np_subjects = np.array(subjects)
    boolean_subjects = np.array([len(subject) > min_len for subject in np_subjects])
    return np_subjects[boolean_subjects]

#filter_subjects_by_length(all_subjects, 150)

def show_most_frequent_topics(messages: List[str], topics=10):
    '''Function to show list of top subjects.'''
    cnt = Counter(messages)
    df = pd.DataFrame(list(cnt.items()), columns=['Betreff', 'Anzahl'])
    return df.sort_values("Anzahl", ascending=False).iloc[:topics].reset_index().drop(columns=["index"])

#show_most_frequent_topics(all_subjects)

def lemmatize_subject(subject: str) -> str:
    '''Function to lemmatize a text string using spaCy.'''
    text = nlp(subject)
    lemmatized_text = " ".join([word.lemma_ for word in text if not word.is_stop and word.is_alpha])
    return lemmatized_text

# erstellung lemmatisierte subjects
all_subjects_lemmatized = [lemmatize_subject(sub) for sub in all_subjects]

with open("subjects_lemmatized.pkl", "wb") as f:
    pickle.dump(all_subjects_lemmatized,f)

def plot_topics_per_year(subject_year_dict: Dict[int,List[str]], search_terms: List[str]) -> None:
    '''Function to plot how often the terms/topic are mentioned each year + list of top topics.'''
    topic_year_dict = dict()
    topics = list()

    for year,subjects in subject_year_dict.items():
        topic_year_dict[year] = 0
        for subject in subjects:
            for kw in search_terms:
                if kw in subject:
                    topic_year_dict[year] += 1
                    #print(subject) # DEBUG
                    topics.append(subject)
                    break
    # plotting 
    years = [str(y) for y in topic_year_dict.keys()]
    sns.lineplot(x=years, y=topic_year_dict.values(), marker='o', markersize=8, linestyle='-', linewidth=2)
    plt.title(f"Yggdrasill Beiträge/Jahr 1997-2023 mit Themen: {', '.join(search_terms)}", fontsize=16)
    plt.xlabel("Jahr", fontsize=13)
    plt.xticks(rotation=45)
    plt.ylabel("Anz. Beiträge", fontsize=13)
    plt.ylim(-10, max(topic_year_dict.values())+10)
    plt.tight_layout()
    plt.show()

    # show most frequent topics
    cnt = Counter(topics)
    print(cnt.most_common(5))

def bow(subjects: List[str], top=10) -> Counter:
    '''Function to create a bag-of-words from list of texts.'''
    cnt = Counter()
    for sub in subjects:
        cnt.update(sub.split())
    # check if words are really german
    counter = 0
    top_words = dict()

    for word,count in cnt.most_common():
        try:
            lang = detect(word)
        except:
            lang = "fail"
        if lang == "de":
            top_words[word] = count
            counter += 1
            if counter == top:
                break
    return top_words

def doc2vec(subjects_list: List[str]) -> pd.DataFrame:
    '''Function to create subject vectors for each subject line with spaCy.'''
    vec_dict = dict()
    for subject in subjects_list:
        doc = nlp(subject)
        if doc.has_vector:
            vec_dict[subject] = doc.vector
    df = pd.DataFrame.from_dict(vec_dict, orient="index")
    return df

vecs = doc2vec(lemmatized_subjects)

with open("doc_vecs.pkl", "wb") as f:
    pickle.dump(vecs, f)

with open("doc_vecs.pkl", "rb") as f:
    vecs = pickle.load(f)

#vecs

def plot_embeddings(embeddings: np.array) -> None:
    '''Function to plot embeddings from umap.'''
    plt.scatter(embeddings[:, 0], embeddings[:, 1], s=5)
    plt.title('UMAP Embeddings (Yggdrasill Analysis)')
    plt.show()

def plot_cluster(embeddings: np.array, labels: List[str]) -> None:
    '''Function to plot embeddings from umap.'''
    plt.scatter(embeddings[:, 0], embeddings[:, 1], c=labels, cmap='Spectral', s=5)
    plt.colorbar()
    plt.title('HDBSCAN Clustering on UMAP Embeddings (Yggdrasill Analysis)')
    plt.show()

	Betreff	Anzahl
0	heute vor 120 jahren ...	142
1	diskriminierung von religionswissenschaftlerinnen? - vorschlag	101
2	geschichte der religionswissenschaft	83
3	easr code of conduct	70
4	religionswissenschaft	68
5	stellenausschreibung	65
6	fachverständnis der religionswissenschaft	64
7	neuerscheinung	59
8	cfp reminder: rewiring the house of god / ifaith (authors' workshop)	55
9	zum begriff "whiteness" am beispiel von ekemini uwan	53

	0	1	...	298	299
Kehrer ueber Scientology	0.648040	0.448960	...	-1.129867	1.687930
Korrektion	1.462600	1.247500	...	-0.586430	1.139300
Tagung England	2.625885	1.048010	...	1.163675	2.816000
Adresse Fuer beitraege	1.907067	-0.024577	...	-0.892833	1.223500
virtuell Ausflug	0.711900	-1.132605	...	-0.471435	0.979915

Data Science Introduction 2024

Data Science in den Geisteswissenschaften: Eine praxisnahe Einführung¶

Übersicht¶

1. Einführung¶

2. Was ist Data Science?¶

2.1 Google Trends¶

2.2 Definitionen¶

3. Data Science Workflow¶

Zwischenfrage: Was sind eigentlich Daten?¶

4. Beispielprojekt: Analyse der Themen einer religionswissenchaftlichen E-Mail-Liste (Yggdrasill)¶

4.1 Datenbeschaffung¶

4.2 Datentransformation¶

4.3 Explorative Datenanalyse (EDA)¶

4. Analyse¶

4.1 Häufigste Themen¶

4.2 Themenverteilung¶

4.3 Bag-of-Words Analyse (BoW)¶

4.4 Themen-Cluster¶

4.4.1 Vektorisierung (spaCy)¶

4.4.2 Clustering (umap und HDBSCAN)¶

5. Zusammenfassung¶

6. Literatur¶

7. Code Appendix¶

6.1 Cleaning/Transformation¶

6.2 EDA¶

6.2.1 Beiträge/Jahr Plot¶

6.2.2 Statistische Kennzahlen¶

6.2.3 Boxplot / Violinplot¶

6.2.4 Filter Betreff¶

6.4 Analyse¶

6.4.1 Häufigste Themen¶

6.4.2 Lemmatisierung¶

6.4.3 Themen/Jahr¶

6.4.4 BoW¶

6.4.5 Clustering¶