import numpy as np
import pandas as pd

from sklearn.metrics import confusion_matrix
from sklearn import metrics
from sklearn.metrics import confusion_matrix

from keras.preprocessing import sequence
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation
from keras.layers import Embedding
from keras.layers import LSTM
from keras.layers import Conv1D, MaxPooling1D,GlobalAveragePooling1D
from sklearn.feature_extraction.text import CountVectorizer
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.models import Sequential
from keras.layers import Dense, Embedding, LSTM
from sklearn.model_selection import train_test_split
from keras.utils.np_utils import to_categorical
from keras.callbacks import EarlyStopping
from keras.callbacks import ModelCheckpoint
from keras.models import model_from_json
import re
import warnings
warnings.filterwarnings('ignore')
from subprocess import check_output
print(check_output(["ls", "\path"]).decode("utf8"))

import nltk
import os
import gc
from keras.preprocessing import sequence,text
from keras.preprocessing.text import Tokenizer
from keras.models import Sequential
from keras.layers import Dense,Dropout,Embedding,LSTM,Conv1D,GlobalMaxPooling1D,Flatten,MaxPooling1D,GRU,SpatialDropout1D,Bidirectional,Conv2D
from keras.callbacks import EarlyStopping
from tensorflow.keras.utils import to_categorical
from keras.losses import categorical_crossentropy
from tensorflow.keras.optimizers import Adam
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score,confusion_matrix,classification_report,f1_score
import matplotlib.pyplot as plt
import warnings
# Preprocessing
from sklearn import preprocessing
from sklearn.utils import class_weight as cw
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer
from bs4 import BeautifulSoup
# Time
import time
import datetime
from tensorflow.keras.optimizers import RMSprop,Adam
from tensorflow.keras.optimizers import Adam
###########################
from keras.callbacks import ModelCheckpoint, EarlyStopping, TensorBoard, ReduceLROnPlateau
# Visualization
import matplotlib.pyplot as plt
import seaborn as sns
###########################
import string
import re
from nltk.corpus import stopwords
#########################
from tensorflow.keras import layers
from tensorflow.keras import regularizers
from tensorflow.keras.layers import BatchNormalization
from keras.regularizers import l2
from tensorflow.keras.utils import to_categorical

# Date and time
def date_time(x):
    if x==1:
        return 'Timestamp: {:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now())
    if x==2:
        return 'Timestamp: {:%Y-%b-%d %H:%M:%S}'.format(datetime.datetime.now())
    if x==3:
        return 'Date now: %s' % datetime.datetime.now()
    if x==4:
        return 'Date today: %s' % datetime.date.today()

input_directory = r"../input/"
output_directory = r"../output/"

if not os.path.exists(output_directory):
    os.mkdir(output_directory)

figure_directory = "../output/figures"
if not os.path.exists(figure_directory):
    os.mkdir(figure_directory)

file_name_pred_batch = figure_directory+r"/result"
file_name_pred_sample = figure_directory+r"/sample"

data = pd.read_csv('\path\dataset')
data.columns

data = data[['text', 'sentiment']]
data.head()

import nltk
nltk.download('punkt')

import nltk
nltk.download('stopwords')

import nltk
nltk.download('wordnet')

##The_stopword
all_stopwords = stopwords.words('arabic')
sw_list = [
'إذ', 'إذا','إذما', 'اذن', 'إذن', 'أف', 'أقل', 'كتر', 'اجو', 'اتلم', 'كثر', 'اكثروا', 'اكثرة',
'اكثرا', 'أكثر', 'الا', 'ألا', 'إلا', 'التي', 'الذي', 'الذين', 'اللاتي', 'اللائي', 'اللتان', 'اللتيا', 'اللتين', 'اللذان', 'اللذين', 'اللواتي', 'إلى', 'إليك', 'إليكم', 'إليكما', 'إليكم', 'إليكن', 'أم', 'أما', 'أما', 'إما', 'آما', 'ان', 'أن',
 'إن', 'إنا', 'أنا', 'انا', 'انتي', 'أنتي', 'أنت', 'أنتت', 'أنتما', 'أنتم', 'أنتما', 'أنتن', 'أنما', 'إنما', 'إنه', 'أنني', 'آه', 'آها', 'أو', 'آوو', 'اوو ', 'او', 'أولاء', 'أولئك ', 'أوه', 'آي', 'أي', 'أيهم', 'أيتها', 'أيها', 'ايي',
'إي', 'أين', 'أين', 'أينما', 'إيه', 'بخ', 'برا روح', 'بتاع', 'بب', 'بابا', 'بعدين', 'بر', 'بلاي', 'بح', 'بس', 'بعد', 'بعض', 'بك', 'بكي', 'بيكم', 'بكم', 'بكما', 'بكن', 'بل', 'بلى', 'بما', 'بماذا', 'بمن', 'بنا', 'به', 'بها',
 'بهم', 'بهما', 'بهن', 'بي', 'بين', 'بيد', 'تلك', 'تلكم', 'تلكما', 'ته', 'تي', 'تين', 'ثم', 'ثمة', 'جاب', 'جة', 'حط', 'خد', 'رابعه', 'راح', 'رح', 'شال', 'صح', 'طخ', 'عااا', 'كدا', 'ليل', 'حاشا', 'حبذا', 'حتى', 'حيث',
' حيثما', 'حين', 'خلا', 'دون', 'ذا', 'ذات', 'ذاك', 'ذان', 'ذانك',
 'ذلك','ذلكم', 'ذلكما', 'ذلكن', 'ذه', 'ذو', 'ذوا', 'ذواتا', 'ذواتي', 'ذي', 'ذين', 'ذينك', 'ريثما', 'ريث', 'سوف', 'سوو', 'سواء', 'سوا', 'سوى ', 'شتان', 'عدا', 'عسى', 'عل', 'على', 'عبر', 'عنا', 'عليك', 'عليكي', 'عليكما',
 'عليكم', 'عليه', 'علا', ' عما', 'عن', 'عند', 'فإذا', 'فإن', 'فلا', 'فمن', 'في', 'فيم', ' فيما', 'فينكم', ' فينم', 'فين', 'فيهما', 'فيهم', 'فيه', 'فيها', ' قد', ' كأنهم', ' كأن', ' كأنما', ' كأي', ' كأين', ' وكدا', ' كذا', 'كذلك', ' كل',
 ' كلا', ' كلاهما', ' كلتا', ' كلما', ' كليكما', ' كليهما', ' كم', ' كما', ' كي', ' كيت', ' كيف', ' كيفما', ' لا', ' ليالي', ' لا لا لا', ' لسا', 'لاسيما', ' لدى', ' لسه', ' لستي', ' لست', ' لستما', ' لستم', ' لستما', 'لستن', ' لسن'
 ' لسنا', ' لعل', ' لك', ' لكم', ' لكما', ' لكن', ' لكنما', 'لكي', ' لكيلا', ' لم', ' لما', ' لن', ' لنا', ' له', ' لها', ' لهم', ' لهما', 'لهن', ' لو', ' لولا', ' لوما', ' لي', ' لئن', ' ليت', ' ليس', ' ليسا', ' ليست', ' ليستا',
 ' هع', ' هم', ' هاتان', ' هاته', ' هاتي', ' هاتين', ' هاك', ' هاهنا' , 'هذه', ' هذي', ' هذين', ' هكذا', ' هل', ' هلا', ' هم', ' هما', ' هن', ' هنا', 'هناك', ' هنالك', ' هو', ' هؤلاء', 'هي', ' هيا', ' هيت', 'هيهات',
' والذين', ' وإذ', ' وإذا', ' وإن', ' ولا', ' ولكن', ' ولو', ' وما', 'ومن', ' وهو', ' وله', ' واد', ' وه', ' يا', ' ااا', ' أبريل', ' أبو', ' أبٌ ', ' أجل ', ' جمع ', ' أحد ', ' أخبر', ' أخذ', ' أخو ', ' أخٌ ', ' أربع ', 'أحمر',
' أرح ', ' أربعا', ' أبكي', ' أرى', ' أسكن', ' أصبح', ' أصلا', ' أضحى', ' أطعم', ' أكتوبر', ' أعطى', ' أعلم', ' أغسطس', ' أفريل', ' أفعل به', ' أفٍّ ', ' ألف', ' ألفى', ' ألوف', ' أم', ' أما', 'أمام', ' أمامك', ' أمامكي', ' أمامكما',
' أمامكم', ' أمامكَ', ' أمد', ' أمس', ' أمسى', ' مسا', ' امسا', ' تج', ' تحوّل', ' تحط', ' تخذ', ' ترك', ' تاني', ' تسع', ' تسعة', ' تسعين', ' تشرين', ' تعسا', ' تعلَّم', ' تفعلان', ' تفعلا', ' تفعلين', ' تفعلوا', ' تفعلون', ' تفعلين',
' جانفي', ' جعل', ' جلل', ' جمعة', ' جميع', ' جنيه', ' جوان', ' جويلية',
 ' جير', ' ء', ' سابع', ' سادس', ' سبت', ' سبتمبر', ' سبحان', ' سبع', ' سبح', ' سير ', ' ست', ' ستة', ' ستكون', ' ستمئة', ' ستمائة', ' ستون', ' ستين', ' سحقا', ' سرا', ' سرعان', ' سقى ', ' سمعا', ' سنة', ' سنتيم', ' سنين',
' سنوات', ' سوف', ' سوى', ' سين', ' صارِ', ' صباح', ' صبر', ' صبرا', ' صبرو', ' صدقا', ' صراحة', ' صفر', ' صهٍ', ' صهْ', ' ض', ' ضاد', ' ضحوة', ' ضد', ' ضامن', ' ضمن', ' ط', ' طاء', 'طاق', 'ّ طالما', ' طرا', ' طفق', 'ّ طَق', ' عشرة',
' عشرات', 'عشرون', ' عشرين', ' عل', ' علق', ' علم', ' ه ', ' ما انفك', ' ما برح', ' مادام', ' مادامو'
, ' مادمنا', ' ماذا', ' ماما', ' متر', ' مرة',' مرات', ' مارس', ' مازال', ' مازالو', ' مازالنا', ' ماي', ' مايزال', ' مايزل', ' مايو', ' مافتئ', ' متىِ', ' مثل', ' مذ', ' مرات', ' مرّة', ' مساء', ' يفعلون', ' يكون', ' يلي', ' يمكن', ' يمين', ' ين', ' يناير',
' يورو', ' يوليو', ' يومان', ' يونيو', ' يوان'
]
all_stopwords.extend(sw_list)

## pre-processing steps
'''
'''
    # Punctiations

punctuations = '''`÷×؛<>_()*&^%][ـ،/:"؟.,'{}~¦+|!”…“–ـ''' + string.punctuation

stop_words = stopwords.words()

    #diacritics
arabic_diacritics = re.compile("""
                             ّ    | # Shadda
                             َ    | # Fatha
                             ً    | # Tanwin Fath
                             ُ    | # Damma
                             ٌ    | # Tanwin Damm
                             ِ    | # Kasra
                             ٍ    | # Tanwin Kasr
                             ْ    | # Sukun
                             ـ     # Tatwil/Kashida
                         """, re.VERBOSE)

def preprocess(text ):

    '''
    Arabic text is entered as a string, and the preprocessed text is output.
    '''

    #remove punctuations
    translator = str.maketrans('', '', punctuations)
    text = text.translate(translator)

    # remove Tashkeel
    text = re.sub(arabic_diacritics, '', text )

    #words and corresponding emoticon
    text = re.sub("مرتبك", "o.o", text)
    text = re.sub("سعيد ", "(:", text)
    text = re.sub("وجهة سعيد جدا", "^_^", text)
    text = re.sub("غاضب", "):<", text)
    text = re.sub("بكاء", ")'：", text)
    text = re.sub("شيطاني", "(:3", text)
    text = re.sub("مللائكي", "o :) ", text)
    text = re.sub("سعيد", "8^) ", text)

    #remove normalization
    text = re.sub("[إأآا]", "ا", text)
    text = re.sub("ى", "ي", text)
    text = re.sub("ؤ", "ء", text)
    text = re.sub("ئ", "ء", text)
    text = re.sub("ة", "ه", text)
    text = re.sub("گ", "ك", text)

    text = ' '.join(word for word in text.split() if word not in stop_words)

    return text

data['text'] = data['text'].apply(preprocess)
print(data.head(5))

#spilt and encode our data
X = data.text
Y = data.sentiment

label_encoder = LabelEncoder()

Y = label_encoder.fit_transform(Y)

Y = to_categorical(Y)

#split data & calculate maxword and max lenx
X_train, X_test, Y_train, Y_test = train_test_split(X,Y, test_size = 0.20, random_state = 123)

max_words = len(set(" ".join(X_train).split()))
#max_len = X_train.apply(lambda x: len(x)).max()

#max_words = 2000
max_len = 100
#, 150, 120, 100
max_words, max_len

num_unique_word=max_words
MAX_REVIEW_LEN=max_len

#Hyperparameter value
max_features = num_unique_word
max_words = MAX_REVIEW_LEN
batch_size = 128
#batch_size = 500
epochs = 10
num_classes=2
embedding_size =128
# Convolution
kernel_size = 5
filters = 64
pool_size = 4

#Tokenize Text
tokenizer = Tokenizer(num_words=max_features)
tokenizer.fit_on_texts(list(X_train))
X_train = tokenizer.texts_to_sequences(X_train)
X_test = tokenizer.texts_to_sequences(X_test)

#sequence padding
X_train = sequence.pad_sequences(X_train, maxlen=max_words)
X_test = sequence.pad_sequences(X_test, maxlen=max_words)
print(X_train.shape,X_test.shape)

maxlen=max_words

#Output Configuration
main_model_dir = output_directory + r"models/"
main_log_dir = output_directory + r"logs/"

try:
    os.mkdir(main_model_dir)
except:
    print("Could not create main model directory")

try:
    os.mkdir(main_log_dir)
except:
    print("Could not create main log directory")

model_dir = main_model_dir + time.strftime('%Y-%m-%d %H-%M-%S') + "/"
log_dir = main_log_dir + time.strftime('%Y-%m-%d %H-%M-%S')

try:
    os.mkdir(model_dir)
except:
    print("Could not create model directory")

try:
    os.mkdir(log_dir)
except:
    print("Could not create log directory")

model_file = model_dir + "{epoch:02d}-val_acc-{val_acc:.2f}-val_loss-{val_loss:.2f}.hdf5"

print("Settting Callbacks")

checkpoint = ModelCheckpoint(
    model_file,
    monitor='val_acc',
    save_best_only=True)

early_stopping = EarlyStopping(
    monitor='val_loss',
    patience=3,
    verbose=1,
    restore_best_weights=True)

reduce_lr = ReduceLROnPlateau(
    monitor='val_loss',
    factor=0.5,
    patience=1,
    verbose=1)

callbacks = [checkpoint, reduce_lr, early_stopping]

callbacks = [early_stopping]

print("Set Callbacks at ", date_time(5))

def plot_performance(history=None, figure_directory=None, ylim_pad=[0, 0]):
    xlabel = 'Epoch'
    legends = ['Training', 'Validation']

    plt.figure(figsize=(20, 5))

    y1 = history.history['accuracy']
    y2 = history.history['val_accuracy']

    min_y = min(min(y1), min(y2))-ylim_pad[0]
    max_y = max(max(y1), max(y2))+ylim_pad[0]

    plt.subplot(121)

    plt.plot(y1)
    plt.plot(y2)

    plt.title('Model Accuracy\n'+date_time(1), fontsize=17)
    plt.xlabel(xlabel, fontsize=15)
    plt.ylabel('Accuracy', fontsize=15)
    plt.ylim(min_y, max_y)
    plt.legend(legends, loc='upper left')
    plt.grid()

    y1 = history.history['loss']
    y2 = history.history['val_loss']

    min_y = min(min(y1), min(y2))-ylim_pad[1]
    max_y = max(max(y1), max(y2))+ylim_pad[1]

    plt.subplot(122)

    plt.plot(y1)
    plt.plot(y2)

    plt.title('Model Loss\n'+date_time(1), fontsize=17)
    plt.xlabel(xlabel, fontsize=15)
    plt.ylabel('Loss', fontsize=15)
    plt.ylim(min_y, max_y)
    plt.legend(legends, loc='upper left')
    plt.grid()
    if figure_directory:
        plt.savefig(figure_directory+"/history")

    plt.show()

# word embedding
def get_coefs(word, *arr):
    return word, np.asarray(arr, dtype='float32')

def get_embed_mat(EMBEDDING_FILE, max_features,embed_dim):
    # word vectors
    embeddings_index = dict(get_coefs(*o.rstrip().rsplit(' ')) for o in open(EMBEDDING_FILE, encoding='utf8'))
    print('Found %s word vectors.' % len(embeddings_index))

    # embedding matrix
    word_index = tokenizer.word_index
    num_words = min(max_features, len(word_index) + 1)
    all_embs = np.stack(embeddings_index.values()) #for random init
    embedding_matrix = np.random.normal(all_embs.mean(), all_embs.std(),
                                        (num_words, embed_dim))
    for word, i in word_index.items():
        if i >= max_features:
            continue
        embedding_vector = embeddings_index.get(word)
        if embedding_vector is not None:
            embedding_matrix[i] = embedding_vector
    max_features = embedding_matrix.shape[0]

    return embedding_matrix

#MC1 Model
model20= Sequential()
model20.add(Embedding(max_features, embedding_size, input_length=maxlen))
model20.add(Dropout(0.5))
model20.add(Conv1D(512, kernel_size=3,padding='same',activation='relu',strides=1))
model20.add(GlobalAveragePooling1D())
model20.add(Dropout(0.5))
model20.add(Dense(
    units=256,
    kernel_regularizer=regularizers.l1_l2(l1=1e-5, l2=1e-4),
    bias_regularizer=regularizers.l2(1e-4),
    activity_regularizer=regularizers.l2(1e-5)
))
model20.add(BatchNormalization())
model20.add(Dropout(0.5))
model20.add(Dense(2,activation='softmax'))
model20.compile(loss='categorical_crossentropy',optimizer='adam',metrics=['accuracy'])
model20.summary()

loss = 'categorical_crossentropy'
# loss = 'binary_crossentropy'
metrics = ['accuracy']

start_time = time.time()
print(date_time(1))
history3=model20.fit(X_train, Y_train, validation_data=(X_test, Y_test),epochs=20, batch_size=batch_size, verbose=1)
elapsed_time = time.time() - start_time
elapsed_time = time.strftime("%H:%M:%S", time.gmtime(elapsed_time))
print("\nElapsed Time: " + elapsed_time)
print("Completed Model Trainning", date_time(1))
#epochs=50,100,200,500

#Visualization
plot_performance(history=history3)

#MC2 Model
model4= Sequential()
model4.add(Embedding(max_features, embedding_size, input_length=maxlen))
model4.add(Conv1D(128,kernel_size=3,padding='same',activation='relu'))
model4.add(MaxPooling1D(pool_size=2))
model4.add(Conv1D(64,kernel_size=3,padding='same',activation='relu'))
model4.add(MaxPooling1D(pool_size=2))
model4.add(Dropout(0.25))
model4.add(SpatialDropout1D(0.25))
model4.add(Bidirectional(GRU(128)))
model4.add(Dropout(0.5))
model4.add(Flatten())
model4.add(Dense(128,activation='relu'))
model4.add(Dropout(0.5))
model4.add(Dense(2,activation='softmax'))
model4.compile(loss='categorical_crossentropy',optimizer=Adam(lr=0.001),metrics=['accuracy'])
model4.summary()

loss = 'categorical_crossentropy'
# loss = 'binary_crossentropy'
metrics = ['accuracy']

start_time = time.time()
print(date_time(1))
history3=model4.fit(X_train, Y_train, validation_data=(X_test, Y_test),epochs=20, batch_size=batch_size, verbose=1)
elapsed_time = time.time() - start_time
elapsed_time = time.strftime("%H:%M:%S", time.gmtime(elapsed_time))
print("\nElapsed Time: " + elapsed_time)
print("Completed Model Trainning", date_time(1))

#Visualization
plot_performance(history=history3)


'''
Description      :  Directly Using Filter, Map, Reduce In Lambda
Function Date    :  21 Feb 2021
Function Author  :  Prasad Dangare
Input            :  Int
Output           :  Int

'''

import functools

def main():

    arr = []
    print("Enter Number Of Elements : ")
    size = int(input())

    for i in range(size):
        print("Enter Elements Number : ", i + 1)
        no = int(input())
        arr.append(no)

    print("Your Entered Data Is : ", arr)

    newdata = list(filter(lambda no: (no % 2 == 0), arr)) # newdata = MarvellousFilter(arr)
    print("After Filtering Data Is : ", newdata)

    newdata1 = list(map(lambda no: no + 2, newdata))#newdata1 = MarvellousMap(newdata)
    print("After Map Is : ", newdata1)

    output = functools.reduce(lambda no1, no2: no1 + no2, newdata1)#output = MarvellousReduce(newdata1)
    print("After Reduce Result Is : ", output)

if __name__ == "__main__":
    main()


from .base import PlatformEventLoop

from pyglet.libs.win32 import _kernel32, _user32, types, constants
from pyglet.libs.win32.types import *

class Win32EventLoop(PlatformEventLoop):
    def __init__(self):
        super(Win32EventLoop, self).__init__()

        self._next_idle_time = None

        # Force immediate creation of an event queue on this thread -- note
        # that since event loop is created on pyglet.app import, whatever
        # imports pyglet.app _must_ own the main run loop.
        msg = types.MSG()
        _user32.PeekMessageW(ctypes.byref(msg), 0,
                             constants.WM_USER, constants.WM_USER,
                             constants.PM_NOREMOVE)

        self._event_thread = _kernel32.GetCurrentThreadId()

        self._wait_objects = []
        self._recreate_wait_objects_array()

        self._timer_proc = types.TIMERPROC(self._timer_proc_func)
        self._timer = _user32.SetTimer(0, 0, constants.USER_TIMER_MAXIMUM, self._timer_proc)

        self._timer_func = None

    def add_wait_object(self, obj, func):
        self._wait_objects.append((obj, func))
        self._recreate_wait_objects_array()

    def remove_wait_object(self, obj):
        for i, (_object, _) in enumerate(self._wait_objects):
            if obj == _object:
                del self._wait_objects[i]
                break
        self._recreate_wait_objects_array()

    def _recreate_wait_objects_array(self):
        if not self._wait_objects:
            self._wait_objects_n = 0
            self._wait_objects_array = None
            return

        self._wait_objects_n = len(self._wait_objects)
        self._wait_objects_array = \
            (HANDLE * self._wait_objects_n)(*[o for o, f in self._wait_objects])

    def start(self):
        if _kernel32.GetCurrentThreadId() != self._event_thread:
            raise RuntimeError('EventLoop.run() must be called from the same ' +
                               'thread that imports pyglet.app')

        self._timer_func = None
        self._polling = False
        self._allow_polling = True

    def step(self, timeout=None):
        self.dispatch_posted_events()

        msg = types.MSG()
        if timeout is None:
            timeout = constants.INFINITE
        else:
            timeout = int(timeout * 1000) # milliseconds

        result = _user32.MsgWaitForMultipleObjects(
            self._wait_objects_n,
            self._wait_objects_array,
            False,
            timeout,
            constants.QS_ALLINPUT)
        result -= constants.WAIT_OBJECT_0

        if result == self._wait_objects_n:
            while _user32.PeekMessageW(ctypes.byref(msg),
                                       0, 0, 0, constants.PM_REMOVE):
                _user32.TranslateMessage(ctypes.byref(msg))
                _user32.DispatchMessageW(ctypes.byref(msg))
        elif 0 <= result < self._wait_objects_n:
            obj, func = self._wait_objects[result]
            func()

        # Return True if timeout was interrupted.
        return result <= self._wait_objects_n

    def notify(self):
        # Nudge the event loop with a message it will discard.  Note that only
        # user events are actually posted.  The posted event will not
        # interrupt the window move/size drag loop -- it seems there's no way
        # to do this.
        _user32.PostThreadMessageW(self._event_thread, constants.WM_USER, 0, 0)

    def set_timer(self, func, interval):
        if func is None or interval is None:
            interval = constants.USER_TIMER_MAXIMUM
        else:
            interval = int(interval * 1000) # milliseconds

        self._timer_func = func
        _user32.SetTimer(0, self._timer, interval, self._timer_proc)

    def _timer_proc_func(self, hwnd, msg, timer, t):
        if self._timer_func:
            self._timer_func()


import os
from time import sleep
from selenium import webdriver
from selenium.webdriver.common.by import By

url = "https://letskodeit.teachable.com/courses"

class DynamicXPathFormat():
    def test(self):
        driver = webdriver.Firefox()
        driver.maximize_window()
        driver.get(url)
        driver.implicitly_wait(5)

        """    # Login - Lecture "How to click and type on a web element"
        login_link = driver.find_element(By.LINK_TEXT, "Login")
        login_link.click()

        email_field = driver.find_element(By.XPATH, "//input[@id='email']")
        email_field.send_keys("test@email.com")

        password_field = driver.find_element(By.ID, "password")
        password_field.send_keys("abcabc")

        login_field = driver.find_element(By.XPATH, "//input[contains(@value,'Login')]")
        login_field.click() """

        # Search for courses -> You don't need to search the course
        # You can select it without searching also
        searchBox = driver.find_element(By.ID, "search-courses")
        searchBox.send_keys("JavaScript")

        # Select Course
        _course = "//div[contains(@class,'course-listing-title') and contains(text(),'{0}')]"
        _courseLocator = _course.format("JavaScript for beginners")

        courseElement = driver.find_element(By.XPATH, _courseLocator)
        courseElement.click()
        sleep(5)

        driver.quit()

ff = DynamicXPathFormat()
ff.test()


import subprocess
import time
import whois
from ip2geotools.databases.noncommercial import DbIpCity
import dns.resolver
import urllib.request
import requests
import re
import time
import platform
import socket
from bs4 import BeautifulSoup as bs
from Wappalyzer import Wappalyzer, WebPage
import warnings
import sys
import os
import phonenumbers
from phonenumbers import geocoder, carrier
from colorama import Fore, Style, init
init()

def banner():
    print(Fore.RED + """
  ____        _     _         __        __   _             _
 / ___| _ __ (_) __| | ___ _ _\ \      / /__| |__   __   _/ |
 \___ \| '_ \| |/ _` |/ _ \ '__\ \ /\ / / _ \ '_ \  \ \ / / |
  ___) | |_) | | (_| |  __/ |   \ V  V /  __/ |_) |  \ V /| |
 |____/| .__/|_|\__,_|\___|_|    \_/\_/ \___|_.__/    \_/ |_|
       |_|
    \t\t[Coded By: BlackSheep4]
    """)
    print(Style.RESET_ALL)

def main():
    print('\n' + "Welcome to SpiderWeb v1 - Information Gathering Tool" + '\n')

    print(Fore.RED + "\t[" + Fore.GREEN + "01" + Fore.RED + "] " + Fore.WHITE + "Operative System Detection")
    print(Fore.RED + "\t[" + Fore.GREEN + "02" + Fore.RED + "] " + Fore.WHITE + "WHOIS Information")
    print(Fore.RED + "\t[" + Fore.GREEN + "03" + Fore.RED + "] " + Fore.WHITE + "Target Geolocation")
    print(Fore.RED + "\t[" + Fore.GREEN + "04" + Fore.RED + "] " + Fore.WHITE + "DNS Dumping")
    print(Fore.RED + "\t[" + Fore.GREEN + "05" + Fore.RED + "] " + Fore.WHITE + "Robots.txt Extraction")
    print(Fore.RED + "\t[" + Fore.GREEN + "06" + Fore.RED + "] " + Fore.WHITE + "Get all the links from the target")
    print(Fore.RED + "\t[" + Fore.GREEN + "07" + Fore.RED + "] " + Fore.WHITE + "Server Detection")
    print(Fore.RED + "\t[" + Fore.GREEN + "08" + Fore.RED + "] " + Fore.WHITE + "Server Technologies Detection")
    print(Fore.RED + "\t[" + Fore.GREEN + "09" + Fore.RED + "] " + Fore.WHITE + "Phone Information Extractor")
    print(Fore.RED + "\n\t[" + Fore.GREEN + "10" + Fore.RED + "] " + Fore.WHITE + "Exit")
    print('\n')
    option = int(input("[+] Choose an option >> "))
    print(Style.RESET_ALL)

    int(option)

    if option == 1:
        target = str(input("[+] Target Domain >> "))
        url = f'http://www.{target}/'
        system = platform.system()
        target2 = socket.gethostbyname(target)

        if system == 'Linux':
            try:
                print('[!] GETTING OPERATIVE SYSTEM')
                time.sleep(1)
                ping = subprocess.Popen(f"ping -c 1 {target2}",shell=True,  stdout=subprocess.PIPE)
                out = ping.communicate()[0] #access stdoutdata
                out = out.split() #split by spaces
                out = str(out)

                ttl = re.search(r'ttl=(\d+)', out).group(1)
                ttl= int(ttl)

                if ttl >= 0 and ttl <= 64:
                    print(Fore.RED + "[" + Fore.GREEN + "+" + Fore.RED + "] " + Fore.WHITE + "Server Operative System: " + Fore.GREEN + "LINUX")
                    print(Style.RESET_ALL)
                elif ttl >= 65 and ttl <= 128:
                    print(Fore.RED + "[" + Fore.GREEN + "+" + Fore.RED + "] " + Fore.WHITE + "Server Operative System: " + Fore.GREEN + "WINDOWS")
                    print(Style.RESET_ALL)
                else:
                    print(Fore.RED + "[" + Fore.GREEN + "+" + Fore.RED + "] " + Fore.WHITE + "Server Operative System:" + Fore.GREEN + "UNKOWN")
                    print(Style.RESET_ALL)

                input('\n' + "Press any key to continue...")
                os.system('clear')

            except (AttributeError):
                print('\n')
                print(Fore.RED + "[" + Fore.GREEN + "!" + Fore.RED + "] " + Fore.WHITE + 'An error occurred. It is possible that the server is down or that you have misspelled the IP or domain.')

        else:
            print('Please use this script in Linux OS')
            input('\n' + "Press any key to continue...")
            os.system('clear')

    elif option == 2:

        target = str(input("[+] Target Domain >> "))

        w = whois.whois(target)
        print(Fore.RED + "[" + Fore.GREEN + "!" + Fore.RED + "] " + Fore.WHITE + f"Getting WHOIS Information from {target}")
        time.sleep(0.5)
        print(Fore.RED + "[" + Fore.GREEN + "+" + Fore.RED + "] " + Fore.WHITE + "Domain Names: " + str(w.get("domain_name")))
        time.sleep(0.3)
        print(Fore.RED + "[" + Fore.GREEN + "+" + Fore.RED + "] " + Fore.WHITE + "Domain Name Servers: " + str(w.get("name_servers"))[2:-2])
        time.sleep(0.3)
        print(Fore.RED + "[" + Fore.GREEN + "+" + Fore.RED + "] " + Fore.WHITE + "Organization: " + str(w.get("org")))
        time.sleep(0.3)
        print(Fore.RED + "[" + Fore.GREEN + "+" + Fore.RED + "] " + Fore.WHITE + "Registration: " + str(w.get("registrar")))
        time.sleep(0.3)
        print(Fore.RED + "[" + Fore.GREEN + "+" + Fore.RED + "] " + Fore.WHITE + "Emails: " + str(w.get("emails")))
        time.sleep(0.3)
        print(Fore.RED + "[" + Fore.GREEN + "+" + Fore.RED + "] " + Fore.WHITE + "DNSSec: " + str(w.get("dnssec")))
        time.sleep(0.3)
        print(Fore.RED + "[" + Fore.GREEN + "+" + Fore.RED + "] " + Fore.WHITE + "Country: " + str(w.get("country")))
        time.sleep(0.3)
        print(Fore.RED + "[" + Fore.GREEN + "+" + Fore.RED + "] " + Fore.WHITE + "State: " + str(w.get("state")))
        time.sleep(0.3)
        print(Fore.RED + "[" + Fore.GREEN + "+" + Fore.RED + "] " + Fore.WHITE + "City: " + str(w.get("city")))
        time.sleep(0.3)
        print(Fore.RED + "[" + Fore.GREEN + "+" + Fore.RED + "] " + Fore.WHITE + "Zipcode: " + str(w.get("zipcode")))
        time.sleep(0.3)
        print(Fore.RED + "[" + Fore.GREEN + "+" + Fore.RED + "] " + Fore.WHITE + "Address: " + str(w.get("address")))

        input('\n' + "Press any key to continue...")
        os.system('clear')

    elif option == 3:

        target = str(input("[+] Target Domain >> "))
        target2 = socket.gethostbyname(target)

        response = DbIpCity.get(target2, api_key="free")
        print(Fore.RED + "[" + Fore.GREEN + "+" + Fore.RED + "] " + Fore.WHITE + "Tracking IP Location")
        time.sleep(0.5)
        latitude = response.latitude
        print(Fore.RED + "[" + Fore.GREEN + "+" + Fore.RED + "] " + Fore.WHITE + "Latitude: " + str(response.latitude))
        time.sleep(0.3)
        longitude = response.longitude
        print(Fore.RED + "[" + Fore.GREEN + "+" + Fore.RED + "] " + Fore.WHITE + "Longitude: " + str(response.longitude))
        time.sleep(0.3)
        print("[+] Google Maps Track: " + f"https://google.es/maps/@{latitude},{longitude},15z?hl=en")

        input('\n' + "Press any key to continue...")
        os.system('clear')

    elif option == 4:

        target = str(input("[+] Target Domain >> "))

        print(Fore.RED + "[" + Fore.GREEN + "+" + Fore.RED + "] " + Fore.WHITE + "Starting DNS Dumping")
        mx = dns.resolver.resolve(target, 'MX')
        soa = dns.resolver.resolve(target, 'SOA')
        ns = dns.resolver.resolve(target, 'NS')
        time.sleep(0.5)

        print(Fore.RED + "[" + Fore.GREEN + "!" + Fore.RED + "] " + Fore.WHITE + "DUMPING MX SERVERS")
        for data in mx:
            print(Fore.RED + "[" + Fore.GREEN + "+" + Fore.RED + "] " + Fore.WHITE + "MX Servers: " + str(data.exchange) + "\t" + " PREFERENCE: " + str(data.preference))
            time.sleep(0.3)
        print(Fore.RED + "[" + Fore.GREEN + "!" + Fore.RED + "] " + Fore.WHITE + "DUMPING DOMAIN NAME SERVERS")
        for data in ns:
            print(Fore.RED + "[" + Fore.GREEN + "+" + Fore.RED + "] " + Fore.WHITE + "Domain Name Servers: " + str(data.target))
            time.sleep(0.3)

        print(Fore.RED + "[" + Fore.GREEN + "!" + Fore.RED + "] " + Fore.WHITE + "DUMPING PRIMARY ZONE DOMAIN")
        for data in soa:
            print(Fore.RED + "[" + Fore.GREEN + "+" + Fore.RED + "] " + Fore.WHITE + "DNS Primary Zone: " + str(data.mname))
            time.sleep(0.3)

        input('\n' + "Press any key to continue...")
        os.system('clear')

    elif option == 5:

        target = str(input("[+] Target Domain >> "))

        print(Fore.RED + "[" + Fore.GREEN + "!" + Fore.RED + "] " + Fore.WHITE + "Trying to get robots.txt")

        robots = f'http://www.{target}/robots.txt'
        file = "robots.txt"

        request = requests.get(robots)
        if request.status_code == 200:
            r = urllib.request.urlopen(robots)
            f = open(file, 'wb')
            f.write(r.read())
            f.close
            print(Fore.RED + "[" + Fore.GREEN + "+" + Fore.RED + "] " + Fore.WHITE + "File Successfully Downloaded!")
        else:
            print(Fore.RED + "[" + Fore.GREEN + "+" + Fore.RED + "] " + Fore.WHITE + f"Seems that {target} doesn´t have any robots.txt")

        input('\n' + "Press any key to continue...")
        os.system('clear')

    elif option == 6:

        target = str(input("[+] Target Domain >> "))
        url = f'http://www.{target}/'

        print(Fore.RED + "[" + Fore.GREEN + "!" + Fore.RED + "] " + Fore.WHITE + f"Getting all links from {target} ")

        req = requests.get(url)
        soup = bs(req.text, 'html.parser') #transform request to unicode and parse to HTML

        for link in soup.find_all('a'):
            time.sleep(0.2)
            print(link.get('href'))

        input('\n' + "Press any key to continue...")
        os.system('clear')

    elif option == 7:

        target = str(input("[+] Target Domain >> "))
        url = f'http://www.{target}/'

        r = requests.get(url)
        server = r.headers['Server']
        print(Fore.RED + "[" + Fore.GREEN + "!" + Fore.RED + "] " + Fore.WHITE + "Getting the Type of Server... ")
        time.sleep(0.5)
        print(Fore.RED + "[" + Fore.GREEN + "+" + Fore.RED + "] " + Fore.WHITE + "Server: " + str(server))

        input('\n' + "Press any key to continue...")
        os.system('clear')

    elif option == 8:

        target = str(input("[+] Target Domain >> "))
        url = f'http://www.{target}/'

        print(Fore.RED + "[" + Fore.GREEN + "!" + Fore.RED + "] " + Fore.WHITE + "Detecting Technologies...")
        #STUPID WARNING XD
        warnings.filterwarnings("ignore", message="""Caught 'unbalanced parenthesis at position 119' compiling regex""", category=UserWarning )

        wappalyzer = Wappalyzer.latest()
        webpage = WebPage.new_from_url(url)
        technologies = wappalyzer.analyze(webpage)
        for t in technologies:
            time.sleep(0.5)
            print(Fore.RED + "[" + Fore.GREEN + "+" + Fore.RED + "] " + Fore.WHITE + "Technology Detected: {}".format(t))

        input('\n' + "Press any key to continue...")
        os.system('clear')

    elif option == 9:
        target = str(input("[+] Phone Target (+12345678901) >> "))
        phone = phonenumbers.parse(target)
        carrier2 = carrier.name_for_number(phone, 'en')
        region = geocoder.description_for_number(phone, 'en')
        possible = phonenumbers.is_possible_number(phone)

        print(Fore.RED + "[" + Fore.GREEN + "+" + Fore.RED + "] " + Fore.WHITE + "Phone Number Validation: " + str(possible))
        print(Fore.RED + "[" + Fore.GREEN + "+" + Fore.RED + "] " + Fore.WHITE + "General Information: "+ str(phone))
        print(Fore.RED + "[" + Fore.GREEN + "+" + Fore.RED + "] " + Fore.WHITE + "Carrier: " + str(carrier2))
        print(Fore.RED + "[" + Fore.GREEN + "+" + Fore.RED + "] " + Fore.WHITE + "Country: " + str(region))

        input('\n' + "Press any key to continue...")
        os.system('clear')

    elif option == 10:
        print(Fore.RED + "[" + Fore.GREEN + "!" + Fore.RED + "] " + Fore.WHITE + "Exiting")
        sys.exit()

    else:
        print("WTF are you trying dude? Select a number of the menu!")

if __name__ == '__main__':
    try:
        while True:
            banner()
            main()
    except:
        print('[!] Stopping the scanner...')
        sys.exit(1)


#%matplotlib inline

"""
Matplotlib Animation Example

author: Jake Vanderplas
email: vanderplas@astro.washington.edu
website: http://jakevdp.github.com
license: BSD
Please feel free to use and modify this, but keep the above information. Thanks!
"""

import numpy as np
from matplotlib import pyplot as plt
from matplotlib import animation, rc
rc('animation', html='html5')

from mpl_toolkits.mplot3d import axes3d

from IPython.display import HTML

def toXYZ(vectors):
    # transform vectors to X, Y, Z form
    vectors = np.array(vectors).T
    #X, Y, Z = vectors

    origins = np.zeros(shape=vectors.shape)

    return vectors, origins # X, Y, Z

def draw_vectors_2d(vectors, origins, fig):
    X, Y = origins
    U, V = vectors

    ax = fig.add_subplot(111)
    eps = 0.1
    ax.set_xlim([U.min() - eps, U.max() + eps])
    ax.set_ylim([V.min() - eps, V.max() + eps])

    ax.set_title("2D vectors",fontsize=14)
    ax.set_xlabel("x",fontsize=12)
    ax.set_ylabel("y",fontsize=12)
    ax.grid(True,linestyle='--',color='0.75')

    ax.quiver(X, Y, U, V, color='rrggbbkkccmmyy', angles='xy', scale_units='xy', scale=1)

    plt.show()

def draw_vectors_3d(vectors, origins, fig):
    # First set up the figure, the axis, and the plot element we want to animate
    ax = fig.add_subplot(111, projection='3d')
    line, = ax.plot([], [], lw=2)

    # initialization function: plot the background of each frame
    def init():
        line.set_data([], [])
        return line,

    # animation function.  This is called sequentially
    def animate(i):
        #x = np.linspace(0, 2, 1000)
        #y = np.sin(2 * np.pi * (x - 0.01 * i))
        #line.set_data(x, y)
        angle = 2
        ax.view_init(elev=None, azim=i*angle)

        return line,

    a1 = [1, 0, 0]
    b1 = [0, 1, 0]
    c1 = [0, 0, 1]
    d1 = np.sum([a1, b1, c1], axis=1)

    # transform vectors to X, Y, Z form
    #vectors = np.array([a1, b1, c1, d1]).T
    #print(vectors)
    U, W, V = vectors
    #origins = np.zeros(shape=vectors.shape)
    X, Y, Z = origins

    #ax.plot(X, Y, Z)
    ax.quiver(X, Y, Z, U, V, W, color='rgbkcmy') # up to 7 different vector colors

    eps = 0.1
    ax.set_xlim([U.min() - eps, U.max() + eps])
    ax.set_ylim([V.min() - eps, V.max() + eps])
    ax.set_zlim([V.min() - eps, V.max() + eps])

    ax.set_title("3D vectors",fontsize=14)
    ax.set_xlabel("x",fontsize=12)
    ax.set_ylabel("y",fontsize=12)
    ax.set_zlabel("z",fontsize=12)
    ax.grid(True,linestyle='--',color='0.75')

    # call the animator.  blit=True means only re-draw the parts that have changed.
    anim = animation.FuncAnimation(fig, animate, init_func=init,
                                   frames=180, interval=100, repeat=True, blit=True)

    # save the animation as an mp4.  This requires ffmpeg or mencoder to be
    # installed.  The extra_args ensure that the x264 codec is used, so that
    # the video can be embedded in html5.  You may need to adjust this for
    # your system: for more information, see
    # http://matplotlib.sourceforge.net/api/animation_api.html
    anim.save('basic_animation.mp4', fps=30, extra_args=['-vcodec', 'libx264'])

    #plt.show()

    plt.close()
    return HTML(anim.to_html5_video())

Machine Learning Summer School/Session1/LinealAlgebra/Linearity and vectors/drawing.py


from pathlib import Path
from unittest.mock import MagicMock, patch
from typing import Iterable

import pytest

import sciety_labs.utils.cache as cache_module
from sciety_labs.utils.cache import (
    DiskSingleObjectCache,
    InMemorySingleObjectCache
)

@pytest.fixture(name='monotonic_mock')
def _monotonic_mock() -> Iterable[MagicMock]:
    with patch.object(cache_module, 'monotonic') as mock:
        yield mock

class TestInMemorySingleObjectCache:
    def test_should_get_loaded_value(self):
        cache = InMemorySingleObjectCache(max_age_in_seconds=10)
        result = cache.get_or_load(load_fn=lambda: 'value_1')
        assert result == 'value_1'

    def test_should_not_call_load_function_multiple_times(self):
        cache = InMemorySingleObjectCache(max_age_in_seconds=10)
        load_fn = MagicMock(name='load_fn')
        load_fn.return_value = 'value_1'
        cache.get_or_load(load_fn=load_fn)
        result = cache.get_or_load(load_fn=load_fn)
        assert result == 'value_1'
        assert load_fn.call_count == 1

    def test_should_reload_if_max_age_reached(self, monotonic_mock: MagicMock):
        cache = InMemorySingleObjectCache[str](
            max_age_in_seconds=60
        )
        load_fn = MagicMock(name='load_fn')
        load_fn.side_effect = ['value_1', 'value_2']
        monotonic_mock.return_value = 100
        cache.get_or_load(load_fn=load_fn)
        monotonic_mock.return_value = 200
        result = cache.get_or_load(load_fn=load_fn)
        assert result == 'value_2'
        assert load_fn.call_count == 2

class TestDiskSingleObjectCache:
    def test_should_get_loaded_value(self, tmp_path: Path):
        cache = DiskSingleObjectCache[str](
            file_path=tmp_path / 'cache.file',
            max_age_in_seconds=10
        )
        result = cache.get_or_load(load_fn=lambda: 'value_1')
        assert result == 'value_1'

    def test_should_not_call_load_function_multiple_times(self, tmp_path: Path):
        cache = DiskSingleObjectCache[str](
            file_path=tmp_path / 'cache.file',
            max_age_in_seconds=10
        )
        load_fn = MagicMock(name='load_fn')
        load_fn.side_effect = ['value_1', 'value_2']
        cache.get_or_load(load_fn=load_fn)
        result = cache.get_or_load(load_fn=load_fn)
        assert result == 'value_1'
        assert load_fn.call_count == 1

    def test_should_call_load_function_after_cache_file_is_removed(self, tmp_path: Path):
        cache = DiskSingleObjectCache[str](
            file_path=tmp_path / 'cache.file',
            max_age_in_seconds=10
        )
        load_fn = MagicMock(name='load_fn')
        load_fn.side_effect = ['value_1', 'value_2']
        cache.get_or_load(load_fn=load_fn)
        cache.file_path.unlink()
        result = cache.get_or_load(load_fn=load_fn)
        assert result == 'value_2'
        assert load_fn.call_count == 2


#!/usr/bin/env python
# coding: utf-8

# In[2]:

import pickle

modelo = open('preco','rb')
lm_new = pickle.load(modelo)
modelo.close()

area = 38
garagem = 2
banheiros = 4
lareira = 4
marmore = 0
andares = 1

entrada = [[area, garagem, banheiros, lareira, marmore, andares]]

print('$ {0:.2f}'.format(lm_new.predict(entrada)[0]))

# In[4]:

# Importando bibliotecas
from ipywidgets import widgets, HBox, VBox
from IPython.display import display

# Criando os controles do formulário
area = widgets.Text(description="Área")
garagem = widgets.Text(description="Garagem")
banheiros = widgets.Text(description="Banheiros")
lareira = widgets.Text(description="Lareira")
marmore = widgets.Text(description="Mármore?")
andares = widgets.Text(description="Andares?")

botao = widgets.Button(description="Simular")

# Posicionando os controles
left = VBox([area, banheiros, marmore])
right = VBox([garagem, lareira, andares])
inputs = HBox([left, right])

# Função de simulação
def simulador(sender):
    entrada=[[
                float(area.value if area.value else 0),
                float(garagem.value if garagem.value else 0),
                float(banheiros.value if banheiros.value else 0),
                float(lareira.value if lareira.value else 0),
                float(marmore.value if marmore.value else 0),
                float(andares.value if andares.value else 0)
             ]]
    print('$ {0:.2f}'.format(lm_new.predict(entrada)[0]))

# Atribuindo a função "simulador" ao evento click do botão
botao.on_click(simulador)

# In[5]:

display(inputs, botao)

# In[ ]:


import pygame
from projects.alien_invasion.settings import Settings
from projects.alien_invasion.ship import Ship
from projects.alien_invasion import game_functions as gf
from pygame.sprite import Group
from projects.alien_invasion.game_stats import GameStats
from projects.alien_invasion.button import Button
from projects.alien_invasion.score_board import ScoreBoard

def run_game() -> None:
    """Initializing new game"""
    ai_settings = Settings()
    pygame.init()  # Inside settings
    screen = pygame.display.set_mode((ai_settings.screen_width, ai_settings.screen_height))  # Create screen area
    pygame.display.set_caption("Alien Invasion")  # Create window title
    play_button = Button(ai_settings, screen, "Play")
    stats = GameStats(ai_settings)
    sb = ScoreBoard(ai_settings, screen, stats)
    ship = Ship(screen, ai_settings)  # create ship
    bullets = Group()  # Create group to save bullets
    aliens = Group()
    gf.create_fleet(ai_settings, screen, ship, aliens)
    while True:  # Game cycle
        gf.check_events(ai_settings, screen, stats, sb, play_button, ship, aliens, bullets)
        if stats.game_active:
            ship.update()
            gf.update_bullets(ai_settings, screen, stats, sb, ship, aliens, bullets)
            gf.update_aliens(ai_settings, screen, stats, sb, ship, aliens, bullets)
        gf.update_screen(ai_settings, screen, stats, sb, ship, aliens, bullets, play_button)

run_game()


import torch

torch.autograd.set_detect_anomaly(True)

class Embedding:
    """
    Embedding class embeds x (input) to (x, sin(2^k x), cos(2^k x), ...)
    """

    def __init__(self, num_freqs: int, scalar_factor: float = 1.0) -> None:

        self._input_dims = 3
        self._num_freqs = num_freqs
        self._max_freq_log2 = num_freqs - 1
        self._periodic_functions = [torch.sin, torch.cos]
        self._scalar_factor = scalar_factor

        self._embedding_functions = []
        self._output_dim = 0

        self._create_embedding_functions()

    def _create_embedding_functions(self):

        # Original raw input "x" is also included in the output
        original_input_fcn = lambda x: x
        self._embedding_functions.append(original_input_fcn)
        self._output_dim += self._input_dims

        # Log scale for sampling
        freq_bands = 2. ** torch.linspace(0., self._max_freq_log2, steps=self._num_freqs)

        for freq in freq_bands:
            for p_fn in self._periodic_functions:
                periodic_fcn = lambda x, periodic_fn=p_fn, freq=freq: periodic_fn(x * freq)
                self._embedding_functions.append(periodic_fcn)
                self._output_dim += self._input_dims

    @property
    def output_dim(self) -> int:
        return self._output_dim

    def embed(self, inputs: torch.Tensor) -> torch.Tensor:
        """
        Run embedding, by passing inputs through all embedding functions and concatenating them.
        """
        return torch.cat([fcn(inputs / self._scalar_factor) for fcn in self._embedding_functions], -1)


from itertools import product

bin_sizes = [4, 8, 16, 32, 64, 128, 512, 1024]
mask_sizes = [1, 2, 4, 8, 16, 32, 64, 128, "_random"]
combinations = list(product(bin_sizes, mask_sizes))
machines = {
    "starariel-0": [],
    "starariel-1": [],
    "starariel-2": [],
    "starariel-3": [],
    "stardale-0": [],
    "stardale-1": [],
    "stardale-2": [],
    "stardale-3": [],
}
# divide the combinations into 8 groups

for i, machine in enumerate(machines):
    machines[machine] = combinations[i::8]

with open("scripts/train.sh", "w") as f:
    f.write("#!/bin/bash\n")
    f.write("if [ \"$1\" == \"--setup\" ]; then\n")
    f.write("\tcd /disk/scratch/s1764494\n")
    f.write("\tgcloud storage cp gs://datasets-cdminix/timit.zip .\n")
    f.write("\tunzip timit.zip\n")
    f.write("\tmv timit data/timit_0\n")
    f.write("\tcp -r data/timit_0 data/timit_1\n")
    f.write("\tcp -r data/timit_0 data/timit_2\n")
    f.write("\tcp -r data/timit_0 data/timit_3\n")
    f.write("\tgcloud storage cp gs://datasets-cdminix/bu_radio.zip .\n")
    f.write("\tunzip bu_radio.zip\n")
    f.write("\tmv bu_radio data/bu_radio_0\n")
    f.write("\tcp -r data/bu_radio_0 data/bu_radio_1\n")
    f.write("\tcp -r data/bu_radio_0 data/bu_radio_2\n")
    f.write("\tcp -r data/bu_radio_0 data/bu_radio_3\n")
    f.write("\tcd /disk/scratch/s1764494/masked-prosody-modeling-evaluation\n")
    f.write("\tgcloud storage cp -R gs://masked-prosody-model/checkpoints .\n")
    f.write("\tmv checkpoints mpm_checkpoints\n")
    f.write("fi\n")
    for machine, combs in machines.items():
        # use --machine to specify the machine
        f.write('if [ "$1" == "--machine" ] && [ "$2" == "{}" ]; then\n'.format(machine))
        gpu_num = machine.split("-")[1]
        burn_preamble = f'\tCUDA_VISIBLE_DEVICES={gpu_num} BURN_PATH="/disk/scratch/s1764494/data/bu_radio_{gpu_num}" HF_DATASETS_CACHE="/disk/scratch/s1764494/data/hf_{gpu_num}" '
        timit_preamble = f'\tCUDA_VISIBLE_DEVICES={gpu_num} TIMIT_PATH="/disk/scratch/s1764494/data/timit_{gpu_num}" HF_DATASETS_CACHE="/disk/scratch/s1764494/data/hf_{gpu_num}" '
        ravdess_preamble = f'\tCUDA_VISIBLE_DEVICES={gpu_num} HF_DATASETS_CACHE="/disk/scratch/s1764494/data/hf_{gpu_num}" '
        linear_command = 'python scripts/train_{dataset}.py configs/mpm_linear.yml --run_name {dataset}_bin{bin}_mask{mask}_linear_{num} --mpm_bin_size {bin} --mpm_mask_size {mask} --use_mpm --mpm_layer 7'
        conformer_command = 'python scripts/train_{dataset}.py configs/mpm_conformer.yml --run_name {dataset}_bin{bin}_mask{mask}_conformer --mpm_bin_size {bin} --mpm_mask_size {mask} --use_mpm --mpm_layer 7'
        for bin_size, mask_size in combs:
            for dataset in ["burn", "timit", "ravdess"]:
                if dataset == "burn":
                    preamble = burn_preamble
                elif dataset == "timit":
                    preamble = timit_preamble
                elif dataset == "ravdess":
                    preamble = ravdess_preamble
                f.write(preamble + linear_command.format(bin=bin_size, mask=mask_size, dataset=dataset, num=1) + " --overwrite_data\n")
                f.write(preamble + linear_command.format(bin=bin_size, mask=mask_size, dataset=dataset, num=2) + "\n")
                f.write(preamble + linear_command.format(bin=bin_size, mask=mask_size, dataset=dataset, num=3) + "\n")
                f.write(preamble + conformer_command.format(bin=bin_size, mask=mask_size, dataset=dataset) + "\n")
        f.write("fi\n")


from __future__ import print_function, division
import torch
from torchvision import models, transforms
import os
from PIL import Image
import json
import scipy.io as scio

class myDataset(object):
    def __init__(self,path,figures, transforms):
        self.transforms = transforms
        self.path=path
        self.figures=figures

    def __getitem__(self, idx):
        img = Image.open(os.path.join(self.path,self.figures[idx])).convert("RGB")

        if self.transforms is not None:
            img = self.transforms(img)

        return img, self.figures[idx]

    def __len__(self):
        return len(self.figures)

data_transforms = {
    'pred': transforms.Compose([
        transforms.Resize((224,224)),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ]),
}

if __name__ == '__main__':

    figpath='./example/'
    figures=['class_fatigue1.jpg','class_fatigue2.jpg','class_nonfatigue1.jpg','class_nonfatigue2.jpg']

    dataset = myDataset(figpath,figures, data_transforms['pred'])
    dataloaders = {'pred': torch.utils.data.DataLoader(dataset)}
    device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')

    model = models.resnet50(pretrained=True)
    model.load_state_dict(torch.load('best.pth'))
    model.eval()
    data=[]
    with torch.no_grad():
        for i, (input, name) in enumerate(dataloaders['pred']):
            input = input.to(device)
            output = model(input)
            _, pred = torch.max(output, 1)
            data.append([str(name),int(pred)])

    with open('pred.json','w') as f:
        json.dump(data,f)


from Project.classes.CsvWriter import CSVWriter
from .Parser import Parser
from bricsagentapplication.model.models import Publication, Organization
from .Service import Service
from bricsagentapplication.cache.Cache import Cache
from webdriver_manager.firefox import GeckoDriverManager
from selenium import webdriver
from selenium.webdriver.firefox.options import Options

class Agent:
    def __init__(self):
        self.service = Service()
        self.cache = Cache()
        self.csv_writer = CSVWriter()
        self.parser = Parser()
        self.in_progress = False
        self.links_count = 0
        self.current_link_number = 0
        self.current_country = ""

    def __new__(cls):
        if not hasattr(cls, 'instance'):
            print('creating new agent')
            cls.instance = super(Agent, cls).__new__(cls)
        return cls.instance

    def create_driver(self):
        opts = Options()
        opts.headless = True
        return webdriver.Firefox(options=opts, executable_path=GeckoDriverManager().install())

    def fill_db_for_country(self, country):
        self.in_progress = True
        self.current_link_number = 0
        self.current_country = country

        driver = self.create_driver()
        self.parser.driver = driver

        links, links_journals = self.parser.get_btns_links_and_journals(country)
        self.csv_writer.write_links_journals_functions_files(links, links_journals)

        links_journals = self.csv_writer.read_links_journals_file(country)
        self.links_count = len(links_journals)

        for link_to_button in links_journals:
            self.current_link_number += 1
            print(country + ", link: ", link_to_button)
            try:
                Publication.objects.get(link_to_btn=link_to_button)
            except Publication.DoesNotExist:
                try:
                    journal_name = links_journals[link_to_button]
                    publication_info = self.parser.get_publication_information(link_to_button, journal_name)
                    if publication_info is not None:
                        publication_info.country = country
                        self.service.save_publication_info(publication_info)
                except KeyError:
                    my_file = open("exceptions.txt", "a")
                    my_file.write('\n we do not know journal of ' + link_to_button + " is\n ")
                    my_file.close()
                    continue

        print("Done")
        self.in_progress = False
        driver.quit()
        # для тестирования: выбираем журнал

        # ! springer
        # if 'Nature' in journal_name:

        # ! sciencedirect
        # if 'Earth and Planetary Science Letters' in journal_name or 'Geochimica et Cosmochimica Acta' in journal_name or 'Water Research' in journal_name:

        # ! acs
        # if 'Organic Letters' in journal_name or 'Macromolecules' in journal_name or 'Journal of the American Chemical Society' in journal_name or 'Inorganic Chemistry' in journal_name or 'The Journal of Physical Chemistry Letters' in journal_name or 'Analytical Chemistry' in journal_name or 'Environmental Science and Technology' in journal_name or 'ACS Nano' in journal_name or 'Nano Letters' in journal_name:
        #
        # ! rsc
        # if 'Chemical Communications' in journal_name or 'Chemical Science' in journal_name:
        #
        # ! pnasorg
        # if 'Proceedings of the National Academy of Sciences of the United States of America' in journal_name:
        #
        # ! advances science
        # if 'Science' == journal_name or 'Science Advances' == journal_name or 'Science Translational Medicine' == journal_name:
        #
        # ! journalsplos
        # if 'PLOS Biology' == journal_name or 'PLOS Genetics' == journal_name:
        #
        # ! jneurosci
        # if 'Journal of Neuroscience' == journal_name or 'Cancer Research' == journal_name:
        #
        # ! jbcorg
        # if 'Journal of Biological Chemistry' == journal_name:
        #
        #  ! jciorg
        #  if 'Journal of Clinical Investigation' == journal_name:
        #
        #  ! cell
        #  if 'Cell' == journal_name:
        #
        #  ! royal
        #  if 'Proceedings of the Royal Society B' == journal_name:
        #
        #  ! meta aip scitation
        #  if 'Applied Physics Letters' == journal_name:
        #
        #         # ? academic oup
        #         # if 'Monthly Notices of the Royal Astronomical Society: Letters' == journal_name:
        #
        #         # ? iop science
        #         # if 'The Astrophysical Journal Letters' == journal_name:
        #
        # ! meta ru press
        # if 'Journal of Experimental Medicine' == journal_name or 'Journal of Cell Biology' == journal_name:
        #
        # ! genome
        # if 'Genome Research' == journal_name:
        #
        #         # ? planet cell
        #         # if 'The Plant Cell' == journal_name:
        #
        #         # ? aanda
        #         # if 'Astronomy & Astrophysics' in journal_name:
        #
        # ! pubs geo science world
        #  https: // www.natureindex.com / article / 10.1130 / g47008.1
        #  https: // www.natureindex.com / article / 10.1130 / g46740.1
        #  https: // www.natureindex.com / article / 10.1130 / g46923.1
        #  https: // www.natureindex.com / article / 10.1130 / g46180.1
        #  if 'Geology' in journal_name:
        #
        # ! wiley
        #  if 'Advanced Functional Materials' == journal_name or 'Advanced Materials' == journal_name or 'Geophysical Research Letters' == journal_name or 'Journal of Geophysical Research: Atmospheres' == journal_name or 'Journal of Geophysical Research: Solid Earth' == journal_name or 'Ecology Letters' == journal_name or 'Angewandte Chemie International Edition' == journal_name:
        #
        # ! journals aps
        # if 'Physical Review' in journal_name:
        #
        # ! elife
        # if 'eLife' == journal_name:
        #
        # ! cell ajhg
        #  if 'American Journal of Human Genetics' == journal_name or 'Cancer Cell' == journal_name or 'Neuron' == journal_name or 'Cell Metabolism ' == journal_name or 'Cell Host & Microbe' == journal_name or 'Cell Stem Cell' == journal_name or 'Current Biology' == journal_name or 'Immunity' == journal_name:

        # self.update_cache(self.parser)

    # def get_top_unis_names(self, country):
    #     additional_driver = self.create_driver()
    #     topOrganizationsNames = self.parser.get_top_unis_names(country, additional_driver)
    #     additional_driver.quit()
    #
    #     organizationsTop = []
    #
    #     for org in topOrganizationsNames:
    #         organizationsTop.append({'id': University.objects.get(name=org).id, 'name': org})
    #     return organizationsTop

    def get_filling_progress(self):
        if self.links_count != 0:
            progress = self.current_link_number/self.links_count*100
        else:
            progress = 0
        return {"country": self.current_country, "progress": progress, "in_progress": self.in_progress}

    # def update_cache(self, parser):
        # todo заменить
        # for country in ['Brazil', 'Russia', 'India', 'China', 'South Africa']:
        #     self.cache.cache_countries_orgs_top(parser.get_country_top_unis_names(country), country)


# ------------------------------------------------------------------------------
# Post-processing to get instance and panoptic segmentation results.
# Written by Bowen Cheng (bcheng9@illinois.edu)
# ------------------------------------------------------------------------------

import torch
import torch.nn.functional as F

from .semantic_post_processing import get_semantic_segmentation

__all__ = ['find_instance_center', 'get_instance_segmentation', 'get_panoptic_segmentation']

def find_instance_center(ctr_hmp, threshold=0.1, nms_kernel=3, top_k=None):
    """
    Find the center points from the center heatmap.
    Arguments:
        ctr_hmp: A Tensor of shape [N, 1, H, W] of raw center heatmap output, where N is the batch size,
            for consistent, we only support N=1.
        threshold: A Float, threshold applied to center heatmap score.
        nms_kernel: An Integer, NMS max pooling kernel size.
        top_k: An Integer, top k centers to keep.
    Returns:
        A Tensor of shape [K, 2] where K is the number of center points. The order of second dim is (y, x).
    """
    if ctr_hmp.size(0) != 1:
        raise ValueError('Only supports inference for batch size = 1')

    # thresholding, setting values below threshold to -1
    ctr_hmp = F.threshold(ctr_hmp, threshold, -1)

    # NMS
    nms_padding = (nms_kernel - 1) // 2
    ctr_hmp_max_pooled = F.max_pool2d(ctr_hmp, kernel_size=nms_kernel, stride=1, padding=nms_padding)
    ctr_hmp[ctr_hmp != ctr_hmp_max_pooled] = -1

    # squeeze first two dimensions
    ctr_hmp = ctr_hmp.squeeze()
    assert len(ctr_hmp.size()) == 2, 'Something is wrong with center heatmap dimension.'

    # find non-zero elements
    ctr_all = torch.nonzero(ctr_hmp > 0)
    if top_k is None:
        return ctr_all
    elif ctr_all.size(0) < top_k:
        return ctr_all
    else:
        # find top k centers.
        top_k_scores, _ = torch.topk(torch.flatten(ctr_hmp), top_k)
        return torch.nonzero(ctr_hmp > top_k_scores[-1])

def group_pixels(ctr, offsets):
    """
    Gives each pixel in the image an instance id.
    Arguments:
        ctr: A Tensor of shape [K, 2] where K is the number of center points. The order of second dim is (y, x).
        offsets: A Tensor of shape [N, 2, H, W] of raw offset output, where N is the batch size,
            for consistent, we only support N=1. The order of second dim is (offset_y, offset_x).
    Returns:
        A Tensor of shape [1, H, W] (to be gathered by distributed data parallel).
    """
    if offsets.size(0) != 1:
        raise ValueError('Only supports inference for batch size = 1')

    offsets = offsets.squeeze(0)
    height, width = offsets.size()[1:]

    # generates a coordinate map, where each location is the coordinate of that loc
    y_coord = torch.arange(height, dtype=offsets.dtype, device=offsets.device).repeat(1, width, 1).transpose(1, 2)
    x_coord = torch.arange(width, dtype=offsets.dtype, device=offsets.device).repeat(1, height, 1)
    coord = torch.cat((y_coord, x_coord), dim=0)

    ctr_loc = coord + offsets
    ctr_loc = ctr_loc.reshape((2, height * width)).transpose(1, 0)

    # ctr: [K, 2] -> [K, 1, 2]
    # ctr_loc = [H*W, 2] -> [1, H*W, 2]
    ctr = ctr.unsqueeze(1)
    ctr_loc = ctr_loc.unsqueeze(0)

    # distance: [K, H*W]
    distance = torch.norm(ctr - ctr_loc, dim=-1)

    # finds center with minimum distance at each location, offset by 1, to reserve id=0 for stuff
    instance_id = torch.argmin(distance, dim=0).reshape((1, height, width)) + 1
    return instance_id

def get_instance_segmentation(sem_seg, ctr_hmp, offsets, thing_list, threshold=0.1, nms_kernel=3, top_k=None,
                              thing_seg=None):
    """
    Post-processing for instance segmentation, gets class agnostic instance id map.
    Arguments:
        sem_seg: A Tensor of shape [1, H, W], predicted semantic label.
        ctr_hmp: A Tensor of shape [N, 1, H, W] of raw center heatmap output, where N is the batch size,
            for consistent, we only support N=1.
        offsets: A Tensor of shape [N, 2, H, W] of raw offset output, where N is the batch size,
            for consistent, we only support N=1. The order of second dim is (offset_y, offset_x).
        thing_list: A List of thing class id.
        threshold: A Float, threshold applied to center heatmap score.
        nms_kernel: An Integer, NMS max pooling kernel size.
        top_k: An Integer, top k centers to keep.
        thing_seg: A Tensor of shape [1, H, W], predicted foreground mask, if not provided, inference from
            semantic prediction.
    Returns:
        A Tensor of shape [1, H, W] (to be gathered by distributed data parallel).
        A Tensor of shape [1, K, 2] where K is the number of center points. The order of second dim is (y, x).
    """
    if thing_seg is None:
        # gets foreground segmentation
        thing_seg = torch.zeros_like(sem_seg)
        for thing_class in thing_list:
            thing_seg[sem_seg == thing_class] = 1

    ctr = find_instance_center(ctr_hmp, threshold=threshold, nms_kernel=nms_kernel, top_k=top_k)
    if ctr.size(0) == 0:
        return torch.zeros_like(sem_seg), ctr.unsqueeze(0)
    ins_seg = group_pixels(ctr, offsets)
    return thing_seg * ins_seg, ctr.unsqueeze(0)

def merge_semantic_and_instance(sem_seg, ins_seg, label_divisor, thing_list, stuff_area, void_label):
    """
    Post-processing for panoptic segmentation, by merging semantic segmentation label and class agnostic
        instance segmentation label.
    Arguments:
        sem_seg: A Tensor of shape [1, H, W], predicted semantic label.
        ins_seg: A Tensor of shape [1, H, W], predicted instance label.
        label_divisor: An Integer, used to convert panoptic id = semantic id * label_divisor + instance_id.
        thing_list: A List of thing class id.
        stuff_area: An Integer, remove stuff whose area is less tan stuff_area.
        void_label: An Integer, indicates the region has no confident prediction.
    Returns:
        A Tensor of shape [1, H, W] (to be gathered by distributed data parallel).
    Raises:
        ValueError, if batch size is not 1.
    """
    # In case thing mask does not align with semantic prediction
    pan_seg = torch.zeros_like(sem_seg) + void_label
    thing_seg = ins_seg > 0
    semantic_thing_seg = torch.zeros_like(sem_seg)
    for thing_class in thing_list:
        semantic_thing_seg[sem_seg == thing_class] = 1

    # keep track of instance id for each class
    class_id_tracker = {}

    # paste thing by majority voting
    instance_ids = torch.unique(ins_seg)
    for ins_id in instance_ids:
        if ins_id == 0:
            continue
        # Make sure only do majority voting within semantic_thing_seg
        thing_mask = (ins_seg == ins_id) & (semantic_thing_seg == 1)
        if torch.nonzero(thing_mask).size(0) == 0:
            continue
        class_id, _ = torch.mode(sem_seg[thing_mask].view(-1, ))
        if class_id.item() in class_id_tracker:
            new_ins_id = class_id_tracker[class_id.item()]
        else:
            class_id_tracker[class_id.item()] = 1
            new_ins_id = 1
        class_id_tracker[class_id.item()] += 1
        pan_seg[thing_mask] = class_id * label_divisor + new_ins_id

    # paste stuff to unoccupied area
    class_ids = torch.unique(sem_seg)
    for class_id in class_ids:
        if class_id.item() in thing_list:
            # thing class
            continue
        # calculate stuff area
        stuff_mask = (sem_seg == class_id) & (~thing_seg)
        area = torch.nonzero(stuff_mask).size(0)
        if area >= stuff_area:
            pan_seg[stuff_mask] = class_id * label_divisor

    return pan_seg

def get_panoptic_segmentation(sem, ctr_hmp, offsets, thing_list, label_divisor, stuff_area, void_label,
                              threshold=0.1, nms_kernel=3, top_k=None, foreground_mask=None):
    """
    Post-processing for panoptic segmentation.
    Arguments:
        sem: A Tensor of shape [N, C, H, W] of raw semantic output, where N is the batch size, for consistent,
            we only support N=1. Or, a processed Tensor of shape [1, H, W].
        ctr_hmp: A Tensor of shape [N, 1, H, W] of raw center heatmap output, where N is the batch size,
            for consistent, we only support N=1.
        offsets: A Tensor of shape [N, 2, H, W] of raw offset output, where N is the batch size,
            for consistent, we only support N=1. The order of second dim is (offset_y, offset_x).
        thing_list: A List of thing class id.
        label_divisor: An Integer, used to convert panoptic id = semantic id * label_divisor + instance_id.
        stuff_area: An Integer, remove stuff whose area is less tan stuff_area.
        void_label: An Integer, indicates the region has no confident prediction.
        threshold: A Float, threshold applied to center heatmap score.
        nms_kernel: An Integer, NMS max pooling kernel size.
        top_k: An Integer, top k centers to keep.
        foreground_mask: A Tensor of shape [N, 2, H, W] of raw foreground mask, where N is the batch size,
            we only support N=1. Or, a processed Tensor of shape [1, H, W].
    Returns:
        A Tensor of shape [1, H, W] (to be gathered by distributed data parallel), int64.
    Raises:
        ValueError, if batch size is not 1.
    """
    if sem.dim() != 4 and sem.dim() != 3:
        raise ValueError('Semantic prediction with un-supported dimension: {}.'.format(sem.dim()))
    if sem.dim() == 4 and sem.size(0) != 1:
        raise ValueError('Only supports inference for batch size = 1')
    if ctr_hmp.size(0) != 1:
        raise ValueError('Only supports inference for batch size = 1')
    if offsets.size(0) != 1:
        raise ValueError('Only supports inference for batch size = 1')
    if foreground_mask is not None:
        if foreground_mask.dim() != 4 and foreground_mask.dim() != 3:
            raise ValueError('Foreground prediction with un-supported dimension: {}.'.format(sem.dim()))

    if sem.dim() == 4:
        semantic = get_semantic_segmentation(sem)
    else:
        semantic = sem

    if foreground_mask is not None:
        if foreground_mask.dim() == 4:
            thing_seg = get_semantic_segmentation(foreground_mask)
        else:
            thing_seg = foreground_mask
    else:
        thing_seg = None

    instance, center = get_instance_segmentation(semantic, ctr_hmp, offsets, thing_list,
                                                 threshold=threshold, nms_kernel=nms_kernel, top_k=top_k,
                                                 thing_seg=thing_seg)
    panoptic = merge_semantic_and_instance(semantic, instance, label_divisor, thing_list, stuff_area, void_label)

    return panoptic, center


import data as d
import numpy as np

def contributors(df, subsets):
  print('Breakdown of volunteers by number of projects contributed to')
  print('=' * len('Breakdown of volunteers by number of projects contributed to'))

  #Get a series listing projects contributed to, indexed by pseudonym and project
  #Although the values in this series are relevant to the problem, we never
  #actually use them -- operations on the index do everything that we need. So we
  #could store any value at all in here.
  projects = df.groupby(['pseudonym', 'project'])['project'].unique()

  #Get a series listing number of projects contributed to, indexed by pseudonym
  counts = df.groupby('pseudonym')['project'].nunique()

  c3 = counts[counts == 3]
  c2 = counts[counts == 2]
  c1 = counts[counts == 1]

  print(f'{"Contributors to all projects:":41}',       f'{len(c3):9}')
  print(f'{"Contributors to exactly 2 projects:":41}', f'{len(c2):9}')
  print(f'{"Contributors to exactly 1 project:":41}',  f'{len(c1):9}')

  print('-' * 30)

  mono_total = 0
  for project in d.HMS, d.SB, d.RBGE:
    #Starting with volunteers who contributed to 1 project, filter down to just the project of interest.
    x = projects.loc[c1.index] #Volunteers contributing to just one project
    x = x.index.get_level_values('project') == project #All rows with project of interest
    x = np.count_nonzero(x) #in this context, counts the Trues
    mono_total += x
    header = f'{project} only:'
    print(f'{header:41} {x:9}')
  print('Total mono contributors:', mono_total, '(should equal contributors to exactly one project, above)')

  print('-' * 30)

  pair_total = 0
  for pair, exclusion in [[[d.HMS, d.SB], d.RBGE], [[d.HMS, d.RBGE], d.SB], [[d.SB, d.RBGE], d.HMS]]:
    x = projects.loc[c2.index] #Volunteers contributing to exactly 2 projects
    x = projects.reindex(c2.index, level = 0)
    x = x[x.index.get_level_values('project') != exclusion]

    #Now each pseudonym that appears twice has contributed to both projects
    #As we started with all pseudonyms that appear exactly twice (x = project.loc[c2.index]), now
    #all pseudonyms appear once or twice. So by marking one instance of each duplicate as True, and
    #everything else as False, we get an array containing a number of Trues that is equal to the number
    #of contributors to each project.
    x = x.index.get_level_values('pseudonym').duplicated()
    x = np.count_nonzero(x) #in this context, counts the Trues

    pair_total += x
    header = f'{pair[0]} and {pair[1]} only:'
    print(f'{header:41} {x:9}')
  print('Total pair contributors:', pair_total, '(should equal contributors to exactly two projects, above)')

  print('-' * 30)

  for project in d.HMS, d.SB, d.RBGE:
    heading = f'Total contributors to {project}:'
    total = np.count_nonzero(projects.index.get_level_values('project') == project)
    print(f'{heading:41}', total)

    x = df[df.project == project] #All classifications on current project
    x = x.pseudonym.value_counts() #Number of classifications per volunteer on current project

    m_count = np.count_nonzero(x == 1)
    heading = f'Mono  contributors to {project}:'
    print(f'{heading:41}', m_count)

    p_count = np.count_nonzero(x != 1)
    heading = f'Multi contributors to {project}:'
    print(f'{heading:41}', p_count)

    assert m_count + p_count == total
    print('Mono + multi ==', m_count + p_count, f'(should equal total {total})')
#This is a way to do it all with one series, indexed by pseudonym and project, with values
#equal to the number of projects that the pseudonym has contributed to. It might not be any
#more efficient than the two-series solution above, and has the further disadvantage that we
#have to divide the counts[counts == 3] equivalent by 3, and counts[counts == 2] equivalent by
#2, as there will be one row for each pseudonum,project combination.
#count = df.groupby('pseudonym').project.nunique()
#count.name = 'proj_count'
#projects = df.join(count, on = 'pseudonym').set_index(['pseudonym', 'project'])['proj_count']
#dup_mask = projects.index.duplicated()
#unique_mask = ~dup_mask
#projects = projects[unique_mask]
#counts = projects