import re
from io import StringIO
from pathlib import Path

import emoji
import joblib
import sklearn
import unicodedata
import numpy as np
import pandas as pd
import streamlit as st

from functools import partial
from emoji import get_emoji_regexp
from flashtext import KeywordProcessor
from sklearn.base import BaseEstimator, TransformerMixin

st.set_page_config(
    page_title="ABSA Restaurant",
    page_icon="🍣"
)

HASHTAG = 'hashtag'

class TextCleanerBase(BaseEstimator, TransformerMixin):
    def __init__(self):
        super().__init__()

        # Find emojis
        emoji = get_emoji_regexp()

        # Create preprocessing function
        self.remove_emoji      = partial(emoji.sub, '')
        self.normalize_unicode = partial(unicodedata.normalize, 'NFC')

    def fit(self, X, y=None):
        return self

    def transform(self, X):
        if not isinstance(X, pd.Series):
            X = pd.Series(X)

        return X.apply(str.lower) \
                .apply(self.remove_emoji) \
                .apply(self.normalize_unicode)

class TextCleaner(TextCleanerBase):
    def __init__(self):
        super().__init__()

        # Find hashtag
        hashtag = re.compile('#\S+')

        # Find price tags
        pricetag = '((?:(?:\d+[,\.]?)+) ?(?:nghìn đồng|đồng|k|vnd|d|đ))'
        pricetag = re.compile(pricetag)

        # Find special characters
        specialchar = r"[\"#$%&'()*+,\-.\/\\:;<=>@[\]^_`{|}~\n\r\t]"
        specialchar = re.compile(specialchar)

        # Spelling correction
        rules = {
            "òa":["oà"], "óa":["oá"], "ỏa":["oả"], "õa":["oã"], "ọa":["oạ"],
            "òe":["oè"], "óe":["oé"], "ỏe":["oẻ"], "õe":["oẽ"], "ọe":["oẹ"],
            "ùy":["uỳ"], "úy":["uý"], "ủy":["uỷ"], "ũy":["uỹ"], "ụy":["uỵ"],
            "ùa":["uà"], "úa":["uá"], "ủa":["uả"], "ũa":["uã"], "ụa":["uạ"],
            "xảy":["xẩy"], "bảy":["bẩy"], "gãy":["gẫy"],
            "không":["k", "hông", "ko", "khong"]}

        kp = KeywordProcessor(case_sensitive=False)
        kp.add_keywords_from_dict(rules)

        # Create preprocessing functions
        self.autocorrect          = kp.replace_keywords
        self.normalize_pricetag   = partial(pricetag.sub, 'giá_tiền')
        self.normalize_hashtag    = partial(hashtag.sub, HASHTAG)
        self.remove_specialchar   = partial(specialchar.sub, '')

    def transform(self, X):
        X = super().transform(X)

        return X.apply(self.autocorrect) \
                .apply(self.normalize_pricetag) \
                .apply(self.normalize_hashtag) \
                .apply(self.remove_specialchar)

pipeline_fp = Path('./model/pipe.joblib')

full_pipeline = joblib.load(pipeline_fp)

def classify_sentence(sentence):
    return full_pipeline.predict([sentence])[0].astype(np.uint)

def multioutput_to_multilabel(y):
    if isinstance(y, pd.DataFrame):
        y = y.values
    nrow = y.shape[0]
    ncol = y.shape[1]
    multilabel = np.zeros((nrow, 3 * ncol), dtype=bool)
    for i in range(nrow):
        for j in range(ncol):
            if y[i, j] != 0:
                pos = j * 3 + (y[i, j] - 1)
                multilabel[i, pos] = True
    return multilabel

def custom_f1_score(y_true, y_pred, average='micro', **kwargs):
    y_true = multioutput_to_multilabel(y_true)
    y_pred = multioutput_to_multilabel(y_pred)
    return sklearn.metrics.f1_score(y_true, y_pred, average=average, **kwargs)

aspects = ["FOOD#PRICES",
           "FOOD#QUALITY",
           "FOOD#STYLE&OPTIONS",
           "DRINKS#PRICES",
           "DRINKS#QUALITY",
           "DRINKS#STYLE&OPTIONS",
           "RESTAURANT#PRICES",
           "RESTAURANT#GENERAL",
           "RESTAURANT#MISCELLANEOUS",
           "SERVICE#GENERAL",
           "AMBIENCE#GENERAL",
           "LOCATION#GENERAL"]

all_attrs = [['PRICES', 'QUALITY', 'STYLE&OPTIONS'],
             ['PRICES', 'QUALITY', 'STYLE&OPTIONS'],
             ['PRICES', 'GENERAL', 'MISCELLANEOUS'],
             ['SERVICE', 'AMBIENCE', 'LOCATION']]

sentiments = ['dne', 'negative', 'neutral', 'positive']

entities = ['FOOD', 'DRINKS', 'RESTAURANT', 'OTHERS']

@st.cache
def combine_entity_attr(entities, all_attrs):
    return [f'{entity}#{attr}' for attrs, entity in zip(
        all_attrs, entities) for attr in attrs]

all_keys = combine_entity_attr(entities, all_attrs)

def display_result(result):
    for key, stmi in zip(all_keys, result):
        if not stmi:
            continue
        st.markdown(f'- **{key}**: {sentiments[stmi]}')

def label_encoder(label):
    y = [np.nan] * len(aspects)
    ap_stm = re.findall('{(.+?), ([a-z]+)}', label)

    for aspect, sentiment in ap_stm:
        idx = aspects.index(aspect)
        y[idx] = sentiment

    return y

def txt2df(uploaded_file):
    stringio = StringIO(uploaded_file.getvalue().decode('utf-8'))
    data = stringio.read().split('\n')

    df = pd.DataFrame()
    df['review'] = [review for review in data[1::4]]
    df[all_keys] = [label_encoder(label) for label in data[2::4]]

    return df

def label_decoder(encoded_label):
    label = []
    for ap_idx, sentiment in enumerate(encoded_label):
        if isinstance(sentiment, str):
            aspect = aspects[ap_idx]
            label.append(f'{{{aspect}, {sentiment}}}')
    return ', '.join(label)

@st.cache
def df2txt(df):
    X = df.review.values
    y = df.drop('review', axis=1).values
    rows = []
    for test_id, (review, label) in enumerate(zip(X, y), 1):
        label = label_decoder(label)
        rows.extend((f'#{test_id}', review, label, ''))
    text = '\n'.join(rows[:-1])
    return text

st.title('ABSA for Restaurant Review')

modes = ["Classify sentence",
         "Annotate data",
         "Compare results"]

st.sidebar.selectbox("Select a mode", modes, key='mode')
st.header(st.session_state.mode)

if st.session_state.mode == modes[0]:
    text = st.text_area(
        "Enter your review here and press Control+Enter to classify")
    if text:
        with st.spinner():
            result = classify_sentence(text)
            display_result(result)

elif st.session_state.mode == modes[1]:

    if 'prev_fileid' not in st.session_state:
        st.session_state.prev_fileid = None

    uploaded_file = st.sidebar.file_uploader('Upload texts', type='txt')

    def refresh():
        state = st.session_state
        doc_id = state.doc_id
        if state.df is None:
            return
        row = state.df.loc[doc_id]
        state.update({k: row[k] if isinstance(row[k], str)
                      else 'dne' for k in all_keys})

    if uploaded_file and st.session_state.prev_fileid != uploaded_file.id:
        df = txt2df(uploaded_file)

        st.session_state.df = df
        st.session_state.doc_id = 0
        st.session_state.ndocs = len(df)
        st.session_state.prev_fileid = uploaded_file.id
        refresh()
    elif uploaded_file is None:
        st.session_state.df = None
        st.session_state.doc_id = None

    # --------------------------------------------------

    if st.session_state.df is not None:
        txt = df2txt(st.session_state.df)

        st.sidebar.download_button(
            label='Export ⬇',
            data=txt,
            file_name='annotated_dataset.txt',
            mime='plain')

    # --------------------------------------------------

    # def on_submit():
    #     state = st.session_state
    #     doc_id = state.doc_id
    #     df = st.session_state.df
    #     df.iloc[doc_id, 1:] = [state[k] if state[k]
    #                            != 'dne' else np.nan for k in all_keys]

    def on_change_sentiment(key):
        state = st.session_state
        doc_id = state.doc_id
        df = state.df
        col_index = all_keys.index(key) + 1
        df.iloc[doc_id, col_index] = state[key] if state[key] != 'dne' else np.nan

    def on_next_prev(go_next):
        state = st.session_state
        doc_id = state.doc_id
        if go_next:
            state.doc_id = min(state.ndocs - 1, doc_id + 1)
        else:
            state.doc_id = max(0, doc_id - 1)
        refresh()

    def annotate_all():
        df = st.session_state.df
        y = full_pipeline.predict(df.review).astype(np.object_)
        y[y == 1] = 'negative'
        y[y == 2] = 'neutral'
        y[y == 3] = 'positive'
        y[y == 0] = np.nan
        df.iloc[:, 1:] = y
        refresh()

    def auto():
        df = st.session_state.df
        doc_id = st.session_state.doc_id
        text = df.loc[doc_id, 'review']
        result = classify_sentence(text)
        df.iloc[doc_id, 1:] = [sentiments[result[i]]
                               if result[i] else np.nan for i in range(len(result))]
        refresh()

    # --------------------------------------------------

    if st.session_state.df is not None:
        state = st.session_state
        doc_id = state.doc_id
        df = state.df
        doc_ids = range(state.ndocs)
        st.selectbox('Choose document', doc_ids,
                     key='doc_id', on_change=refresh)
        current_doc = df.loc[doc_id]
        st.write(current_doc[0])
        st.write(f'**{label_decoder(current_doc[1:])}**')

    if st.session_state.df is not None:
        cols = st.columns(4)
        cols[0].button('⏮ Prev', on_click=on_next_prev, args=(False, ))
        cols[1].button('Next ⏭', on_click=on_next_prev, args=(True, ))
        cols[2].button('Auto 🐢', on_click=auto)
        cols[3].button('Auto ⚡', on_click=annotate_all)

        containers = []

        for entity in entities:
            container = st.container()
            container.subheader(entity)
            containers.append(container)

        rows = [container.columns(len(attrs))
                for container, attrs in zip(containers, all_attrs)]

        for row, entity, attrs in zip(rows, entities, all_attrs):
            for column, attr in zip(row, attrs):
                key = f'{entity}#{attr}'
                column.radio(attr, sentiments, key=key,
                             on_change=on_change_sentiment, args=(key,))

        st.dataframe(st.session_state.df)
    else:
        st.write('Upload text file to begin.')
else:
    file1 = st.sidebar.file_uploader("Upload annotator 1's data", type='txt')
    df1 = None
    if file1 is not None:
        df1 = txt2df(file1)

    file2 = st.sidebar.file_uploader("Upload annotator 2's data", type='txt')
    df2 = None
    if file2 is not None:
        df2 = txt2df(file2)

    file3 = st.sidebar.file_uploader("Upload goal data", type='txt')
    df3 = None
    if file3 is not None:
        df3 = txt2df(file3)

    if df1 is not None and df2 is not None and df3 is not None:
        if (len(df1) != len(df2) != len(df3)
                or df1.review[0] != df2.review[0]
                or df1.review[0] != df3.review[0]):

            st.error('The annotated data are different')
        else:
            mapping = {np.nan: 0, 'negative': 1, 'neutral': 2, 'positive': 3}

            y1 = df1.iloc[:, 1:].replace(mapping).astype(np.uint8).values
            y2 = df2.iloc[:, 1:].replace(mapping).astype(np.uint8).values
            y3 = df3.iloc[:, 1:].replace(mapping).astype(np.uint8).values

            y1 = multioutput_to_multilabel(y1)
            y2 = multioutput_to_multilabel(y2)
            y3 = multioutput_to_multilabel(y3)

            interagree = custom_f1_score(y1, y2)
            benchmark1 = custom_f1_score(y3, y1)
            benchmark2 = custom_f1_score(y3, y2)

            st.write("**Inter-Annotator Agreement:**", interagree)
            st.write(f"**{file1.name} benchmark:**", benchmark1)
            st.write(f"**{file2.name} benchmark:**", benchmark2)

            st.subheader(file1.name)
            st.dataframe(df1)
            st.subheader(file2.name)
            st.dataframe(df2)
            st.subheader(file3.name)
            st.dataframe(df3)
    else:
        st.write("Upload 2 annotators' data and goal data to compare")


import os

from google_cloud_pipeline_components.aiplatform import (
    AutoMLTabularTrainingJobRunOp,
    EndpointCreateOp,
    ModelDeployOp,
    TabularDatasetCreateOp,
)
from kfp.v2 import dsl

PIPELINE_ROOT = os.getenv("PIPELINE_ROOT")
PROJECT = os.getenv("PROJECT")
DATASET_SOURCE = os.getenv("DATASET_SOURCE")
PIPELINE_NAME = os.getenv("PIPELINE_NAME", "covertype")
DISPLAY_NAME = os.getenv("MODEL_DISPLAY_NAME", PIPELINE_NAME)
TARGET_COLUMN = os.getenv("TARGET_COLUMN", "Cover_Type")
SERVING_MACHINE_TYPE = os.getenv("SERVING_MACHINE_TYPE", "n1-standard-16")

@dsl.pipeline(
    name=f"{PIPELINE_NAME}-vertex-automl-pipeline",
    description=f"AutoML Vertex Pipeline for {PIPELINE_NAME}",
    pipeline_root=PIPELINE_ROOT,
)
def create_pipeline():

    dataset_create_task = TabularDatasetCreateOp(
        display_name=DISPLAY_NAME,
        bq_source=DATASET_SOURCE,
        project=PROJECT,
    )

    automl_training_task = AutoMLTabularTrainingJobRunOp(
        project=PROJECT,
        display_name=DISPLAY_NAME,
        optimization_prediction_type="classification",
        dataset=dataset_create_task.outputs["dataset"],
        target_column=TARGET_COLUMN,
    )

    endpoint_create_task = EndpointCreateOp(
        project=PROJECT,
        display_name=DISPLAY_NAME,
    )

    model_deploy_task = ModelDeployOp(  # pylint: disable=unused-variable
        model=automl_training_task.outputs["model"],
        endpoint=endpoint_create_task.outputs["endpoint"],
        deployed_model_display_name=DISPLAY_NAME,
        dedicated_resources_machine_type=SERVING_MACHINE_TYPE,
        dedicated_resources_min_replica_count=1,
        dedicated_resources_max_replica_count=1,
    )


import angr
import nose

def test_file_unlink():
    # Initialize a blank state with an arbitrary errno location
    state = angr.SimState(arch="AMD64", mode="symbolic")
    state.libc.errno_location = 0xa0000000
    state.libc.errno = 0

    # Create a file 'test'
    fd = state.posix.open(b'test', 1)
    state.posix.close(fd)

    # Ensure 'test' was in fact created
    nose.tools.assert_in(b'/test', state.fs._files)

    # Store the filename in memory
    path_addr = 0xb0000000
    state.memory.store(path_addr, b'test\x00')

    # Unlink 'test': should return 0 and leave ERRNO unchanged
    unlink = angr.SIM_PROCEDURES['posix']['unlink']()
    state.scratch.sim_procedure = unlink
    rval = unlink.execute(state, arguments=[path_addr]).ret_expr
    nose.tools.assert_equal(rval, 0)
    nose.tools.assert_equal(state.solver.eval(state.libc.errno), 0)

    # Check that 'test' was in fact deleted
    nose.tools.assert_equal(state.fs._files, {})

    # Unlink again: should return -1 and set ERRNO to ENOENT
    unlink = angr.SIM_PROCEDURES['posix']['unlink']()
    state.scratch.sim_procedure = unlink
    rval = unlink.execute(state, arguments=[path_addr]).ret_expr
    nose.tools.assert_equal(rval, -1)
    nose.tools.assert_equal(state.solver.eval(state.libc.errno), state.posix.ENOENT)


import sys
from collections import deque
import unittest

class Solution:
    def minCost(self, knownWizards):
        # 10 wizards
        # find the min cost between 0 and 9
        minCost = [sys.maxsize for _ in range(10)]
        minCost[0] = 0

        queue = deque()
        queue.append(0)
        while queue:
            curr = queue.popleft()

            for knownW in knownWizards[curr]:
                cost = minCost[curr] + self.wizardCost(curr, knownW)
                if cost < minCost[knownW]:
                    minCost[knownW] = cost
                    queue.append(knownW)

        return minCost[9]

    def wizardCost(self, i, j):
        return (i - j) ** 2

class SolutionTest(unittest.TestCase):
    sol = Solution()

    def testMinCost(self):
        test = [[] for _ in range(10)]
        test[0] = [1, 4, 5]
        test[4] = [9]
        self.assertEqual(41, self.sol.minCost(test))

if __name__=='__main__':
    unittest.main()


## In this script color thresholding takes place, where we want to find how many BLUE balls exist in the picture!

import cv2

# Import and Preview Image with BLUE balls
image = cv2.imread('/Users/konstantinosgyftodimos/Documents/OpenCV_Projects/Images and Figures/threshold_red_ball.png')
print(image)
mean_value = cv2.mean(image)
print(mean_value)

lab_image = cv2.cvtColor(image,cv2.COLOR_BGR2Lab)
lab_value = cv2.mean(lab_image)
print(lab_value)

# Import and Preview Image with ALL balls
image = cv2.imread('/Users/konstantinosgyftodimos/Documents/OpenCV_Projects/Images and Figures/threshold_color_balls.jpg')
cv2.imshow('input image',image)
cv2.waitKey(-1)

# This time we are not removing the color, but we are adding blurr
blurred = cv2.GaussianBlur(image, (15,15), None, )
cv2.imshow('blurred version', blurred)
cv2.waitKey(-1)

# inRange creates a binary image in a range (similar to threshold but in a range)
# Remember we want the blue balls ;)

# blue ball should become white and everything else should become black
binary=cv2.inRange(blurred, (165, 0, 0), (255, 175, 155))
cv2.imshow('output',binary)
cv2.waitKey(-1)

contours, hierarchy = cv2.findContours(binary, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
print('There are:', len(contours), 'Blue Balls in the image')

# Draw Contours, Draw around the Footballs and number them!
cv2.drawContours(image, contours, -1, (0, 255, 0), 2)
cv2.imshow('Draw Contours', image)
cv2.waitKey(-1)


from cx_Freeze import setup, Executable

import sys
base = 'Win32GUI' if sys.platform=='win32' else None

import os
import os.path
PYTHON_INSTALL_DIR = os.path.dirname(os.path.dirname(os.__file__))
os.environ['TCL_LIBRARY'] = os.path.join(PYTHON_INSTALL_DIR, 'tcl', 'tcl8.6')
os.environ['TK_LIBRARY'] = os.path.join(PYTHON_INSTALL_DIR, 'tcl', 'tk8.6')

options = {
    'build_exe': {
        'include_files':[
            os.path.join(PYTHON_INSTALL_DIR, 'DLLs', 'tk86t.dll'),
            os.path.join(PYTHON_INSTALL_DIR, 'DLLs', 'tcl86t.dll'),
         ],
    },
}

executables = [
    Executable('tkedit.py', base=base)
]

setup(name='tkedit',
      version = '1.0',
      description = 'A tkinter text editor',
      options = options,
      executables = executables)


from django import forms
from django.utils.safestring import mark_safe

class SimpleMDE(forms.Textarea):
    class Media:
        js = (
            'https://cdn.jsdelivr.net/simplemde/latest/simplemde.min.js',
        )
        css = {
           'all': ('https://cdn.jsdelivr.net/simplemde/latest/simplemde.min.css',)
        }

    def __init__(self, attrs={'id':'textarea_id'}):
        self.attrs = attrs
        super(self.__class__, self).__init__(self.attrs)

    def render(self, name, value, attrs=None, renderer=None):
        rendered = super(self.__class__, self).render(name, value, attrs=None, renderer=None)
        return rendered + mark_safe("""<script>
            var simplemde = new SimpleMDE({{ element: document.getElementById('{0}'), 	renderingConfig: {{singleLineBreaks: false}} }});
            </script>""".format(self.attrs.get('id'))
            )


import os

import re

import requests
from bs4 import BeautifulSoup

import pandas as pd
import numpy as np

import pycountry

url_base = 'https://eoi-tax.com'

path_to_dir = os.path.dirname(os.path.abspath(__file__))
path_to_data = os.path.join(path_to_dir, 'data')

def get_one_jurisdiction_data(href):
    response = requests.get(url_base + href)

    soup = BeautifulSoup(response.content, 'lxml')

    tables = soup.find_all('table', class_='table table-hover table-condensed table-responsive')

    table = str(tables[0])
    table = table.replace('\n', '')

    new_list = []
    pattern = '	'

    for elt in re.split('', table):
        if re.match(pattern, elt):
            continue
        else:
            new_list.append(elt)

    data = {
        'partner_jurisdiction': [],
        'agreement_type': [],
        'agreement': [],
        'signed': [],
        'entered_into_force': [],
        'eoi': []
    }

    for elt in new_list[1:]:
        temp = elt.split('')

        jurisdiction = temp[0]
        jurisdiction = jurisdiction[jurisdiction.find('>') + 1:jurisdiction[1:].find('<') + 1]

        elt = temp[1:].copy()

        for i in range(0, int(len(elt) / 5)):
            agreement_type = elt[i * 5 + 0]
            agreement = elt[i * 5 + 1]
            signed = elt[i * 5 + 2]
            entered_into_force = elt[i * 5 + 3]
            eoi = elt[i * 5 + 4]

            agreement_type = agreement_type[agreement_type.find('>') + 1:agreement_type[1:].find('<') + 1]
            agreement = agreement[agreement.find('>') + 1:agreement[1:].find('<') + 1]
            signed = signed.replace('	', '')
            entered_into_force = entered_into_force.replace('	', '')
            eoi = eoi[:eoi.find('<')]

            data['partner_jurisdiction'].append(jurisdiction)
            data['agreement_type'].append(agreement_type)
            data['agreement'].append(agreement)
            data['signed'].append(signed)
            data['entered_into_force'].append(entered_into_force)
            data['eoi'].append(eoi)

    return pd.DataFrame(data)

def get_all_jurisdictions_data():
    output = []

    response = requests.get(url_base + '/jurisdictions/')
    soup = BeautifulSoup(response.content, 'lxml')

    for col in soup.find_all(class_='col-md-2'):
        for country in col.find_all('a'):
            country_name = country.text.replace('\n', '')
            country_iso2_code = re.findall('flag-icon-(..)', str(country))[0]

            data = get_one_jurisdiction_data(country.get('href'))

            data['parent_jurisdiction_name'] = country_name
            data['parent_jurisdiction_iso2_code'] = country_iso2_code

            output.append(data)

    df = pd.concat(output)

    df['partner_jurisdiction'] = df['partner_jurisdiction'].map(
        lambda country_name: 'British Virgin Islands' if country_name == 'Virgin Islands (British)' else country_name
    )
    df['partner_jurisdiction'] = df['partner_jurisdiction'].map(
        lambda country_name: 'Democratic Republic of the Congo' if country_name == 'Congo (the Democratic Republic of the)' else country_name
    )
    df['partner_jurisdiction'] = df['partner_jurisdiction'].map(
        lambda country_name: 'State of Palestine' if country_name == 'Palestine, State of' else country_name
    )

    path_to_geographies = os.path.join(path_to_data, 'geographies.csv')
    geographies = pd.read_csv(path_to_geographies)

    df = df.merge(
        geographies[['NAME', 'CODE']],
        how='left',
        left_on='partner_jurisdiction',
        right_on='NAME'
    )

    codes_to_impute = {
        'Bonaire, Sint Eustatius and Saba': 'BES',
        'Falkland Islands [Malvinas]': 'FLK',
        'Réunion': 'REU',
        'Holy See': 'VAT'
    }

    df['CODE'] = df.apply(
        lambda row: codes_to_impute.get(row['partner_jurisdiction'], row['CODE']),
        axis=1
    )

    df = df.drop(columns=['NAME'])
    df = df.rename(
        columns={
            'CODE': 'PARTNER_COUNTRY_CODE'
        }
    )

    df['parent_jurisdiction_iso3_code'] = df['parent_jurisdiction_iso2_code'].map(
        lambda code: pycountry.countries.get(alpha_2=code.upper()).alpha_3
    )

    df = df.rename(
        columns={
            'parent_jurisdiction_iso3_code': 'PARENT_COUNTRY_CODE'
        }
    )
    df = df.drop(columns=['parent_jurisdiction_name', 'partner_jurisdiction', 'parent_jurisdiction_iso2_code'])

    df['signed'] = pd.to_datetime(df['signed'])

    df['entered_into_force'] = df['entered_into_force'].map(lambda x: np.nan if x == 'No' else x)
    df['entered_into_force'] = pd.to_datetime(df['entered_into_force'])

    return df.copy()

if __name__ == '__main__':
    df = get_all_jurisdictions_data()

    df.to_csv(os.path.join(path_to_data, 'tax_treaties.csv'), index=False)


from core.module import Module
from core.prompt_listener import PromptListener
from message_client import MessageClient
import json

class GmcpModule(Module, PromptListener):

    client_data = {
        "Client": "TinyFugue",
        "Version": "6"
        }

    support_data = [
            "Core 1",
            "Char 1",
            "Room 1",
            "Comm.Channel 1",
            "Char.Afflictions 1",
            "Char.Defences 1",
            "IRE.Composer 1",
            ]

    def __init__(self, *args):
        super(GmcpModule, self).__init__(*args)

        self.prompt_count = 0
        self.gmcp_count = 0
        self.message_client = MessageClient("localhost", 12000)

        self.mud.eval("/def -h'CONNECT aetolia' = /python_call main.cb register_gmcp")
        self.mud.eval("/def -waetolia -hGMCP = /python_call main.cb handle_gmcp \%*")
        self.state["callback_handler"].registerCallback(self.handle_gmcp)
        self.state["callback_handler"].registerCallback(self.register_gmcp)

        self.state["alias_builder"].build({
            "gmcp_dump": self.dump_gmcp
            })

    def parse_prompt(self, line):
        if self.gmcp_count:
            return
        self.prompt_count += 1
        if (self.prompt_count % 3) == 0:
            self.mud.warn("Received %d prompts and no GMCP" % self.prompt_count)
            self.register_gmcp()

    def register_gmcp(self):
        self.mud.info("Registering for GMCP")
        self.mud.gmcp("Core.Hello", self.client_data)
        self.mud.gmcp("Core.Supports.Set", self.support_data)

    def handle_gmcp(self, *payloads):
        payload = " ".join(payloads)
        self.gmcp_count += 1
        lines = payload.split(" ", 1)
        key = lines[0]
        package = lines[1]
        data = json.loads(package)
        self.state["callback_handler"].triggerGmcpCallback(key, data)
        if key == "Comm.Channel.Text":
            self.message_client.send(data["text"])
        else:
            self.state["gmcp"][key] = data

    def dump_gmcp(self):
        self.mud.info("Dumping GMCP to gmcp_dump.json")
        with open("gmcp_dump.json", "w") as f:
            f.write(json.dumps(self.state["gmcp"], indent=4, sort_keys=True, separators=(",",": ")))


from __future__ import absolute_import, unicode_literals
import objc
from . import fnd as Foundation
from PyObjCTools.TestSupport import *
import os

NSObject = objc.lookUpClass('NSObject')

def S(*args):
    return b''.join(args)

FUNCTIONS=[
    ( 'NSHomeDirectory', S(objc._C_ID)),
    ( 'NSIsFreedObject', S(objc._C_NSBOOL, objc._C_ID) ),
    ( 'NSCountFrames', S(objc._C_UINT) ),
    ( 'NSClassFromString', S(objc._C_CLASS, objc._C_ID) ),
]

class TestBundleFunctions (TestCase):
    def setUp(self):
        self.bundle = Foundation.NSBundle.bundleForClass_(Foundation.NSBundle)

    def testSimple(self):
        for bundle in (None, self.bundle):
            d = {}
            objc.loadBundleFunctions(self.bundle, d, FUNCTIONS)

            self.assertIn('NSIsFreedObject', d)
            self.assertIn('NSCountFrames', d)
            self.assertIn('NSHomeDirectory', d)

            # Don't use this API, it is unsupported and causes warnings.
            #fn = d[u'NSIsFreedObject']
            #obj = NSObject.alloc().init()
            #value = fn(obj)
            #self.assertTrue(not value)

            fn = d['NSHomeDirectory']
            value = fn()
            self.assertEqual(value, os.path.expanduser('~'))

            fn = d['NSClassFromString']
            value = fn('NSObject')
            self.assertIs(value, NSObject)

            # Need to look for a different example, NSCountFrames crashes
            # (that is the actual function, not the dynamic wrapper)
            #fn = d[u'NSCountFrames']
            #import Foundation
            #fn = Foundation.NSCountFrames
            #value = fn()
            #self.assertIsInstance(value, int)

if __name__ == "__main__":
    main()


from pyspark.sql import SparkSession
import pandas as pd

df = pd.read_csv("permit-inspections.csv", delimiter=";")
print(df.shape)
appName = "PySpark Partition Example"
master = "local[1]"

spark = SparkSession.builder \
    .appName(appName) \
    .master(master) \
    .getOrCreate()

print(spark.version)


from PIL import Image
import torch
from torch.utils.data import Dataset
import torchvision.transforms.functional as TF

class MyDataSet(Dataset):

    def __init__(self, images_path1: list, images_path2: list, images_target: list, transform=None):
        self.images_path1 = images_path1
        self.images_path2 = images_path2
        self.images_target = images_target
        self.transform = transform

    def __len__(self):
        return len(self.images_path1)

    def __getitem__(self, item):
        img1 = Image.open(self.images_path1[item])
        img2 = Image.open(self.images_path2[item])
        target = Image.open(self.images_target[item])

        #img1, img2, target = map(TF.to_tensor, (img1, img2, target))
        if self.transform is not None:
            img1 = self.transform(img1)
            img2 = self.transform(img2)
            target = self.transform(target)

        return img1, img2, target


import pandas as pd
import numpy as np
import math
import ipywidgets as widgets
from ipywidgets import VBox, Label, HBox
from IPython.display import display

from pandas_profiling import ProfileReport
from pandas.plotting import scatter_matrix

##Seaborn for fancy plots.
#%matplotlib inline
import matplotlib.pyplot as plt
import seaborn as sns
plt.rcParams["figure.figsize"] = (8,8)

class edaDF:

##########################################################################
### Once connected to the class edaDF, please start by .user_choice() ####
## And enter what type of Data Exploration you would like to perform  ####
##########################################################################

    # Calss Initial constructor to define:
    # self.data
    # self.target
    # self.cat to define Categorical columns
    #self.num to define Numerical columns
    def __init__(self, data, target):
        self.data=data
        self.target=target
        # Identify Categorical, Numerical, Boolean type Columns
        self.cat=self.data.select_dtypes(exclude=[np.number]) # Categorical Columns
        self.num=self.data.select_dtypes(include=[np.number]) # Numerical Columns
        self.bool=self.data.select_dtypes('bool') # Boolean Column, {True, False}
        print('SUCCESSFULLY CONNECTED TO THE EDAdf')
        print('Please start with .user_input()')
        #print('THIS IS A BASIC EXPLORATORY DATA ANALYSIS: ')

    # Method 1: Display .info() of data
    def info(self):
        return self.data.info()

    # Method 2: Describe the data
    def describe(self):
        return self.data.describe().T

    # Method 3: Describe the data
    def Null_Values(self):
        return self.data.isnull().sum()

    # Method 4: Display the Target varaiable
    def giveTarget(self):
        return self.target

    # Method 5: Diplay Countplots for Categorical Columns
    def countPlots(self):
        n = len((self.cat).columns)
        cols = 2
        figure, ax = plt.subplots(math.ceil(n/cols), cols, figsize=(20,20))
        r = 0
        c = 0
        for col in (self.cat).columns:
            # https://github.com/mwaskom/seaborn/issues/1582
            splot=sns.countplot(data=self.data, x=col, hue=self.target, ax=ax[r][c])
            for p in splot.patches:
                splot.annotate(format(p.get_height(), '.2f'), (p.get_x() + p.get_width() / 2., p.get_height()),
                        ha = 'center', va = 'center', xytext = (0, 2), textcoords = 'offset points')
            c += 1
            if c == cols:
                r += 1
                c = 0
        plt.tight_layout()
        return figure

    # Method 6: Diplay Histplot for the Numerical Columns
    def histPlots(self, kde=True):

        n = len((self.num).columns)
        cols = 2
        figure, ax = plt.subplots(math.ceil(n/cols), cols, figsize=(20,20))
        r = 0
        c = 0
        for col in (self.num):
            if (self.num[col]).nunique() == 2:
                print(f'Variable/Column, {col}, has only 2 UNIQUE values, therefore a  horizontal CountPlot is displayed.')
                print('********************************************************************')
                splot=sns.countplot(data=self.data, x=col, hue=self.target, ax=ax[r][c])
                for p in splot.patches:
                    splot.annotate(format(p.get_height(), '.2f'), (p.get_x() + p.get_width() / 2., p.get_height()),
                        ha = 'center', va = 'center', xytext = (0, 2), textcoords = 'offset points')
                c+=1
                if c== cols:
                    r+=1
                    c=0
            else:
            #print("r:",r,"c:",c)
                sns.histplot(data=self.data, x=col, hue=self.target, kde=True, ax=ax[r][c])
                c += 1
                if c == cols:
                    r += 1
                    c = 0
        plt.tight_layout()
        plt.show()
        return figure

    # Method 7: Diplay Boxplotx for the Numerical Columns to view outliers
    def Boxplot(self):
        for cols in self.num.columns:
            if self.num[cols].nunique()==2:
                print(f'Column/Variable, {cols} will not have a Boxplot as it only has 2 unique values')
                print()

        n = len(self.num.columns)
        cols = 2
        figure, ax = plt.subplots(math.ceil(n/cols), cols, figsize=(20,20))
        r = 0
        c = 0
        for col in self.num.columns:
            if self.num[col].nunique() != 2:
                sns.boxplot(data=self.data, x=col, ax=ax[r][c])
                #counter += 1
                c += 1
                if c == cols:
                    r += 1
                    c = 0
        plt.tight_layout()
        plt.show()
        return figure

    # Method 8: Scatter_Matrix for all Numerical Columns
    def Scatterplot(self):
        #scatter_matrix=sns.pairplot(self.num, corner=True)
        attributes=(self.num).columns
        scatter=scatter_matrix((self.data)[attributes], figsize=(20,20), hist_kwds=None)
        plt.tight_layout()
        plt.show()
        return scatter

    # Method 9: Correlation between all Numerical Columns
    def correlation(self):
        self.corr=(self.num).corr(method='spearman')
        return self.corr

    # Method 10: Find Top 4 Correlations to Target variable
    def TOP4Corr(self):
        corr_matrix=(self.num).corr(method='spearman')
        print(f'Top 3 Correlations to Target:{self.target}, are:')
        return display(corr_matrix[self.target].sort_values(ascending=False).head(4))

    # Method 11: Diplay basic_EDA of dataframe with widget tabs
    def basic_EDA(self):
        out1 = widgets.Output()
        out2 = widgets.Output()
        out3 = widgets.Output()
        out4 = widgets.Output()
        out5 = widgets.Output()
        out6 = widgets.Output()
        out7 = widgets.Output()
        out8 = widgets.Output()
        out9 = widgets.Output()

        tab = widgets.Tab(children = [out1, out2, out3, out4, out5, out6, out7, out8, out9])
        tab.set_title(0, 'Overall Info')
        tab.set_title(1, 'Columns Info')
        tab.set_title(2, 'Describe Data')
        tab.set_title(3, 'Missing_Values')
        tab.set_title(4, 'Categorical Plots')
        tab.set_title(5, 'Numerical Plots')
        tab.set_title(6, 'BoxPlot:Numerical')
        tab.set_title(7, 'ScatterPlot: Numerical')
        tab.set_title(8, 'Correlation: Numerical')
        display(tab)

        with out1:
            display(self.info())

        with out2:
            print('#############################################')
            print()
            print('Number of Categorical Columns:',len((self.cat).columns))
            print(*self.cat, sep=', ')
            print()
            print('#############################################')
            print()
            print('Number of Numerical Columns:',len((self.num).columns))
            print(*self.num, sep=', ')
            print()
            print('#############################################')
            print()
            print('Number of Variables/Columns with only 2 Unique Values:')
            for unik in self.data.columns:
                if self.data[unik].nunique() == 2:
                    display(unik),
            print('#############################################')
            print()
            print('Number of Boolean Columns:',len((self.bool).columns))
            print(*self.bool, sep=', ')
            print()
            print()
            print('#############################################')
            print('Target Variable is set to:',self.target)
            print('#############################################')

        with out3:
            display(self.describe())

        with out4:
            display(self.Null_Values())

        with out5:
            fig5 = self.countPlots()
            plt.show(fig5)

        with out6:
            fig6 = self.histPlots()
            plt.show(fig6)

        with out7:
            fig7 = self.Boxplot()
            plt.show(fig7)

        with out8:
            print('Realtionship between Numerical Columns')
            fig8 = self.Scatterplot()

        with out9:
            fig9 = self.correlation()
            fig10 = self.TOP4Corr()
            display(fig9)
            display(fig10)

    # Method 12: User Input on what type of Data Exploration they like to see
    def user_choice(self):
        user_choice = input("Do you want to use BASIC exploration or PANDAS profiling? (BASIC/PANDAS): ")
        # Check user choice between BASIC and PANDAS
        if user_choice == "BASIC":
            self.basic_EDA()
        elif user_choice == "PANDAS":
            user_input=input('Do you want this to show as HTML or within the IDE? (HTML/IDE): ')
            if user_input == 'HTML':
                self.profiling_html()
            elif user_input == 'IDE':
                self.profiling_IDE()
            else:
                print("Invalid choice. Please choose 'HTML' or 'IDE'.")
        else:
            print("Invalid choice. Please choose 'basic' or 'profiling'.")

    # Method 13: Using Pandas_Profiling to generate a report in html.
    def profiling_html(self):
        profile = ProfileReport(self.data)
        profile.to_file("PANDAS_PROFILE_DATA.html")
        print("Data profile has been saved as PANDAS_PROFILE_DATA.html")
        print('###########   InStructionS on Viewing HTML File ############')
        print("YOU CAN OPEN THE HTML FILE USING THE VSC Extention called Open in Browser")
        print("Once installed, right click on file and choose OPEN with HTML Viewer")

    # Method 14: Display Pandas Profile Report within IDE
    def profiling_IDE(self):
        profile_IDE = ProfileReport(self.data)
        display(profile_IDE)


from pathlib import Path
import numpy as np
import pandas as pd
import random
import time
import logging
import json
from dataclasses import dataclass, field
from typing import List

from wisardlib.encoders.thermometer import (
    ThermometerEncoder,
    DistributiveThermometerEncoder,
)
from wisardlib.rams.dict_ram import DictRAM
from wisardlib.rams.bloom_filter_ram import (
    CountingBloomFilterRAM,
    CountMinSketchRAM,
    CountingCuckooRAM,
    HeavyHittersRAM,
    StreamThresholdRAM,
)
from wisardlib.builder import build_symmetric_wisard
from wisardlib.utils import untie_by_first_class
from sklearn.metrics import accuracy_score, f1_score

encoders = {
    "thermometer": ThermometerEncoder,
    "distributive-thermometer": DistributiveThermometerEncoder,
}

rams = {
    "dict": DictRAM,
    "count-bloom": CountingBloomFilterRAM,
    "count-min-sketch": CountMinSketchRAM,
    "count-cuckoo": CountingCuckooRAM,
    "heavy-hitters": HeavyHittersRAM,
    "stream-threshold": StreamThresholdRAM,
}

datasets = {
    "iris": {"path": Path("experiments/data/iris/data.pkl")}, # OK
    "wine": {"path": Path("experiments/data/wine/data.pkl")},  # OK
    "breast_cancer": {"path": Path("experiments/data/breast_cancer/data.pkl")},  # OK
    "ecoli": {"path": Path("experiments/data/ecoli/data.pkl")},  #
    "letter": {"path": Path("experiments/data/letter/data.pkl")}, # OK
    "satimage": {"path": Path("experiments/data/satimage/data.pkl")}, # OK
    "segment": {"path": Path("experiments/data/segment/data.pkl")}, #~OK
    "glass": {"path": Path("experiments/data/glass/data.pkl")}, # OK
    "mnist": {"path": Path("experiments/data/mnist/data.pkl")}, # OK
    "vehicle": {"path": Path("experiments/data/vehicle/data.pkl")}, # OK
    "motion_sense": {"path": Path("experiments/data/motion_sense/data.pkl")}, # OK
    "sensorless_drive": {"path": Path("experiments/data/sensorless_drive/data.pkl")}, # OK
}

@dataclass
class RAMConfig:
    name: str
    RAM_cls_name: str
    RAM_cls_kwargs: dict = field(default_factory=dict)

def load_dataset(name: str, root_data_path: Path) -> tuple:
    data_path = datasets[name]["path"]
    data_path = root_data_path / data_path
    (x_train, y_train), (x_test, y_test) = np.load(data_path, allow_pickle=True)
    return (x_train, y_train), (x_test, y_test)

def discretize_labels(labels: np.ndarray):
    single_labels = list(set(labels))
    return np.array([single_labels.index(l) for l in labels])

def set_random_seeds(seed: int = None):
    if not seed:
        seed = int(time.time())
    random.seed(seed)
    np.random.seed(seed)
    return seed

def run_experiment(
    exp_name: str,
    output_path: Path,
    ram_configs=List[RAMConfig],
    dataset_name: str = None,
    encoder_name: str = "thermometer",
    encoder_kwargs: dict = None,
    tuple_size=16,
    number_runs=3,
    bleach: int | List[int] = 1,
    root_data_path: Path = ".",
):
    exp_id = str(time.time())
    exp_dir = Path(output_path) / exp_name

    if isinstance(bleach, int):
        bleach = [bleach]

    # Load dataset
    (x_train, y_train), (x_test, y_test) = load_dataset(
        dataset_name, Path(root_data_path)
    )

    y_train = discretize_labels(y_train)
    y_test = discretize_labels(y_test)

    # Encode dataset
    encoder_cls = encoders[encoder_name]
    encoder_kwargs = encoder_kwargs or dict()
    encoder = encoder_cls(**encoder_kwargs)
    logging.info("Encoding datasets...")
    encoder.fit(x_train)
    x_train = [x.ravel() for x in encoder.transform(x_train)]
    x_test = [x.ravel() for x in encoder.transform(x_test)]

    # Get params
    indices = len(x_train[0].ravel())
    n_classes = max(y_train) + 1

    results = []
    times = []

    for config_no, config in enumerate(ram_configs):
        for run_no in range(number_runs):
            seed = set_random_seeds()
            exp_start_time = time.time()
            logging.info(
                f"Running experiment (config={config_no+1}/{len(ram_configs)}) (run={run_no+1}/{number_runs}): {config}"
            )
            # logging.info("Creating model...")
            model = build_symmetric_wisard(
                RAM_cls=rams[config.RAM_cls_name],
                RAM_creation_kwargs=config.RAM_cls_kwargs,
                number_of_rams_per_discriminator=indices // tuple_size,
                number_of_discriminators=n_classes,
                indices=indices,
                tuple_size=tuple_size,
                shuffle_indices=True,
            )

            # logging.info("Training model...")
            train_start = time.time()
            model.fit(x_train, y_train)
            train_end = time.time()

            print(str(model[0]))

            for bleach_val in bleach:
                logging.info(f"Setting bleach to: {bleach_val}")
                model.bleach = bleach_val
                predict_start = time.time()
                y_pred = model.predict(x_test)
                predict_end = time.time()
                y_pred, ties = untie_by_first_class(y_pred)

                model_size = model.size()
                acc = accuracy_score(y_test, y_pred)
                f1_macro = f1_score(y_test, y_pred, average="macro")
                f1_micro = f1_score(y_test, y_pred, average="micro")
                f1_weighted = f1_score(y_test, y_pred, average="weighted")
                logging.info(
                    f"Accuracy: {acc:.3f}, f1-score (weighted): {f1_weighted:.3f}, ties: {ties}, size: {model_size}"
                )

                results.append(
                    {
                        "bleach": bleach_val,
                        "accuracy": acc,
                        "f1 weighted": f1_weighted,
                        "f1 macro": f1_macro,
                        "f1 micro": f1_micro,
                        "ties": ties,
                        "run": run_no + 1,
                        "train time": train_end - train_start,
                        "predict time": predict_end - predict_start,
                        "ram name": config.name,
                        "ram kwargs": json.dumps(config.RAM_cls_kwargs),
                        "tuple size": tuple_size,
                        "dataset name": dataset_name,
                        "encoder": encoder_name,
                        "encoder kwargs": json.dumps(encoder_kwargs),
                        "experiment name": exp_name,
                        "model size": model.size(),
                        "train samples": len(y_train),
                        "test samples": len(y_test),
                        "classes": n_classes,
                        "rams per discriminator": indices // tuple_size,
                        "discriminators": n_classes,
                        "seed": seed,
                        "indices": indices,
                    }
                )
            exp_end_time = time.time()
            times.append(exp_end_time - exp_start_time)
            logging.info(
                f"The run took {exp_end_time-exp_start_time:.3f} seconds (estimated remaining time: {len(ram_configs)*number_runs*np.mean(times)-config_no*number_runs*np.mean(times)-(run_no+1)*np.mean(times):.2f} seconds...)\n"
            )

    df = pd.DataFrame(results)
    exp_dir.mkdir(exist_ok=True, parents=True)
    exp_path = exp_dir / f"{exp_id}.csv"
    df.to_csv(exp_path, index=False)
    print(f"Results saved to: {exp_path}")

    return df


"""
sample code for assign4.py

load_sst can be used to read the files from sst, which can be downloaded from this link:

  https://nlp.stanford.edu/sentiment/trainDevTestTrees_PTB.zip

load_embeddings can be used to read files in the text format. Here's a link to

  word2vec - https://drive.google.com/file/d/0B7XkCwpI5KDYNlNUTTlSS21pQmM/edit?usp=sharing
  GloVe (300D 6B) - http://nlp.stanford.edu/data/glove.840B.300d.zip

The word2vec file is saved in a binary format and will need to be converted to text format. This can be done by installing gensim:

  pip install --upgrade gensim

Then running this snippet:

  from gensim.models.keyedvectors import KeyedVectors

  model = KeyedVectors.load_word2vec_format('path/to/GoogleNews-vectors-negative300.bin', binary=True)
  model.save_word2vec_format('path/to/GoogleNews-vectors-negative300.txt', binary=False)

To train:

  python assign4.py

To write test predictions:

  python assign4.py --eval_only_mode

"""

import argparse
import os
import sys
import json
import random

import numpy as np

import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from torch.autograd import Variable

import nltk
from nltk.corpus import stopwords
# from nltk.tokenize import word_tokenize

import pandas as pd

PAD_TOKEN = '_PAD_'
UNK_TOKEN = '_UNK_'

mydir = os.path.dirname(os.path.abspath(__file__))

# Methods for loading SST data

def sentiment2label(x):
  if x >= 0 and x <= 0.2:
    return 0
  elif x > 0.2 and x <= 0.4:
    return 1
  elif x > 0.4 and x <= 0.6:
    return 2
  elif x > 0.6 and x <= 0.8:
    return 3
  elif x > 0.8 and x <= 1:
    return 4
  else:
    raise ValueError('Improper sentiment value {}'.format(x))

# This function was placed here if we want to do anything to the tokens before sending them out.
def tokenizePhrase(phrase):

    phraseWords = phrase.split(' ')

    return phraseWords

def read_dictionary_txt_with_phrase_ids(dictionary_path, phrase_ids_path, labels_path=None):
  print('Reading data dictionary_path={} phrase_ids_path={} labels_path={}'.format(
    dictionary_path, phrase_ids_path, labels_path))

  with open(phrase_ids_path) as f:
    phrase_ids = set(line.strip() for line in f)

  with open(dictionary_path) as f:
    examples_dict = dict()
    for line in f:
      parts = line.strip().split('|')
      phrase = parts[0]
      phrase_id = parts[1]

      if phrase_id not in phrase_ids:
        continue

      example = dict()
      example['phrase'] = phrase.replace('(', '-LRB').replace(')', '-RRB-')
      example['tokens'] = tokenizePhrase(example['phrase'])
      example['example_id'] = phrase_id
      example['label'] = None
      examples_dict[example['example_id']] = example

  if labels_path is not None:
    with open(labels_path) as f:
      for i, line in enumerate(f):
        if i == 0:
          continue
        parts = line.strip().split('|')
        phrase_id = parts[0]
        sentiment = float(parts[1])
        label = sentiment2label(sentiment)

        if phrase_id in examples_dict:
          examples_dict[phrase_id]['label'] = label

  examples = [ex for _, ex in examples_dict.items()]

  print('Found {} examples.'.format(len(examples)))

  return examples

def build_vocab(datasets):
  vocab = dict()
  vocab[PAD_TOKEN] = len(vocab)
  vocab[UNK_TOKEN] = len(vocab)
  for data in datasets:
    for example in data:
      for word in example['tokens']:
        if word not in vocab:
          vocab[word] = len(vocab)

  print('Vocab size: {}'.format(len(vocab)))

  return vocab

class TokenConverter(object):
  def __init__(self, vocab):
    self.vocab = vocab
    self.unknown = 0

  def convert(self, token):
    if token in self.vocab:
      id = self.vocab.get(token)
    else:
      id = self.vocab.get(UNK_TOKEN)
      self.unknown += 1
    return id

def convert2ids(data, vocab):
  converter = TokenConverter(vocab)
  for example in data:
    example['tokens'] = list(map(converter.convert, example['tokens']))
  print('Found {} unknown tokens.'.format(converter.unknown))
  return data

def concatenateVocabsEmbeddings(v1, e1, v2, e2):

  #compare vocabs,
  commonVocab = set(v1.keys()) & set(v2.keys())
  newEmbedings = np.ndarray(shape=(len(commonVocab),len(e1[0]) + len(e2[0])))

  # for every common word, find its location in each of the vocab dictionaries.
  # then get that words embedding in each of the embeddings and concatenate them.
  i = 0
  for word in commonVocab:
    id1 = v1[word]
    id2 = v2[word]

    newEmbedings[i] = np.concatenate((e1[id1], e2[id2]))
    i = i + 1

  # now make the commonVocab back into a dictionary with keys = word and value = index (starting at 0)
  generatedVocab = dict((word,index) for word in commonVocab for index in range(len(commonVocab)))

  return generatedVocab, newEmbedings

def load_data_and_embeddings(data_path, phrase_ids_path, embeddings_path):
  labels_path = os.path.join(data_path, 'sentiment_labels.txt')
  dictionary_path = os.path.join(data_path, 'dictionary.txt')
  train_data = read_dictionary_txt_with_phrase_ids(dictionary_path, os.path.join(phrase_ids_path, 'phrase_ids.train.txt'), labels_path)
  validation_data = read_dictionary_txt_with_phrase_ids(dictionary_path, os.path.join(phrase_ids_path, 'phrase_ids.dev.txt'), labels_path)
  test_data = read_dictionary_txt_with_phrase_ids(dictionary_path, os.path.join(phrase_ids_path, 'phrase_ids.test.txt'))
  vocab = build_vocab([train_data, validation_data, test_data])

  if options.useEmbeddingNumber == 1:
    # use embedding for glove
    vocab, embeddings = load_embeddings(options.embeddings, vocab, cache=True)
  elif options.useEmbeddingNumber == 2:
    # use embedding for word2vec (google)
    vocab, embeddings = load_embeddings(options.embeddings2, vocab, cache=True)
  elif options.useEmbeddingNumber == 3:
    # if we are doing double embedding then run it on both and concatenate the results
    vocab1, embeddings1 = load_embeddings(options.embeddings, vocab, cache=True)
    vocab2, embeddings2 = load_embeddings(options.embeddings2, vocab, cache=True)

    #Merge the vocabs and embeddings
    vocab, embeddings = concatenateVocabsEmbeddings(vocab1, embeddings1, vocab2, embeddings2)

  else:
    print("Embedding mode not support, please specify one of the options (1,2,3), reverting to standard embeddings:")
    vocab, embeddings = load_embeddings(options.embeddings, vocab, cache=True)

  train_data = convert2ids(train_data, vocab)
  validation_data = convert2ids(validation_data, vocab)
  test_data = convert2ids(test_data, vocab)
  return train_data, validation_data, test_data, vocab, embeddings

def load_embeddings(path, vocab, cache=False, cache_path=None):
  rows = []
  new_vocab = [UNK_TOKEN]

  if cache_path is None:
    cache_path = path + '.cache'

  # Use cache file if it exists.
  if os.path.exists(cache_path):
    path = cache_path

  print("Reading embeddings from {}".format(path))

  # first pass over the embeddings to vocab and relevant rows
  with open(path) as f:
    for line in f:
      word, row = line.split(' ', 1)
      if word == UNK_TOKEN:
        raise ValueError('The unk token should not exist w.in embeddings.')
      if word in vocab:
        rows.append(line)
        new_vocab.append(word)

  # optionally save relevant rows to cache file.
  if cache and not os.path.exists(cache_path):
    with open(cache_path, 'w') as f:
      for line in rows:
        f.write(line)
      print("Cached embeddings to {}".format(cache_path))

  # turn vocab list into a dictionary
  new_vocab = {w: i for i, w in enumerate(new_vocab)}

  print('New vocab size: {}'.format(len(new_vocab)))

  assert len(rows) == len(new_vocab) - 1

  # create embeddings matrix
  embeddings = np.zeros((len(new_vocab), 300), dtype=np.float32)
  for i, line in enumerate(rows):
    embeddings[i+1] = list(map(float, line.strip().split(' ')[1:]))

  return new_vocab, embeddings

# Batch Iterator

def prepare_data(data,maxKernelSize):
  # pad data
  maxlen = max(map(len, data))
  if (maxlen < maxKernelSize):
    maxlen = maxKernelSize

  data = [ex + [0] * (maxlen-len(ex)) for ex in data]

  # wrap in tensor
  if torch.cuda.is_available():
    return torch.cuda.LongTensor(data)
  else:
    return torch.LongTensor(data)

def prepare_labels(labels):
  try:
    if torch.cuda.is_available():
      return torch.cuda.LongTensor(labels)
    else:
      return torch.LongTensor(labels)
  except:
    return labels

def batch_iterator(dataset, batch_size, forever=True, maxEpochs=200):
  dataset_size = len(dataset)
  order = None
  nbatches = dataset_size // batch_size

  def init_order():
    return random.sample(range(dataset_size), dataset_size)

  def get_batch(start, end):
    batch = [dataset[ii] for ii in order[start:end]]
    data = prepare_data([ex['tokens'] for ex in batch],7) # the number should be changed to the maximum kernel size +1
    labels = prepare_labels([ex['label'] for ex in batch])
    example_ids = [ex['example_id'] for ex in batch]
    return data, labels, example_ids

  order = init_order()

  while True:

    for i in range(nbatches):

      start = i*batch_size
      end = (i+1)*batch_size
      yield get_batch(start, end)

    if nbatches*batch_size < dataset_size:
      yield get_batch(nbatches*batch_size, dataset_size)

    if not forever:
      break

    order = init_order()

# Models

class BagOfWordsModel(nn.Module):
  def __init__(self, embeddings):
    super(BagOfWordsModel, self).__init__()
    self.embed = nn.Embedding(embeddings.shape[0], embeddings.shape[1], sparse=True)
    self.embed.weight.data.copy_(torch.from_numpy(embeddings))
    self.classify = nn.Linear(embeddings.shape[1], 5)

  def forward(self, x):
    return self.classify(self.embed(x).sum(1))

# Train a CNN Sentiment Classifier using pre-trained
# and frozen skip-gram word2vec as your word embedding
# on the Sentiment data from above
class CNNClassifier(nn.Module):

    def __init__(self, vocab, embeddings, output_size, kernel_dim=100, kernel_sizes=(3, 4, 5), dropout=0.5):
        super(CNNClassifier,self).__init__()

        # kernel_sizes should now be working
        # print("Embedding Size: " + str(embeddings.shape[0]))
        # print("Embedding Size: " + str(embeddings.shape[1]))

        embeddingsPosition = 1

        self.embedding = nn.Embedding(len(vocab), embeddings.shape[embeddingsPosition])
        self.convs = nn.ModuleList([nn.Conv2d(1, kernel_dim, (K, embeddings.shape[embeddingsPosition])) for K in kernel_sizes])

        # kernal_size = (K,D)
        self.dropout = nn.Dropout(dropout)
        self.fc = nn.Linear(len(kernel_sizes) * kernel_dim, output_size)

    def init_weights(self, pretrained_word_vectors, is_static=False): # finetuned with is_static to true
        self.embedding.weight = nn.Parameter(torch.from_numpy(pretrained_word_vectors).float())
        if is_static:
            self.embedding.weight.requires_grad = False

    def forward(self, inputs, is_training=False):
        inputs = self.embedding(inputs).unsqueeze(1) # (B,1,T,D)
        inputs = [F.relu(conv(inputs)).squeeze(3) for conv in self.convs] #[(N,Co,W), ...]*len(Ks)
        inputs = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in inputs] #[(N,Co), ...]*len(Ks)

        concated = torch.cat(inputs, 1)

        if is_training:
            concated = self.dropout(concated) # (N,len(Ks)*Co)
        out = self.fc(concated)
        return F.log_softmax(out,1)

# Utility Methods

def checkpoint_model(step, val_err, model, opt, save_path):
  save_dict = dict(
    step=step,
    val_err=val_err,
    model_state_dict=model.state_dict(),
    opt_state_dict=opt.state_dict())
  torch.save(save_dict, save_path)

def load_model(model, opt, load_path):
  load_dict = torch.load(load_path)
  step = load_dict['step']
  val_err = load_dict['val_err']
  model.load_state_dict(load_dict['model_state_dict'])
  opt.load_state_dict(load_dict['opt_state_dict'])
  return step, val_err

# Main

def run_validation(model, dataset, options):
  err = 0
  count = 0

  numPerLabels = []
  numCorrectPredLabels = []

  validationFile = open(options.validationFileName,'w')

  # making the run_validation print out the stats for each label
  for data, labels, _ in batch_iterator(dataset, options.batch_size, forever=False):
    outp = model(Variable(data))
    loss = nn.NLLLoss()(F.log_softmax(outp), Variable(labels))
    acc = (outp.data.max(1)[1] == labels).sum() / data.shape[0]
    err += (1-acc) * data.shape[0]
    count += data.shape[0]

    for i in range(len(outp.data.max(1)[1])):
      validationFile.write('{},{}\n'.format(labels[i],outp.data.max(1)[1][i]))

  validationFile.close()
  err = err / count
  print('Ev-Err={}'.format(err))

  return err

def run_test(model, dataset, options):
  print('Writing predictions to {}'.format(os.path.abspath(options.predictions)))

  preds_dict = dict()

  for data, _, example_ids in batch_iterator(dataset, options.batch_size, forever=False):
    outp = model(Variable(data))
    preds = outp.data.max(1)[1]

    for id, pred in zip(example_ids, preds):
      preds_dict[id] = pred

  with open(options.predictions, 'w') as f:
    for id, pred in preds_dict.items():
      f.write('{}|{}\n'.format(id, pred))

def run(options):

  train_data, validation_data, test_data, vocab, embeddings = \
    load_data_and_embeddings(options.data, options.ids, options.embeddings)

  model = CNNClassifier(vocab, embeddings, 5)

  # embeddings should be initialized differently for finetuning
  if (options.fineTuned):
    model.init_weights(embeddings,is_static=True)
  else:
    model.init_weights(embeddings,is_static=False)

  USE_CUDA = torch.cuda.is_available()
  if USE_CUDA:
    torch.cuda.set_device(0) # just use the first gpu available.

  FloatTensor = torch.cuda.FloatTensor if USE_CUDA else torch.FloatTensor
  LongTensor = torch.cuda.LongTensor if USE_CUDA else torch.LongTensor
  ByteTensor = torch.cuda.ByteTensor if USE_CUDA else torch.ByteTensor

  if USE_CUDA:
      model = model.cuda()

  if (options.fineTuned):
    opt = optim.SGD(filter(lambda x: x.requires_grad, model.parameters()), lr = 3e-4)
  else:
    opt = optim.SGD(model.parameters(), lr=3e-4)

  step = 0
  best_val_err = 1

  if options.eval_only_mode:
    step, best_val_err = load_model(model, opt, options.model)
    print('Model loaded from {}\nstep={} best_val_err={}'.format(options.model, step, best_val_err))
    run_test(model, test_data, options)
    # sys.exit()
    # maybe not exit, instead just return out of the run function,
    # this allows multiple tests in series without having to re-run the script.
    #  Especially useful for running long runs throughout the day when I was on campus.
    return

  # print("Entering batch iterator loop")
  for data, labels, _ in batch_iterator(train_data, options.batch_size, forever=True):
    outp = model(Variable(data))
    loss = nn.NLLLoss()(F.log_softmax(outp), Variable(labels))
    acc = (outp.data.max(1)[1] == labels).sum() / data.shape[0]

    opt.zero_grad()
    loss.backward()
    opt.step()

    logFile = open(options.stepLogFileName,'a')
    validationLog = open(options.validationLogFileName,'a')

    if step % options.log_every == 0:
      print('Step={} Tr-Loss={} Tr-Acc={}'.format(step, loss.data[0], acc))
      logFile.write('{},{},{}\n'.format(step, loss.data[0], acc))

    if step % options.eval_every == 0:
      val_err = run_validation(model, validation_data, options)
      validationLog.write('{},{}\n'.format(step, val_err))
      # early stopping
      if val_err < best_val_err:
        best_val_err = val_err
        print('Checkpointing model step={} best_val_err={}.'.format(step, best_val_err))
        checkpoint_model(step, val_err, model, opt, options.model)

      logFile.close()
      validationLog.close()

    if step == options.maxNumSteps:
      print('Completed Training with step={} best_val_err={}.'.format(step, best_val_err))
      break

    step += 1

if __name__ == '__main__':

  # Set a seed for numpy, pytorch
  np.random.seed(0)
  torch.manual_seed(0)

  if torch.cuda.is_available():
    print("Cuda is Available, will try and use it")
  else:
    print("Cuda not available, not able to use")

# Only glove
# create the CNN and train
  parser = argparse.ArgumentParser()
  parser.add_argument('--ids', default=mydir, type=str)
  parser.add_argument('--data', default=os.path.expanduser('data/stanfordSentimentTreebank'), type=str)
  parser.add_argument('--embeddings', default=os.path.expanduser('data/glove/glove.840B.300d.txt'), type=str) # this is the first embedding, please don't change this.
  parser.add_argument('--model', default=os.path.join(mydir, 'model-cnnTweaked-glove.ckpt'), type=str) # this should be the filename the model is saved to
  parser.add_argument('--predictions', default=os.path.join(mydir, 'predictions-cnnTweaked-glove.txt'), type=str) # this is the file name to which predictions will be saved.
  parser.add_argument('--log_every', default=500, type=int) # this will log the training accuracy every # steps
  parser.add_argument('--eval_every', default=4000, type=int) # this will evaluate and provide validation error every # steps
  parser.add_argument('--batch_size', default=64, type=int) # this is the size of a batch sent to the gpu for computation
  parser.add_argument('--eval_only_mode', action='store_true') # this should be set to store_true while training and store_false while testing

  parser.add_argument('--embeddings2', default=os.path.expanduser('data/GoogleNews-vectors-negative300.txt'), type = str) # storing the second embedding allows for combination to happen, please don't change this
  parser.add_argument('--useEmbeddingNumber', default=1, type = int) # this takes value 1, 2 or 3 (1 = glove, 2 = word2vec, 3 = both)

  parser.add_argument('--maxNumSteps', default = 800000, type = int) # this is the number of steps the model will be trained for.

  parser.add_argument('--fineTuned', default = True, type = bool) # make this true for finetuned, false otherwise

  parser.add_argument('--validationFileName', default=os.path.expanduser('model-cnnTweaked-glove-validationTruths.txt'), type = str) # this is the filename required for writing the validation predictions and truth values to a test file for comparison (and analysis)
  parser.add_argument('--stepLogFileName', default=os.path.expanduser('model-cnnTweaked-glove-log.txt'), type = str) # this is the filename where the step number, accuracy and errpr of training will be logged.
  parser.add_argument('--validationLogFileName', default=os.path.expanduser('model-cnnTweaked-glove-val-log.txt'), type = str) # this is the filename where the step number, accuracy and error of validation will be logged.

  options = parser.parse_args()

  print(json.dumps(options.__dict__, sort_keys=True, indent=4))

  run(options)


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import random

# In[2]:

'''
This function is used to
read the three files given
'''
def read_files():
    prices = pd.read_csv('prices.csv')
    audit = pd.read_csv('auditors.csv')
    stores = pd.read_json('stores.json')
    return prices,audit,stores

# In[62]:

def write_csv(df):
    df.to_csv('crsstab.csv', sep = '\t')

# In[3]:

'''
This function takes in as input the dataframe, oldname of the column
and new name of the column and rename the old column to new column
and returns the updated dataframe
'''
def rename_column(df,oldname,newname):
    df = df.rename(columns ={oldname: newname})
    return df

# In[4]:

'''
This function is used to join two tables.
It takes as input the two tables to be joined, method of join
and the column id on which join is to be performed.
Returns the merged table
'''

def join_table(table1, table2, method, column_id):
    table = table1.merge(table2, how = method, on = column_id)
    return table

# In[8]:

'''
This function forms the cross tabulation table.
It takes in as input the table, list of values as index, columns which would be
added and the values that the columns will hold.
Returns the cross tabulated table.
'''
def create_crss_tab(table,index_values,columns,values):
    crss_tab_df = table.pivot_table(index = index_values, columns = columns, values = values).reset_index()
    return crss_tab_df

# In[24]:

'''
This function randomly selects 'p' different products from the
fully merged table and store them in a list and returns the list.
'''
def select_random_products(p):
    product = []
    for x in range(p):
        y =random.randint(0,1000)
        product.append(full_mrg.loc[(full_mrg['index'] == y)].iloc[0][7])
    return product

# In[29]:

'''
This function takes as input the dataframe
calculate the interquatile range of the prices per product
and find the maximum and minimum price per product to identify the
outliers. Any outlier data lies outside the 3* Interquatile range.

Returns the dataframe with maximum and minimum prices within which
the prices should lie per product.
'''
def find_max_min_price(df):
    #finding the third quartile
    iqr = df.groupby('UPC').quantile(0.75).drop(columns = ['Store ID'], axis = 1).reset_index()
    iqr = iqr.rename(columns = {'Price':'Third_quartile'})

    #finding the first quartile
    iqr1 = df.groupby('UPC').quantile(0.25).drop(columns = ['Store ID'], axis = 1).reset_index()
    iqr1 = iqr1.rename(columns = {'Price':'First_quartile'})

    #merging the first quartile and the third quatile data frame
    iqr = iqr.merge(iqr1, on = 'UPC')

    # finding the interquartile range
    iqr['IQR'] = iqr['Third_quartile'] - iqr['First_quartile']
    #3 times the interquartile range
    iqr['IQR'] = 3*iqr['IQR']

    #finding the maximum and minimum prices per product
    iqr['Maximum_Price'] = iqr['Third_quartile'] + iqr['IQR']
    iqr['Minimum_Price'] = iqr['First_quartile'] - iqr['IQR']

    #dropping the columns not required
    iqr = iqr.drop(columns = ['Third_quartile','First_quartile','IQR'])
    return iqr

# In[46]:

'''
This function calculates in percent the occurence of
each of the minimum outlier values.
Returns the result as a list
'''

def calc_percent():
    percent_prices = []
    sum_total = minimum_price_df.Price.value_counts().sum()
    for values in minimum_price_df.Price.value_counts():
        percent_prices.append(values/sum_total)
    return percent_prices

# In[35]:

'''
This function returns a list of all the unique values
in a given column of the dataframe.

It takes as input the dataframe and the column whose unique
values are to be found.
'''

def get_labels(df,field):
    labels = []
    for x in df[field]:
        if str(x) not in labels:
            x = str(x)
            labels.append(x)
    return labels

# In[42]:

'''
This function plot the pie chart given the labels and values.
'''

def pie_chart(labels,values):
    colors = ['r', 'g']
    plt.pie(values, labels=labels, startangle=50, autopct='%1.1f%%')
    plt.show()

# In[53]:

'''
This function calculates the mean price per region of previously selected random
products and store it in a list.
Returns the list as a tuple.
'''
def mean_price_per_region():
    north_california = []
    kansas = []
    new_york = []
    texas = []
    for i in range (0,len(product)):
        nc_prc = full_mrg[(full_mrg['UPC'] == product[i]) & (full_mrg['Region'] == 'Northern California') & (full_mrg['Price'] != 1.99) ].agg('mean').Price
        kansas_prc = full_mrg[(full_mrg['UPC'] == product[i]) & (full_mrg['Region'] == 'Kansas') & (full_mrg['Price'] != 1.99) ].agg('mean').Price
        ny_prc = full_mrg[(full_mrg['UPC'] == product[i]) & (full_mrg['Region'] == 'New York') & (full_mrg['Price'] != 1.99)].agg('mean').Price
        texas_prc = full_mrg[(full_mrg['UPC'] == product[i]) & (full_mrg['Region'] == 'Texas') & (full_mrg['Price'] != 1.99)].agg('mean').Price
        nc_prc = nc_prc.item()
        kansas_prc = kansas_prc.item()
        ny_prc = ny_prc.item()
        texas_prc = texas_prc.item()
        north_california.append(nc_prc)
        kansas.append(kansas_prc)
        new_york.append(ny_prc)
        texas.append(texas_prc)
    nc = tuple(north_california)
    tx = tuple(texas)
    kn = tuple(kansas)
    ny = tuple(new_york)
    return nc,tx,kn,ny

# In[54]:

def bplt_price_per_region():
    n_groups = len(product)
    north_california = nc
    kansas = kn
    new_york = ny
    texas = tx

    # create plot
    fig, ax = plt.subplots()
    index = np.arange(n_groups)
    bar_width = 0.15
    opacity = 0.8

    rects1 = plt.bar(index, north_california, bar_width,
    alpha=opacity,
    color='b',
    label='north_california')

    rects2 = plt.bar(index + bar_width, kansas, bar_width,
    alpha=opacity,
    color='g',
    label='kansas')

    rects3 = plt.bar(index + 2*bar_width, new_york, bar_width,
    alpha=opacity,
    color='r',
    label='new_york')

    rects4 = plt.bar(index + 3*bar_width, texas, bar_width,
    alpha=opacity,
    color='y',
    label='texas')

    plt.xlabel('Products')
    plt.ylabel('Price')
    plt.title('Price by Region')
    plt.xticks(index + 3*bar_width, ('P1', 'P2', 'P3', 'P4','P5','P6','P7','P8','P9','P10','P11','P12','P13','P14','P15'))
    plt.legend()

    plt.tight_layout()
    plt.show()

# In[56]:

'''
This function calculates the mean price per store of previously selected random
products and store it in a list.
Returns the list as a tuple.
'''

def mean_price_per_store():
    Walmart = []
    Trader_Joes = []
    Safeway = []
    Whole_Foods = []
    Wegmans = []

    for i in range (0,len(product)):
        walmrt_prc = full_mrg[(full_mrg['UPC'] == product[i]) & (full_mrg['Banner'] == 'Walmart') & (full_mrg['Price'] != 1.99)].agg('mean').Price
        trader_joes_prc = full_mrg[(full_mrg['UPC'] == product[i]) & (full_mrg['Banner'] == 'Trader Joes') & (full_mrg['Price'] != 1.99)].agg('mean').Price
        safeway_prc = full_mrg[(full_mrg['UPC'] == product[i]) & (full_mrg['Banner'] == 'Safeway') & (full_mrg['Price'] != 1.99)].agg('mean').Price
        wholefood_prc = full_mrg[(full_mrg['UPC'] == product[i]) & (full_mrg['Banner'] == 'Whole Foods') & (full_mrg['Price'] != 1.99)].agg('mean').Price
        wegmans_prc = full_mrg[(full_mrg['UPC'] == product[i]) & (full_mrg['Banner'] == 'Wegmans') & (full_mrg['Price'] != 1.99)].agg('mean').Price

        walmart_prc = walmrt_prc.item()
        trader_joes_prc = trader_joes_prc.item()
        safeway_prc = safeway_prc.item()
        wholefood_prc = wholefood_prc.item()
        wegmans_prc = wegmans_prc.item()

        Walmart.append(walmart_prc)
        Trader_Joes.append(trader_joes_prc)
        Safeway.append(safeway_prc)
        Whole_Foods.append(wholefood_prc)
        Wegmans.append(wegmans_prc)

    wlmt = tuple(Walmart)
    tjoes = tuple(Trader_Joes)
    safway = tuple(Safeway)
    wholefood = tuple(Whole_Foods)
    wgmns = tuple(Wegmans)

    return wlmt,tjoes,safway,wholefood,wgmns

# In[58]:

def bplt_price_per_store():
    # data to plot
    n_groups = len(product)
    walmart  = wlmt
    trader_joes = tjoes
    safeway = safway
    whole_food = wholefood
    wegmans = wgmns

    # create plot
    fig, ax = plt.subplots()
    index = np.arange(n_groups)
    bar_width = 0.15
    opacity = 0.8

    rects1 = plt.bar(index, walmart, bar_width,
    alpha=opacity,
    color='b',
    label='walmart')

    rects2 = plt.bar(index + bar_width, trader_joes, bar_width,
    alpha=opacity,
    color='g',
    label='trader_joes')

    rects3 = plt.bar(index + 2*bar_width, safeway, bar_width,
    alpha=opacity,
    color='r',
    label='safeway')

    rects4 = plt.bar(index + 3*bar_width, whole_food, bar_width,
    alpha=opacity,
    color='y',
    label='whole_food')

    rects5 = plt.bar(index + 4*bar_width, wegmans, bar_width,
    alpha=opacity,
    color='black',
    label='wegmans')

    plt.xlabel('Stores')
    plt.ylabel('Price')
    plt.title('Price by Stores')
    plt.xticks(index + 3*bar_width, ('P1', 'P2', 'P3', 'P4','P5','P6','P7','P8','P9','P10','P11','P12','P13','P14','P15'))
    plt.legend()

    plt.tight_layout()
    plt.show()

# In[ ]:

# In[63]:

if __name__ == '__main__':
    #calling the function to read the files
    prices,audit,stores = read_files()

    #calling the function to rename the column 'Region' in the audit table to 'Auditor Region'
    audit = rename_column(audit,'Region','Auditor Region')

    # calling the function to merge the tables stores and price
    str_prc_mrg = join_table(stores,prices,'left','Store ID')

    #calling the function to form the pivot table
    crss_tab = create_crss_tab(str_prc_mrg,['Banner','UPC'],'Region','Price')

    #replacing nan values with hyphon as given in the example
    values = {'Kansas':'-','New York':'-','Northern California':'-','Texas':'-'}
    crss_tab = crss_tab.fillna(value = values)

    #writing the file
    write_csv(crss_tab)

    #joining all the three tables
    full_mrg = join_table(str_prc_mrg,audit,'inner','Auditor ID')

    #calling the function to get the maximum and minimum possible price of each product
    iqr_df = find_max_min_price(full_mrg)

    #calling the function to merge the table having min and max val to the fully merged table
    full_mrg = join_table(full_mrg, iqr_df,'inner','UPC')

    #separtating the outlier values in new df
    minimum_price_df = full_mrg[(full_mrg['Price'] < full_mrg['Minimum_Price']) ]
    #checking how many outlier data has been given by each auditor
    print(minimum_price_df.First.value_counts())

    #checking the different outlier values
    print(minimum_price_df.Price.value_counts())

    #checking how many times different auditors have given the price 1.99
    print(minimum_price_df[minimum_price_df['Price'] == 1.99].First.value_counts())

    #calling the function to calculate in percent each of the minimum outlier value
    percent_prices = calc_percent()

    #calling the function to get labels for the pie chart of minimum values
    labels = get_labels(minimum_price_df,'Price')

    #calling the function to plot the pie chart of the minimum values
    pie_chart(labels,percent_prices)

    #adding index to the fully merged table
    full_mrg = full_mrg.reset_index()

    #calling the function to select random products from the table
    product = select_random_products(15)

    #calling the function to get mean price per regions
    nc,tx,kn,ny = mean_price_per_region()

    #creating the bar plot of prices per region
    bplt_price_per_region()

    #calling the function to get mean price per stores
    wlmt,tjoes,safway,wholefood,wgmns = mean_price_per_store()

    bplt_price_per_store()

# In[ ]:


import discord
from discord import app_commands
from discord.ext import commands

class Test(commands.Cog):
    def __init__(self, bot : commands.Bot) -> None:
        self.bot = bot

    @app_commands.command(name="test")
    async def test(self, interaction : discord.Interaction):
        await interaction.response.send_message("Eagle has landed!", ephemeral=True)

    # @commands.command(name="test")
    # async def test(self,ctx,args):
    #     await ctx.reply(args)

async def setup(bot : commands.Bot):
    await bot.add_cog(Test(bot))


"""API-related classes for the ODIN server.

This module implements a number of classes used by the ODIN server for routing and handling
API calls.

Tim Nicholls, STFC Application Engineering Group
"""

import importlib
import logging
import json

import tornado.web

from odin.http.routes.route import Route
from odin.util import PY3
from odin.http.handlers.base import ApiError, API_VERSION
if PY3:
    from odin.http.handlers.async_api import AsyncApiHandler as ApiHandler
    from odin.async_util import run_async
else:
    from odin.http.handlers.api import ApiHandler

class ApiVersionHandler(tornado.web.RequestHandler):
    """API version handler to allow client to resolve supported version.

    This request handler implements the GET verb to allow a call to the appropriate URI to return
    the supported API version as JSON.
    """

    def get(self):
        """Handle API version GET requests."""
        accept_types = self.request.headers.get('Accept', 'application/json').split(',')
        if "*/*" not in accept_types and 'application/json' not in accept_types:
            self.set_status(406)
            self.write('Requested content types not supported')
            return

        self.write(json.dumps({'api': API_VERSION}))

class ApiAdapterListHandler(tornado.web.RequestHandler):
    """API adapter list handler to return a list of loaded adapters.

    This request hander implements the GET verb to allow a call to the appropriate URI to return
    a JSON-encoded list of adapters loaded by the server.
    """

    def initialize(self, route):
        """Initialize the API adapter list handler.

        :param route: ApiRoute object calling the handler (allows adapters to be resolved)
        """
        self.route = route

    def get(self, version):
        """Handle API adapter list GET requests.

        This handler returns a JSON-encoded list of adapters loaded into the server.

        :param version: API version
        """
        # Validate the API version explicity - can't use the validate_api_request decorator here
        if version != str(API_VERSION):
            self.set_status(400)
            self.write("API version {} is not supported".format(version))
            return

        # Validate the accept type requested is appropriate
        accept_types = self.request.headers.get('Accept', 'application/json').split(',')
        if '*/*' not in accept_types and 'application/json' not in accept_types:
            self.set_status(406)
            self.write('Request content types not supported')
            return

        self.write({'adapters': [adapter for adapter in self.route.adapters]})

class ApiRoute(Route):
    """ApiRoute - API route object used to map handlers onto adapter for API calls."""

    def __init__(self, enable_cors=False, cors_origin="*"):
        """Initialize the ApiRoute object.

        This constructor initialises the API route object, defining handlers for version, adapter
        list and adapter API calls.

        :param enable_cors: flag to enable CORS request support
        :param cors_origin: CORS allowed origins
        """
        super(ApiRoute, self).__init__()

        # Define a default handler which can return the supported API version
        self.add_handler((r"/api/?", ApiVersionHandler))

        # Define a handler which can return a list of loaded adapters
        self.add_handler((r'/api/(.*?)/adapters/?', ApiAdapterListHandler, dict(route=self)))

        # Build a dict of params to be passed to API handler initialisation calls
        handler_params = dict(route=self, enable_cors=enable_cors, cors_origin=cors_origin)

        # Define the handler for API calls. The expected URI syntax, which is
        # enforced by the validate_api_request decorator, is the following:
        #
        #    /api/<version>/<subsystem>/<action>....
        #
        # The second pattern allows an API adapter to be accessed with or without
        # a trailing slash for maximum compatibility
        self.add_handler((r"/api/(.*?)/(.*?)/(.*)", ApiHandler, handler_params))
        self.add_handler((r"/api/(.*?)/(.*?)/?", ApiHandler, handler_params))

        self.adapters = {}

    def register_adapter(self, adapter_config, fail_ok=True):
        """Register an API adapter with the APIRoute object.

        Based on the adapter_config object passed in as an argument, this method attempts to
        load the specified module and create an instance of the adapter class to be used
        to handle requests to the API route on the appropriate path.

        :param adapter_config: AdapterConfig object for the adapter
        :param fail_ok: Allow the adapter import and registration to fail without raising an error
        """
        # Resolve the adapter module and class name from the dotted module path in the config object
        (module_name, class_name) = adapter_config.module.rsplit('.', 1)

        # Try to import the module, resolve the class in the module and create an instance of it
        try:
            adapter_module = importlib.import_module(module_name)
            adapter_class = getattr(adapter_module, class_name)
            if PY3 and adapter_class.is_async:
                adapter = run_async(adapter_class, **adapter_config.options())
            else:
                adapter = adapter_class(**adapter_config.options())
            self.adapters[adapter_config.name] = adapter

        except (ImportError, AttributeError) as e:
            logging.error(
                "Failed to register API adapter %s for path %s with dispatcher: %s",
                adapter_config.module, adapter_config.name, e)
            if not fail_ok:
                raise ApiError(e)
        else:
            logging.debug(
                "Registered API adapter class %s from module %s for path %s",
                class_name, module_name, adapter_config.name
            )

    def has_adapter(self, subsystem):
        """Determine if ApiRoute object has adapter for subsystem.

        :param subsystem: subsystem to check for adapter
        :return: True of adapter present for subsystem
        """
        return subsystem in self.adapters

    def adapter(self, subsystem):
        """Return adapter for subsystem.

        :param subsystem: subsystem to return adapter for
        :return: adapter for subsystem
        """
        return self.adapters[subsystem]

    def cleanup_adapters(self):
        """Clean up state of registered adapters.

        This calls the cleanup method present in any registered adapters, allowing them to
        clean up their state (e.g. connected hardware) in a controlled fashion at shutdown.
        """
        for adapter_name, adapter in self.adapters.items():
            try:
                cleanup_method = getattr(adapter, 'cleanup')
                if PY3 and adapter.is_async:
                    run_async(cleanup_method)
                else:
                    cleanup_method()
            except AttributeError:
                logging.debug("Adapter %s has no cleanup method", adapter_name)

    def initialize_adapters(self):
        """Initialize all the adapters after they have been registered by the route.

        This calls the initialize method present in any registered adapters, passing
        the dictionary of listed adapters to each, for inter adapter communication.
        """
        for adapter_name, adapter in self.adapters.items():
            try:
                initialize_method = getattr(adapter, 'initialize')
                if PY3 and adapter.is_async:
                    run_async(initialize_method, self.adapters)
                else:
                    initialize_method(self.adapters)
            except AttributeError:
                logging.debug("Adapter %s has no initialize method", adapter_name)


#!/usr/bin/env python
import argparse
import sys
import math
import pickle
import numpy as np

from obspy.core import UTCDateTime, read, Stream
from obspy.signal import PPSD
from obspy.signal.invsim import paz_to_freq_resp
from obspy.signal.spectral_estimation import get_nlnm, get_nhnm
from matplotlib.mlab import csd
from math import pi
from matplotlib.pyplot import (figure,axes,plot,xlabel,ylabel,title,subplot,legend,savefig,show,xscale, xlim)

###############################################################################
#Code to do Sleeman self-noise
#By Adam Ringler
#This code takes data along with an instrument type and make a self-noise plot
#
#Here are the current methods
#getpaz()
###############################################################################

def getpaz(sensor):
#Function to get the poles and zeros
#Input the sensor type
#Output is a paz object
	debugpaz = False
	if debugpaz:
		print('Sensor type: ' + sensor)
	if sensor == 'T-120':
		paz = {'gain': 8.318710*10**17, 'zeros': [0 + 0j, 0 + 0j, -31.63 + 0j,
			-160.0 + 0j, -350.0 + 0j, -3177.0 + 0j], 'poles':[-0.036614 + 0.037059j,
			-0.036614 - 0.037059j, -32.55 + 0j, -142.0 + 0j, -364.0  + 404.0j,
			-364.0 - 404.0j, -1260.0 + 0j, -4900.0 + 5204.0j, -4900.0 - 5204.0j,
			-7100.0 + 1700.0j, -7100.0 - 1700.0j], 'sensitivity': 3.4153*10**18}
	elif sensor == 'T-120PH':
		paz = {'poles': [-0.03859+0.03649j, -0.03859-0.03649j, \
			-190, -158+193j, -158-193j, -639+1418j, -639-1418j], \
			'zeros': [0,0,-106,-158], \
			'sensitivity': 1201.0*1.695*(10**9)*(2**26)/40}
	elif sensor == 'TC-Reftek':
		paz = {'gain': 8.184*10**11, 'zeros': [0 + 0j, 0 + 0j, -434.1 + 0j],
			'poles':[-0.03691 + 0.03712j, -0.03691 - 0.03712j, -371.2 + 0j,
			-373.9  + 475.5j, -373.9 - 475.5j, -588.4 + 1508.0j, -588.4 - 1508.0j],
			'sensitivity': 4.7143*10**8}
	elif sensor == 'STS-2HG':
		paz = {'gain': 5.96806*10**7, 'zeros': [0, 0], 'poles': [-0.035647 - 0.036879j,
			-0.035647 + 0.036879j, -251.33, -131.04 - 467.29j, -131.04 + 467.29j],
			'sensitivity': 3.355500*10**10}
	elif sensor == 'STS-4B':
		paz = {'gain': 5.96806*10**7, 'zeros': [0, 0], 'poles': [-0.035647 - 0.036879j,
			-0.035647 + 0.036879j, -251.33, -131.04 - 467.29j, -131.04 + 467.29j],
			'sensitivity': 2.5166*10**9}
	elif sensor == 'CMG-3T':
		paz = {'gain': 5.71508*10**8, 'zeros': [0, 0], 'poles': [-0.037008 - 0.037008j,
			-0.037008 + 0.037008j, -502.65, -1005.0, -1131.0],
			'sensitivity': 3.3554*10**10}
	elif sensor == 'KS-54000':
		paz = {'gain': 86298.5, 'zeros': [0, 0], 'poles': [-59.4313,
			-22.7121 + 27.1065j, -22.7121 + 27.1065j, -0.0048004, -0.073199],
			'sensitivity': 3.3554*10**9}
	elif sensor == '151-120':
		paz = {'gain': 60828500, 'zeros': [0, 0, -18.18], 'poles': [-772.03,
			-20.9581, -180.3310 + 189.922j, -180.3310 - 189.922j,
			-0.0370184 + 0.0370296j, -0.0370184 - 0.0370296j],
			'sensitivity': 2.0*10**3}
	elif sensor == 'STS-1':
		paz = {'gain': 3948.58, 'zeros': [0, 0], 'poles': [-0.01234 - 0.01234j,
			-0.01234 + 0.01234j, -39.18 - 49.12j, -39.18 + 49.12j],
			'sensitivity': 3.3554432*10**9}
	elif sensor == 'M2166':
		paz = {'gain': 3948.58, 'zeros': [0, 0], 'poles': [-0.01234 - 0.01234j,
			-0.01234 + 0.01234j, -39.18 - 49.12j, -39.18 + 49.12j],
			'sensitivity': 1.00663296*10**9}
	else:
		paz={'gain': 1, 'zeros': [1 + 0j], 'poles': [1 + 0j], 'sensitivity': 1}

	if debugpaz:
		print(paz)
	return paz

def computeresp(resp,delta,lenfft):
	respval = paz_to_freq_resp(resp['poles'],resp['zeros'],resp['sensitivity'],t_samp = delta,
		nfft=lenfft,freq = False)
	respval = np.absolute(respval*np.conjugate(respval))
	respval = respval[1:]
	return respval

def cp(tr1,tr2,lenfft,lenol,delta):
	sr = 1/delta
	cpval,fre = csd(tr1.data,tr2.data,NFFT=lenfft,Fs=sr,noverlap=lenol,scale_by_freq=True)
	fre = fre[1:]
	cpval = cpval[1:]
	return cpval, fre

def choptocommon(stream):
#A function to chop the data to a common time window
	stimes = []
	etimes = []

	for trace in stream:
		stimes.append(trace.stats.starttime)
		etimes.append(trace.stats.endtime)
	newstime = stimes[0]
	newetime = etimes[0]

	for curstime in stimes:
		if debug:
			print(curstime)
		if curstime >= newstime:
			newstime = curstime

	for curetime in etimes:
		if debug:
			print(curetime)
		if curetime <= newetime:
			newetime = curetime

	if debug:
		print(newstime)
		print(newetime)
		print(stream)
	for trace in stream:
		trace.trim(starttime=newstime,endtime=newetime)
	if debug:
		print(stream)
	return stream

def getargs():
#Start up the parser
	parser = argparse.ArgumentParser(description='Program to make self-noise plots')

	parser.add_argument('-s1','--sensor1', type = str, action = "store", dest="sensorType1", \
		default = '', help="Type of Sensor  possible sensors include: " + "STS-1 " + "M2166 " + \
		"151-120 " + "KS-54000 " + "STS-4B " + "CMG-3T " + "STS-2HG " + "T-120 " + "TC-Reftek " \
		, required = False)

	parser.add_argument('-s2','--sensor2', type = str, action = "store", dest="sensorType2", \
		default = '', help="Type of Sensor  possible sensors include: " + "STS-1 " + "M2166 " + \
		"151-120 " + "KS-54000 " + "STS-4B " + "CMG-3T " + "STS-2HG " + "T-120 " + "TC-Reftek " \
		, required = False)

	parser.add_argument('-s3','--sensor3', type = str, action = "store", dest="sensorType3", \
		default = '', help="Type of Sensor  possible sensors include: " + "STS-1 " + "M2166 " + \
		"151-120 " + "KS-54000 " + "STS-4B " + "CMG-3T " + "STS-2HG " + "T-120 " + "TC-Reftek " \
		, required = False)

	parser.add_argument('-v','--verbose',action = "store_true", dest = "debug", \
		default = False, help="Run in verbose mode")

	parser.add_argument('-d1','--data1',type = str, nargs='+', action = "store", dest = "data1", \
		default = '', help="Data for PDF", required = True)

	parser.add_argument('-d2','--data2',type = str, nargs='+', action = "store", dest = "data2", \
		default = '', help="Data for PDF", required = True)

	parser.add_argument('-d3','--data3',type = str, nargs='+', action = "store", dest = "data3", \
		default = '', help="Data for PDF", required = True)

	parser.add_argument('-len', type = int, action = "store", default=20000, \
		help="Length of PSD window in seconds default=20000 s", \
		required=False, dest = "len")

	parser.add_argument('-overlap', type = float, action = "store", default=0.5, \
		help="Overlap of windows default=0.5", \
		required=False, dest = "overlap")

	parser.add_argument('-minper', type = float, action = "store", default=0.01, \
		help="Lower period limit default=0.01 s", \
		required=False, dest = "minper")

	parser.add_argument('-maxper', type = float, action = "store", default=1000.0, \
		help="Upper period limit default=1000.0 s", \
		required=False, dest = "maxper")

	parserval = parser.parse_args()
	return parserval

parserval = getargs()

if parserval.debug:
	debug = True
else:
	debug = False

#Get the response
if debug:
	print('Here is the sensor 1:' + parserval.sensorType1)
	print('Here is the sensor 2:' + parserval.sensorType2)
	print('Here is the sensor 3:' + parserval.sensorType3)

try:
	pazval1 = getpaz(parserval.sensorType1)
	pazval2 = getpaz(parserval.sensorType2)
	pazval3 = getpaz(parserval.sensorType3)
except:
	print('Can not get the responses for the instruments')
	sys.exit()

#Read in the data
try:
	st = Stream()
	for dataString in parserval.data1:
		if debug:
			print('Reading in the data trace 1:' + dataString)
		st += read(dataString)
	for dataString in parserval.data2:
		if debug:
			print('Reading in the data trace 2:' + dataString)
		st += read(dataString)
	for dataString in parserval.data3:
		if debug:
			print('Reading in the data trace 3:' + dataString)
		st += read(dataString)

	st.merge()

except:
	print('Unable to read the data')
	sys.exit()

if debug:
	for tr in st:
		print('Here is the data stream: ' + str(tr))
	print('Here is the window length of your PSDs: ' + str(parserval.len))
	print('Here is the overlap: ' + str(parserval.overlap))

st = choptocommon(st)
delta = st[0].stats.delta

try:
	inst1resp = computeresp(pazval1,delta,parserval.len)
	inst2resp = computeresp(pazval2,delta,parserval.len)
	inst3resp = computeresp(pazval3,delta,parserval.len)

	p11, _ = cp(st[0],st[0],parserval.len,parserval.overlap,delta)
	p22, _ = cp(st[1],st[1],parserval.len,parserval.overlap,delta)
	p33, _ = cp(st[2],st[2],parserval.len,parserval.overlap,delta)

	p21, _ = cp(st[1],st[0],parserval.len,parserval.overlap,delta)
	p13, _ = cp(st[0],st[2],parserval.len,parserval.overlap,delta)
	p23, fre1 = cp(st[1],st[2],parserval.len,parserval.overlap,delta)

	n11 = ((2*pi*fre1)**2)*(p11 - p21*p13/p23)/inst1resp
	n22 = ((2*pi*fre1)**2)*(p22 - np.conjugate(p23)*p21/np.conjugate(p13))/inst2resp
	n33 = ((2*pi*fre1)**2)*(p33 - p23*np.conjugate(p13)/p21)/inst3resp
except:
	print('Unable to compute the spectra')
	sys.exit()

NLNMper,NLNMpower = get_nlnm()
NHNMper,NHNMpower = get_nhnm()

titlelegend = 'Self-Noise: ' + str(st[0].stats.starttime.year) + ' ' + str(st[0].stats.starttime.julday) + ' ' + \
str(st[0].stats.starttime.hour) + ':' + str(st[0].stats.starttime.minute) + ':' + str(st[0].stats.starttime.second) + \
' ' + str(st[0].stats.npts*delta) + ' seconds'
noiseplot = figure(1)
subplot(1,1,1)
title(titlelegend,fontsize=12)
plot(1/fre1,10*np.log10(np.abs(((2*pi*fre1)**2)*p11/inst1resp)),'r',label='PSD ' + st[0].stats.station + ' ' + \
st[0].stats.location + ' ' + st[0].stats.channel + ' ' + parserval.sensorType1)
plot(1/fre1,10*np.log10(np.abs(((2*pi*fre1)**2)*p22/inst2resp)),'b',label='PSD ' + st[1].stats.station + ' ' + \
st[1].stats.location + ' ' + st[1].stats.channel + ' ' + parserval.sensorType2)
plot(1/fre1,10*np.log10(np.abs(((2*pi*fre1)**2)*p33/inst3resp)),'g',label='PSD ' + st[2].stats.station + ' ' + \
st[2].stats.location + ' ' + st[2].stats.channel + ' ' + parserval.sensorType3)
plot(1/fre1,10*np.log10(np.abs(n11)),'r:',label='Noise ' + st[0].stats.station + ' ' + st[0].stats.location + ' ' + \
st[0].stats.channel + ' ' + parserval.sensorType1)
plot(1/fre1,10*np.log10(np.abs(n22)),'b:',label='Noise ' + st[1].stats.station + ' ' + st[1].stats.location + ' ' + \
st[1].stats.channel + ' ' + parserval.sensorType2)
plot(1/fre1,10*np.log10(np.abs(n33)),'g:',label='Noise ' + st[2].stats.station + ' ' + st[2].stats.location + ' ' + \
st[2].stats.channel + ' ' + parserval.sensorType3)
plot(NLNMper,NLNMpower,'k')
plot(NHNMper,NHNMpower,'k')
legend(prop={'size':12})
xlabel('Period (s)')
ylabel('Power rel. 1 (m/s^2)^2/Hz')
xscale('log')
xlim((np.amin(1/fre1), np.amax(1/fre1) ))
savefig('NOISE' + str(st[0].stats.starttime.year) + str(st[0].stats.starttime.julday) + \
	str(st[0].stats.starttime.hour) + str(st[0].stats.starttime.minute) + \
st[0].stats.station + st[0].stats.location + st[0].stats.channel + \
st[1].stats.station + st[1].stats.location + st[1].stats.channel + \
st[2].stats.station + st[2].stats.location + st[2].stats.channel + \
'.jpg', format = 'jpeg', dpi=400)

titlelegend = 'Time Series: ' + str(st[0].stats.starttime.year) + ' ' + str(st[0].stats.starttime.julday) + ' ' + \
str(st[0].stats.starttime.hour) + ':' + str(st[0].stats.starttime.minute) + ':' + str(st[0].stats.starttime.second) + \
' ' + str(st[0].stats.npts*delta) + ' seconds'
tval=np.arange(0,st[0].stats.npts / st[0].stats.sampling_rate, st[0].stats.delta)
tseriesplot = figure(2)
title(titlelegend,fontsize=12)

subplot(311)
title(titlelegend,fontsize=12)
plot(tval,st[0].data,'r',label='TSeries ' + st[0].stats.station + ' ' + \
st[0].stats.location + ' ' + st[0].stats.channel + ' ' + parserval.sensorType1)
legend(prop={'size':12}, loc=2)
xlim((0, np.amax(tval) ))
subplot(312)
plot(tval,st[1].data,'b',label='TSeries ' + st[1].stats.station + ' ' + \
st[1].stats.location + ' ' + st[1].stats.channel + ' ' + parserval.sensorType2)
legend(prop={'size':12}, loc=2)
xlim((0, np.amax(tval) ))
subplot(313)
plot(tval,st[2].data,'g',label='TSeries ' + st[2].stats.station + ' ' + \
st[2].stats.location + ' ' + st[2].stats.channel  + ' ' + parserval.sensorType3)
xlabel('Time (s)')
ylabel('Counts')
legend(prop={'size':12}, loc=2)
xlim((0, np.amax(tval) ))
savefig('TSERIES' + str(st[0].stats.starttime.year) + str(st[0].stats.starttime.julday) + \
	str(st[0].stats.starttime.hour) + str(st[0].stats.starttime.minute) + \
st[0].stats.station + st[0].stats.location + st[0].stats.channel + \
st[1].stats.station + st[1].stats.location + st[1].stats.channel + \
st[2].stats.station + st[2].stats.location + st[2].stats.channel + \
'.jpg', format = 'jpeg', dpi=400)