import streamlit as st
import pandas as pd
import numpy as np
import re
import json
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
import math
import warnings
warnings.filterwarnings('ignore')
from PIL import Image

# Page setup
st.set_page_config(page_title = "Python Tweets Search Engine", page_icon = "🐍", layout = "wide")
st.title("Python Tweets Search Engine")
df = pd.read_csv('preprocessed_data.csv').fillna('')

inverted_index = json.load(open("inverted_index.json"))
# Define a function to tokenize and clean the text
def clean_text(text):
    text = re.sub(r"http\S+", "", text)  # Remove URLs
    text = re.sub(r'[^\w\s]', '', text)  # Remove punctuation
    text = text.lower()  # Convert text to lowercase
    return text.split()

# ------------------------------------------------------------------------------------------------------------

# Define the Boolean model function
def boolean_model(query):
    #corpus = pd.read_csv('preprocessed_data.csv')['content'].tolist()
    corpus_raw = pd.read_csv('raw_data.csv')
    # Pre-process the query
    query = clean_text(query)
    # Split query into terms
    if not query:
        return []
    terms = query

    # Find matching documents for each term
    results = []
    #univ_set = set([x for x in range(len(corpus_raw))])
    for i, term in enumerate(terms):
        if term in inverted_index:
            if terms[i-1] != 'not':
                results.append(inverted_index[term])
            else:
                #results.append(univ_set.difference(set(inverted_index[term])))
                pass
        else:
            results.append(set())
    #print(results)
    # Combine the sets using Boolean operators
    combined_results = set()
    for i, term_result in enumerate(results):
        term_result = set(term_result) # convert list to set
        if i == 0:
            combined_results = term_result
        else:
            if terms[i-1] == 'and':
                combined_results = combined_results.intersection(term_result)
            elif terms[i-1] == 'or':
                combined_results = combined_results.union(term_result)

    # Get the documents matching all terms
    # matching_docs = [corpus[i] for i in combined_results]

    df = corpus_raw
    return df[df.index.isin(combined_results)]

# ------------------------------------------------------------------------------------------------------------

# Define a function to handle wildcard queries
def handle_wildcard_query(query):
    pattern = query.replace('*', '.*')
    regex = re.compile(pattern)
    matching_terms = [term for term in inverted_index.keys() if regex.match(term)]
    doc_ids = set([doc_id for term in matching_terms for doc_id in inverted_index[term]])
    return doc_ids

# ------------------------------------------------------------------------------------------------------------

# Define a function to handle phrase queries
def handle_phrase_query(query):
    query = re.sub(r"http\S+", "", query)  # Remove URLs
    query = re.sub(r'[^\w\s]', '', query)  # Remove punctuation
    query_terms = query.lower().split()
    phrase_docs = []
    for i in range(len(df)):
        doc = df.iloc[i]
        doc_text = doc['content']
        for pos in range(len(doc_text.split())):
            if doc_text.split()[pos] == query_terms[0]:
                match = True
                for j in range(1, len(query_terms)):
                    if pos+j >= len(doc_text.split()):
                        match = False
                        break
                    next_term = doc_text.split()[pos+j]
                    if not next_term == query_terms[j]:
                        match = False
                        break
                if match:
                    phrase_docs.append(i)
                    break
    return phrase_docs

# ------------------------------------------------------------------------------------------------------------

# Define a function to calculate precision and recall
def calc_precision_recall(relevant_docs, retrieved_docs):
    tp = len(set(relevant_docs) & set(retrieved_docs))
    fp = len(retrieved_docs) - tp
    fn = len(relevant_docs) - tp
    precision = tp / (tp + fp) if tp + fp > 0 else 0
    recall = tp / (tp + fn) if tp + fn > 0 else 0
    return precision, recall

# ------------------------------------------------------------------------------------------------------------

# Example usage
def query_app(wq, pq):
    wildcard_query = wq
    phrase_query = pq
    wildcard_doc_ids = handle_wildcard_query(wildcard_query)
    phrase_doc_ids = handle_phrase_query(phrase_query)
    print(f'Wild card query: {wildcard_query}, matching doc ids: {wildcard_doc_ids}')
    print(f'Phrase query: {phrase_query}, matching doc ids: {phrase_doc_ids}')

# ------------------------------------------------------------------------------------------------------------

def query_pr_app(wq, pq, relevant_docs):
    wildcard_query = wq
    phrase_query = pq
    wildcard_doc_ids = handle_wildcard_query(wildcard_query)
    phrase_doc_ids = handle_phrase_query(phrase_query)
    print(f'Wild card query: {wildcard_query}, matching doc ids: {wildcard_doc_ids}')
    print(f'Phrase query: {phrase_query}, matching doc ids: {phrase_doc_ids}')
    print('---')
    print('Evaluation:')
    print(f'Number of relevant documents: {len(relevant_docs)}')
    wildcard_precision, wildcard_recall = calc_precision_recall(relevant_docs, wildcard_doc_ids)
    print(f'Wild card query precision: {wildcard_precision}, recall: {wildcard_recall}')
    phrase_precision, phrase_recall = calc_precision_recall(relevant_docs, phrase_doc_ids)
    print(f'Phrase query precision: {phrase_precision}, recall: {phrase_recall}')

# ------------------------------------------------------------------------------------------------------------

def retrieve_using_cosine_similarity(query, num_docs = 5):
    # Tokenize and clean the query
    query_tokens = clean_text(query)
    corpus = df['content'].tolist()
    corpus_raw = pd.read_csv('raw_data.csv')['content'].tolist()
    # Retrieve documents containing at least one query term
    candidate_doc_ids = set()
    for query_token in query_tokens:
        if query_token in inverted_index:
            candidate_doc_ids.update(inverted_index[query_token])

    # Calculate the cosine similarity between the query and candidate documents
    candidate_docs = [corpus[doc_id] for doc_id in candidate_doc_ids]
    vectorizer = TfidfVectorizer()
    tfidf_matrix = vectorizer.fit_transform(candidate_docs)
    query_vector = vectorizer.transform([query])
    cosine_similarities = cosine_similarity(query_vector, tfidf_matrix).flatten()

    # Sort the candidate documents by cosine similarity in descending order and get the top documents
    document_indices = cosine_similarities.argsort()[::-1][:num_docs]
    return [corpus.index(candidate_docs[index]) for index in document_indices]

# ------------------------------------------------------------------------------------------------------------

def log_likelihood(query, num_docs):
    corpus = df['content'].tolist()
    query = re.sub(r"http\S+", "", query)  # Remove URLs
    query = re.sub(r'[^\w\s]', '', query)  # Remove punctuation
    query_tokens = query.lower().split()
    query_likelihood = {}
    for token in query_tokens:
        if token in query_likelihood:
            query_likelihood[token] += 1
        else:
            query_likelihood[token] = 1
    query_length = sum(query_likelihood.values())
    for token in query_likelihood:
        query_likelihood[token] = query_likelihood[token] / query_length

    # Retrieve the documents that contain any of the query tokens
    retrieved_docs = set()
    for token in query_tokens:
        if token in inverted_index:
            retrieved_docs.update(inverted_index[token])

    # Compute the likelihood of each retrieved document
    doc_likelihoods = {}
    for doc_id in retrieved_docs:
        doc_tokens = corpus[doc_id].lower().split()
        doc_length = len(doc_tokens)
        likelihood = 0
        for token in query_likelihood:
            count = doc_tokens.count(token)
            token_likelihood = count / doc_length if count > 0 else 1 / (doc_length + 1)
            likelihood += math.log(token_likelihood) * query_likelihood[token]
        doc_likelihoods[doc_id] = likelihood

    # Rank the retrieved documents by their likelihood
    sorted_docs = sorted(doc_likelihoods.items(), key=lambda x: x[1], reverse=True)

    # Get the top N documents
    document_indices = [index for index, (doc_id, likelihood) in enumerate(sorted_docs[:num_docs]) if doc_id in retrieved_docs]

    # Return the indices of the top N documents
    return [corpus.index(sorted_docs[index][0]) for index in document_indices]

# ------------------------------------------------------------------------------------------------------------

# Define a function to retrieve documents using cosine similarity with relevance feedback
def retrieve_using_cosine_similarity_with_feedback(query, rel_list, num_docs = 5, alpha = 1, beta = 0.75, gamma = 0.15):
    # Transform the query using the vectorizer
    corpus = df['content'].tolist()
    corpus_raw = pd.read_csv('raw_data.csv')['content'].tolist()
    # Create a TF-IDF vectorizer and transform the corpus
    vectorizer = TfidfVectorizer()
    tfidf_matrix = vectorizer.fit_transform(corpus)
    query_vector = vectorizer.transform([query])

    # Calculate the cosine similarity between the query and all documents in the corpus
    cosine_similarities = cosine_similarity(query_vector, tfidf_matrix).flatten()

    # Sort the documents by cosine similarity in descending order and get the top documents
    document_indices = cosine_similarities.argsort()[::-1][:num_docs]
    top_documents = [(corpus_raw[index], cosine_similarities[index]) for index in document_indices]
    # Print the top documents
    print(document_indices)
    print(f"Showing top {num_docs} documents that are most similar to the query '{query}':\n")
    for i, (text, cosine_sim) in enumerate(top_documents):
        print(f"Rank {i+1} (Cosine Similarity: {cosine_sim:.4f}):")
        print(text)
        print("Reason: The document has a high cosine similarity score with the query.\n")

    # Get feedback from the user on the relevance of the search results
    relevant_doc_indices = []
    non_relevant_doc_indices = []
    print(rel_list, type(rel_list))
    for i in range(len(top_documents)):
        if(str(i) in rel_list):
            relevant_doc_indices.append(document_indices[i])
        else:
            non_relevant_doc_indices.append(document_indices[i])

    # Calculate the new query vector using the Rocchio algorithm
    relevant_doc_vectors = tfidf_matrix[relevant_doc_indices]
    non_relevant_doc_vectors = tfidf_matrix[non_relevant_doc_indices]
    new_query_vector = alpha * query_vector + beta * relevant_doc_vectors.mean(axis=0) - gamma * non_relevant_doc_vectors.mean(axis=0)

    # Calculate the cosine similarity between the new query vector and all documents in the corpus
    cosine_similarities = cosine_similarity(np.asarray(new_query_vector), tfidf_matrix).flatten()

    # Sort the documents by cosine similarity in descending order and get the top documents
    document_indices = cosine_similarities.argsort()[::-1][:num_docs]
    top_documents = [(corpus_raw[index], cosine_similarities[index]) for index in document_indices]
    print(document_indices, top_documents)
    print(type(document_indices), type(top_documents))
    # Print the reranked top documents
    print(f"\nShowing top {num_docs} reranked documents that are most similar to the query '{query}':\n")
    for i, (text, cosine_sim) in enumerate(top_documents):
        print(f"Rank {i+1} (Cosine Similarity: {cosine_sim:.4f}):")
        print(text)
        print("Reason: The document has a high cosine similarity score with the reranked query.\n")
    return list(document_indices)

# ------------------------------------------------------------------------------------------------------------

# Test the Boolean model
option = st.selectbox(
    'Type of query :',
    ('Boolean', 'Phrase', 'Wildcard', 'Cosine Similarity' , 'Relevance'))
N_cards_per_row = 3
max_results = 24

image = Image.open("icon.png")
resized_image = image.resize((300, 300))
st.sidebar.image(resized_image, width = 250)

for _ in range(5):
    st.sidebar.text("\n")
st.sidebar.text("This app is to serve as a front-end \nfor the tweets dataset search \nengine system implemented for\nAIRIW Assignment 1 in Python.")
df1 = pd.read_csv('raw_data.csv')
st.info("Search tweets by Boolean, Phrase, Wildcard, Cosine , Likelihood or Relevant")
text_search = st.text_input("Enter your query :")

if st.button('Go'):
    st.success("Searching... Your query is being processed !!!")
    if(option == 'Boolean'):
        df_search = boolean_model(text_search)
    elif(option == 'Phrase'):
        df_search = df1[df1.index.isin(handle_phrase_query(text_search))]
    elif(option == 'Wildcard'):
        df_search = df1[df1.index.isin(handle_wildcard_query(text_search))]
    elif(option == 'Cosine Similarity'):
        df_search = df1[df1.index.isin(retrieve_using_cosine_similarity(text_search, max_results))]
    # elif(option == 'Log Likelihood'):
    #     df_search = df1[df1.index.isin(log_likelihood(text_search, max_results))]
    elif(option == 'Relevance'):
        rel_lis = st.text_input("Enter relevant docs as a list")
        if rel_lis:
            st.write('Feedback submitted! New results are: ')
            df_search = df1[df1.index.isin(retrieve_using_cosine_similarity_with_feedback(text_search, rel_lis.split(','), max_results))]
        else:
            df_search = df1[df1.index.isin(retrieve_using_cosine_similarity(text_search, max_results))]
    df_search = df_search[:max_results]
    if text_search:
        with st.expander("Click to see dataframe view"):
            st.write(df_search)
    for n_row, row in df_search.reset_index().iterrows():
        i = n_row % N_cards_per_row
        if i == 0:
            st.write("---")
            cols = st.columns(N_cards_per_row, gap = "large")
        # draw the card
        with cols[n_row % N_cards_per_row]:
            st.caption(f"(Result No.: {n_row}) Tweet:")

            st.markdown(f"**{row['content'].strip()}**")
            st.markdown(f"*{row['publish_date'].strip()}*")


#threading better due to network I/O hinderances
#from threading import Thread

#multiprocessing used for cpu intensive processes (no networking hinderances)
from multiprocessing import Process, Queue
from time import time

def check_value_in_list(x, j, num_of_processes, queue):
	max_num_to_check = 10**8
	lower_bnd = int((j * max_num_to_check)/num_of_processes)
	upper_bnd = int(((j + 1) * max_num_to_check)/num_of_processes)

	num_of_hits = 0
	for i in range(lower_bnd, upper_bnd):
		if i in x:
			num_of_hits += 1

	queue.put((lower_bnd, upper_bnd, num_of_hits))

#num_threads = 4
def run():
	comparison_list = [1,2,3]
	num_processes = 4
	queue = Queue()

	processes = []
	for i in range(num_processes):
		t = Process(target=check_value_in_list, args=(comparison_list, i, num_processes, queue))
		processes.append(t)

	for t in processes:
		t.start()

	for t in processes:
		t.join()

	queue.put("DONE")

	while True:
		v = queue.get()
		if v == "DONE":
			break

		lower, upper, num_of_hits = v
		print("Between", lower, "and", upper, "we have", num_of_hits, "values in the list")

if __name__ == "__main__":
	start_time = time()
	run()
	print("Script time:", time() - start_time, " seconds")


from __future__ import absolute_import
#typing

import numpy
#overrides
import torch
from torch.nn.modules.linear import Linear
import torch.nn.functional as F

from allennlp.common.checks import check_dimensions_match
from allennlp.data import Vocabulary
from allennlp.modules import Seq2SeqEncoder, TimeDistributed, TextFieldEmbedder
from allennlp.models.model import Model
from allennlp.nn import InitializerApplicator, RegularizerApplicator
from allennlp.nn.util import get_text_field_mask, sequence_cross_entropy_with_logits
from allennlp.training.metrics import CategoricalAccuracy

class SimpleTagger(Model):
    u"""
    This ``SimpleTagger`` simply encodes a sequence of text with a stacked ``Seq2SeqEncoder``, then
    predicts a tag for each token in the sequence.

    Parameters
    ----------
    vocab : ``Vocabulary``, required
        A Vocabulary, required in order to compute sizes for input/output projections.
    text_field_embedder : ``TextFieldEmbedder``, required
        Used to embed the ``tokens`` ``TextField`` we get as input to the model.
    encoder : ``Seq2SeqEncoder``
        The encoder (with its own internal stacking) that we will use in between embedding tokens
        and predicting output tags.
    initializer : ``InitializerApplicator``, optional (default=``InitializerApplicator()``)
        Used to initialize the model parameters.
    regularizer : ``RegularizerApplicator``, optional (default=``None``)
        If provided, will be used to calculate the regularization penalty during training.
    """

    def __init__(self, vocab            ,
                 text_field_embedder                   ,
                 encoder                ,
                 initializer                        = InitializerApplicator(),
                 regularizer                                  = None)        :
        super(SimpleTagger, self).__init__(vocab, regularizer)

        self.text_field_embedder = text_field_embedder
        self.num_classes = self.vocab.get_vocab_size(u"labels")
        self.encoder = encoder
        self.tag_projection_layer = TimeDistributed(Linear(self.encoder.get_output_dim(),
                                                           self.num_classes))

        check_dimensions_match(text_field_embedder.get_output_dim(), encoder.get_input_dim(),
                               u"text field embedding dim", u"encoder input dim")
        self.metrics = {
                u"accuracy": CategoricalAccuracy(),
                u"accuracy3": CategoricalAccuracy(top_k=3)
        }

        initializer(self)

    #overrides
    def forward(self,  # type: ignore
                tokens                             ,
                tags                   = None,
                metadata                       = None)                           :
        # pylint: disable=arguments-differ
        u"""
        Parameters
        ----------
        tokens : Dict[str, torch.LongTensor], required
            The output of ``TextField.as_array()``, which should typically be passed directly to a
            ``TextFieldEmbedder``. This output is a dictionary mapping keys to ``TokenIndexer``
            tensors.  At its most basic, using a ``SingleIdTokenIndexer`` this is: ``{"tokens":
            Tensor(batch_size, num_tokens)}``. This dictionary will have the same keys as were used
            for the ``TokenIndexers`` when you created the ``TextField`` representing your
            sequence.  The dictionary is designed to be passed directly to a ``TextFieldEmbedder``,
            which knows how to combine different word representations into a single vector per
            token in your input.
        tags : torch.LongTensor, optional (default = None)
            A torch tensor representing the sequence of integer gold class labels of shape
            ``(batch_size, num_tokens)``.
        metadata : ``List[Dict[str, Any]]``, optional, (default = None)
            metadata containg the original words in the sentence to be tagged under a 'words' key.

        Returns
        -------
        An output dictionary consisting of:
        logits : torch.FloatTensor
            A tensor of shape ``(batch_size, num_tokens, tag_vocab_size)`` representing
            unnormalised log probabilities of the tag classes.
        class_probabilities : torch.FloatTensor
            A tensor of shape ``(batch_size, num_tokens, tag_vocab_size)`` representing
            a distribution of the tag classes per word.
        loss : torch.FloatTensor, optional
            A scalar loss to be optimised.

        """
        embedded_text_input = self.text_field_embedder(tokens)
        batch_size, sequence_length, _ = embedded_text_input.size()
        mask = get_text_field_mask(tokens)
        encoded_text = self.encoder(embedded_text_input, mask)

        logits = self.tag_projection_layer(encoded_text)
        reshaped_log_probs = logits.view(-1, self.num_classes)
        class_probabilities = F.softmax(reshaped_log_probs, dim=-1).view([batch_size,
                                                                          sequence_length,
                                                                          self.num_classes])

        output_dict = {u"logits": logits, u"class_probabilities": class_probabilities}

        if tags is not None:
            loss = sequence_cross_entropy_with_logits(logits, tags, mask)
            for metric in list(self.metrics.values()):
                metric(logits, tags, mask.float())
            output_dict[u"loss"] = loss

        if metadata is not None:
            output_dict[u"words"] = [x[u"words"] for x in metadata]
        return output_dict

    #overrides
    def decode(self, output_dict                         )                           :
        u"""
        Does a simple position-wise argmax over each token, converts indices to string labels, and
        adds a ``"tags"`` key to the dictionary with the result.
        """
        all_predictions = output_dict[u'class_probabilities']
        all_predictions = all_predictions.cpu().data.numpy()
        if all_predictions.ndim == 3:
            predictions_list = [all_predictions[i] for i in range(all_predictions.shape[0])]
        else:
            predictions_list = [all_predictions]
        all_tags = []
        for predictions in predictions_list:
            argmax_indices = numpy.argmax(predictions, axis=-1)
            tags = [self.vocab.get_token_from_index(x, namespace=u"labels")
                    for x in argmax_indices]
            all_tags.append(tags)
        output_dict[u'tags'] = all_tags
        return output_dict

    #overrides
    def get_metrics(self, reset       = False)                    :
        return dict((metric_name, metric.get_metric(reset)) for metric_name, metric in list(self.metrics.items()))

SimpleTagger = Model.register(u"simple_tagger")(SimpleTagger)


import cv2
import numpy as np
import pandas as pd
import json
from scipy.spatial.distance import cdist
import os

# Get fps of given video
def getFps(path):

    vidObj = cv2.VideoCapture(path)
    fps = vidObj.get(cv2.CAP_PROP_FPS)
    print("Frames per second using video.get(cv2.CAP_PROP_FPS) : {0}".format(fps))
    vidObj.release()
    return int(fps)

# Given the path to video, the fps and the second of needed frame, it returns the frame in jpg format ( needed for testing and trials )
def saveFrame(path, fps, frame_no):

    # Path to video file
    vidObj = cv2.VideoCapture(path)

    # Used as counter variable
    count = 0
    success = 1
    frame_count = 0

    while success:

        success, img = vidObj.read()

        if frame_count == frame_no:
            cv2.imwrite("frame"+str(frame_count)+".jpg",img)
            break

        # Catch the frames per second
        if count % fps == 0:
            frame_count = frame_count + 1

        count += 1

# Path: Path to the video to capture descriptors
# Fps: Fps of the video
# Interval: Array with two elements that indicate the start and end time of video to capture ([0,420] for first 7 min)
# No_of_descriptors: SIFT captures many descriptors most of which are unnecessary. This parameter determines the number of descriptors to capture with biggest blobs.
# Can be reduced to some extent with efficiency concerns.
# Folder_to_save: Descriptors are saved to a subfolder under ./descriptors. Name of the subfolder should be given.
# Function saves 3 files:
# * address.json: Mapping of descriptors to frames  ({"352":2} means descriptor in 352. row is the first descriptor of frame 2)
# * descriptors.npy: A 2d numpy array where each row is a descriptor (which is a 128 byte array). Each frame has no_of_descriptors rows in this array.
# * angles.npy: A 2d array that keeps principle angle of each keypoint in a frame in each row.
# (Each row has no_of_descriptors elements since there are no_of_descriptors keypoints for each frame. And there are as many rows as the number of frames captured.)
# Ex. interval = [20,40] and no_of_descriptors = 150
# Then the frames between 20. and 40. seconds of the given video are analyzed.
# descriptors.npy will have the shape (150*20, 128) since each row is a descriptor and total number of descriptors is 150*20
# angles.npy will have the shape (20,150) since each row is a frame and each descriptor is a column
def captureDescriptors(path, fps, interval, folder_to_save, no_of_descriptors=150):

    # Path to video file
    vidObj = cv2.VideoCapture(path)

    # Used as counter variable
    count = 0
    success = 1
    start = interval[0]
    end = interval[1]
    detect = cv2.xfeatures2d.SIFT_create(no_of_descriptors)

    all_desc = None
    all_angles =[]
    for i in range(start):
        all_angles.append([])
    first = True
    rowcount = 0
    frame_address = {}  # the mapping from row of decriptors to the frame number
    frame_count = start  # we catch the frame by second

    while success:

        if (count / fps) >= end:
            break

        success, img = vidObj.read()

        if (count / fps) < start:
            count += 1
            continue

        # Catch the frames per second
        if count % fps == 0:
            img = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
            keypoints, descriptors = detect.detectAndCompute(img,None)
            angles = [int(key.angle) for key in keypoints]
            all_angles.append(angles)
            if first:
                all_desc = descriptors
                first = False
            else:
                all_desc = np.concatenate((all_desc, descriptors))

            frame_address[rowcount] = frame_count
            rowcount = rowcount + len(descriptors)
            frame_count = frame_count + 1

        count += 1

    if not os.path.exists("./descriptors/"+folder_to_save):
        os.mkdir("./descriptors/"+folder_to_save)

    np.save("./descriptors/"+folder_to_save+"/angles", all_angles)
    np.save("./descriptors/"+folder_to_save+"/descriptors", all_desc)
    with open('./descriptors/'+folder_to_save+'/address.json', 'w') as fp:
        json.dump(frame_address, fp)
    print("Features saved")

# Path: Path to the video to analyze
# Fps: Fps of the video
# Interval: Array with two elements that indicate the start and end time of video to analyze ([420,840] between 7. and 14. mins)
# No_of_descriptors: SIFT captures many descriptors most of which are unnecessary. This parameter determines the number of descriptors to capture with biggest blobs
# Desc: descriptors.npy which is obtained by captureDescriptors()
# Sq: address.json which is obtained by captureDescriptors()
# Ang: angles.npy which is obtained by captureDescriptors()
# Ratio: When a descriptor is compared to a set of descriptors, we call the most similar pair a "match".
# To call it a "good match", we need that the distance of the match must me smaller than a ratio of the second best match.
# If ratio = 0.7, distances of first two matches are d1 and d2, the match with distance of d1 is a good match if d1 < 0.7*d2.
# We only count the good matches, thus ratio is an important parameter.
# Dumpfile: The file to write the matching results. (need to be a .csv)
# Function reads the given interval of the video, extracts the SIFT features of each frame, then compares the features with the ones in database.
# For our case, the database is given with desc, sq, ang. This can be changed. With the comparison, match results are written to a .csv file.
def analyzeFrames(path, interval, desc, sq, ang, no_of_descriptors, fps, dumpfile, ratio = 0.75):

    # Path to video file
    vidObj = cv2.VideoCapture(path)

    # Used as counter variable
    count = 0
    success = 1
    start = interval[0]
    end = interval[1]
    detect = cv2.xfeatures2d.SIFT_create(no_of_descriptors)
    first = True

    while success:

        if (count / fps) >= end:
            break

        success, img = vidObj.read()

        if (count / fps) < start:
            count += 1
            continue

        # Catch the frames per second
        if count % fps == 0:

            frame_no = int(count/fps)
            print(frame_no)
            img = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
            new_keypoints, new_descriptors = detect.detectAndCompute(img,None)
            angles = [int(key.angle) for key in new_keypoints]
            d = np.array(cdist(new_descriptors, desc))

            matches, matched, glob_match = getMatchFromDistance(sq, d, ratio)
            startidx = 0
            for key, value in sq.items():
                if value == matched:
                    startidx = int(key)
                    break
            matched_ang1 = []
            matched_ang2 = []
            for m in glob_match:
                new_idx = m[0]
                old_idx = m[1]
                if old_idx>=startidx and old_idx <startidx + no_of_descriptors:
                    idx = old_idx - startidx
                    angle1 = angles[new_idx]
                    angle2 = ang[matched][idx]
                    matched_ang1.append(angle1)
                    matched_ang2.append(angle2)
            angle, _ = detectAngle(matched_ang1, matched_ang2)
            writeMatches(frame_no, len(sq), matches, matched, angle, first, dumpfile)
            if first:
                first = False

        count += 1

# d: The distance matrix between descriptors of a frame and the set of descriptors in the database.
# Shape of d is (n,m) if current frame has n descriptors and there are m descriptors in database.
# d_ij = Distance between the ith descriptor of the frame and jth descriptor in the database.
# Function returns 3 things:
# * matches: An array that counts the number of matches between the current frame and each of the frames in database.
# * matched: argmax(matches) , the frame that is the best match of the current frame (test frame)
# * glob_match: An array of tuples where each element (i,j) is a pair of indices of matched descriptors.
# (i,j) means that ith descriptor of test frame is matched with jth descriptor in database. We get this to find relative angles.
def getMatchFromDistance(sq, d, ratio):
    rows, _ = d.shape
    matches = [0 for _ in range(len(sq))]
    indices = []
    glob_match = []
    for i in range(rows):
        row = d[i]
        min1, min2 = np.partition(row, 1)[0:2]
        if min1 < ratio*min2:
            # means this is a good match
            idx = np.where(row == min1)[0][0]
            indices.append(idx)
            glob_match.append((i,idx))
    for idx in indices:
        last = '0'
        for k in sq:
            if idx > int(k):
                last = k
                continue
            else:
                matched_square = sq[last]
                matches[matched_square] += 1
                break
    matched = np.argmax(matches)
    return matches, matched, glob_match

# http://amroamroamro.github.io/mexopencv/matlab/cv.SIFT.detectAndCompute.html
# Gets two arrays of angles to compare. Arrays have one to one correspondence. That is, ith elements of both arrays belong to matched keypoints.
# Difference between each corresponding pair of angles is calculated.
# The most common difference is inferred to be the relative angle between test frame and matched database frame.
def detectAngle(angles1, angles2):

    counter = np.array([0 for i in range(360) ])
    for i in range(len(angles1)):
        diff = angles1[i] - angles2[i]
        if diff < 0:
            diff += 360
        counter[diff] += 1
    return np.argmax(counter), np.max(counter)

# Matching results are written to a csv file.
def writeMatches(frame_no, no_of_frames, matches, matched, angle, first, dumpfile):

    if not os.path.exists("./matches"):
        os.mkdir("./matches")

    total_matches = sum(matches)
    max_match = matches[matched]

    if not first:
        df = pd.read_pickle(dumpfile)

    else:
        columns = ["Frame no","Matched Frame", "Angle" ,"Total Matches", "Max Match"]
        for i in range(no_of_frames):
            columns.append(i)
        df = pd.DataFrame(columns=columns)
    dic = {"Frame no": [frame_no], "Matched Frame": [matched], "Angle":[angle], "Total Matches":[total_matches], "Max Match":[max_match]}
    for i in range(no_of_frames):
        dic[i] = [matches[i]]
    df2 = pd.DataFrame(dic, index=[0])
    df = pd.concat([df, df2], sort=False)

    df.to_pickle(dumpfile)

# folder name of the run, will appear under matches directory
folder = "whitesquares"

# parameters of captureDescriptors()
train_video = "./videos/karolar_2.mov"
train_fps = 30
train_interval = [0,430]
train_descriptors = 150

# parameters of analyzeFrames()
query_video = "./videos/karolar_2.mov"
query_fps = 30
query_interval = [430,1320]
query_descriptors = 150
ratio = 0.75

# make it false if the descriptors in the database are being used
train = True

test = False

if train:
    captureDescriptors(path = train_video,fps = train_fps, interval = train_interval, folder_to_save = folder, no_of_descriptors = train_descriptors)

if test:
    with open('./descriptors/'+folder+'/address.json', 'r') as fp:
        sq = json.load(fp)
    with open('./descriptors/'+folder+'/descriptors.npy', 'rb') as f:
        desc = np.load(f)
    with open('./descriptors/'+folder+'/angles.npy', 'rb') as f:
        ang = np.load(f,allow_pickle=True)

    analyzeFrames(path = query_video, interval = query_interval, desc = desc, sq = sq, ang = ang, no_of_descriptors = query_descriptors,
                    fps = query_fps, folder = './matches/'+folder+'.csv', ratio = ratio)

    df = pd.read_pickle("./matches/"+folder+".csv")
    df.to_csv("./matches/"+folder+".csv")


import json

from django.views.generic import DetailView, ListView, View, CreateView
from django.core.exceptions import ImproperlyConfigured
from django.http import (
    HttpResponse,
    HttpResponseBadRequest,
    HttpResponseRedirect
)
from django.contrib.auth.decorators import login_required
from django.utils.decorators import method_decorator
from django.shortcuts import get_object_or_404
from django.core.urlresolvers import reverse
from django.contrib import messages
from django.utils.translation import ugettext_lazy as _

from .models import Suggestion, SuggestionCopy
from .forms import AddSuggestionForm

class IndexView(DetailView):
    """
    The landing page for the site
    """
    model = Suggestion
    template_name = 'suggestions/index.html'

    def get_context_data(self, **kw):
        ctx = super(IndexView, self).get_context_data(**kw)
        ctx['recent_suggestions'] = SuggestionCopy.objects.all()[:5]
        return ctx

    def get_object(self, queryset=None):
        if queryset is None:
            queryset = self.get_queryset()

        try:
            return queryset.order_by('?')[0]
        except IndexError:
            raise ImproperlyConfigured('No suggestions are installed')

    def render_to_response(self, ctx, **kw):
        if 'format' in self.request.GET:
            if self.request.GET['format'] == 'json':
                return HttpResponse(
                    json.dumps(
                        {
                            'id': ctx['object'].id,
                            'suggestion': str(ctx['object']),
                            'url': ctx['object'].get_absolute_url(),
                            'split_text': ctx['object'].split()
                        }
                    ),
                    content_type='application/json'
                )
            return HttpResponseBadRequest('Format not supported')
        return super(IndexView, self).render_to_response(ctx, **kw)

class SuggestionView(DetailView):
    """
    A view for a single Suggestion
    """
    model = Suggestion

class LoginRequiredMixin(object):
    """
    Mixin to ensure a user is logged in; basically applies the login_required
    decorator from auth module.
    """
    @method_decorator(login_required)
    def dispatch(self, *ar, **kw):
        return super(LoginRequiredMixin, self).dispatch(*ar, **kw)

class GetSuggestionCopyQSMixin(object):
    """
    We want to get the 10 latest suggestions that the user has had copied
    or if there are none we create a new one.
    """
    def get_queryset(self):
        queryset = self.request.user.suggestions.all()[:10]
        if not queryset.count():
            SuggestionCopy.objects.create_random_for_user(self.request.user)
            return self.request.user.suggestions.all()[:10]
        return queryset

class UserView(LoginRequiredMixin, GetSuggestionCopyQSMixin, ListView):
    """
    The logged in user's view
    """
    template_name = 'suggestions/user.html'

class JSONResponseMixin(object):
    def render_to_response(self, ctx, **kw):
        return HttpResponse(json.dumps(ctx), 'application/json')

class SkipSuggestionView(
    LoginRequiredMixin,
    GetSuggestionCopyQSMixin,
    JSONResponseMixin,
    View
):
    """
    Skip over the current suggestion for the user and return a new suggestion
    """
    def get(self, request, *ar, **kw):
        self.get_queryset()[0].delete()
        SuggestionCopy.objects.create_random_for_user(self.request.user)
        queryset = self.get_queryset()
        return self.render_to_response({'suggestion': queryset[0].data})

class GetSuggestionCopySingleMixin(object):
    def get_queryset(self):
        queryset = self.request.user.suggestions.all()
        if not queryset.count():
            SuggestionCopy.objects.create_random_for_user(self.request.user)
            return self.request.user.suggestions.all()
        return queryset

    def get_object(self, id):
        return get_object_or_404(self.get_queryset(), pk=id)

class ActionSuggestionView(
    LoginRequiredMixin,
    GetSuggestionCopySingleMixin,
    JSONResponseMixin,
    View
):
    """
    Mark the current suggestion for the user as actioned and return a new
    suggestion
    """
    def get(self, request, *ar, **kw):
        obj = self.get_object(kw['id'])
        obj.suggestion.actioned_by.add(request.user)
        suggestion = SuggestionCopy.objects.create_random_for_user(
            request.user
        )
        return self.render_to_response({'suggestion': suggestion.data})

class LikeSuggestionView(
    LoginRequiredMixin,
    GetSuggestionCopySingleMixin,
    JSONResponseMixin,
    View
):
    """
    Mark a suggestion as liked by the user and return the amount of likes
    """
    def get(self, request, *ar, **kw):
        obj = self.get_object(kw['id'])
        obj.suggestion.liked_by.add(request.user)
        return self.render_to_response(
            {'likes': obj.suggestion.liked_by.count()}
        )

class PutBackView(
    LoginRequiredMixin,
    GetSuggestionCopySingleMixin,
    JSONResponseMixin,
    View
):
    """
    Put a crossed suggestion back to current by making a copy of it
    """
    def get(self, request, *ar, **kw):
        obj = self.get_object(kw['id'])
        self.get_queryset()[0].delete()
        suggestion = SuggestionCopy.objects.create_from_suggestion_for_user(
            obj.suggestion,
            request.user
        )
        return self.render_to_response({'suggestion': suggestion.data})

class UpdateTextView(
    LoginRequiredMixin,
    GetSuggestionCopySingleMixin,
    JSONResponseMixin,
    View
):
    """
    Update the text for "them" in the selected suggestion
    """
    def post(self, request, *ar, **kw):
        obj = self.get_object(kw['id'])
        if not 'text' in request.POST:
            return HttpResponseBadRequest('No text supplied')
        obj.them_text = request.POST['text']
        obj.save()
        return self.render_to_response({'status': 'success'})

class AddSuggestionView(LoginRequiredMixin, CreateView):
    """
    Allow a logged in user to add their own suggestion for review that can
    be added (by an admin) to the pool of suggestions given on the site
    """
    template_name = 'suggestions/add.html'
    form_class = AddSuggestionForm

    def get_success_url(self):
        return reverse('suggestions:add')

    def form_valid(self, form):
        form.save(self.request.user)
        messages.success(self.request, _('Thank you for your suggestion'))
        return HttpResponseRedirect(self.get_success_url())


from time import sleep
import psycopg2
import os
import subprocess
from datetime import datetime
from pytz import timezone
import filecmp

db_name = os.environ['POSTGRES_DB']
db_user = os.environ['POSTGRES_USER']
db_pass = os.environ['PGPASSWORD']
db_host =  os.environ['POSTGRES_HOST']
db_port = os.environ['POSTGRES_PORT']
#frequency = 18000 if os.environ['BACKUP_INTERVAL'] is None else os.environ['BACKUP_INTERVAL']
#/mnt/data/db
KEEP_ALL = 0

def purgeBackup(old_file,new_file):
    if filecmp.cmp(old_file,new_file) == True:
        os.remove(old_file)

def loadBackup(backup_file):
    ps = subprocess.Popen(
        ['psql', '-h', db_host, '-U', db_user, '-d', db_name, '-f', backup_file],
        stdout=subprocess.PIPE
    )
    output = ps.communicate()[0]
    for line in output.splitlines():
        print(line)

def getFileInfo(path):
    result={}
    result['path'] = path
    # file modification timestamp of a file
    m_time = os.path.getmtime(path)
    # file creation timestamp in float
    c_time = os.path.getctime(path)

    # convert timestamp into DateTime object
    result['modifiedOn'] = datetime.fromtimestamp(m_time)
    # convert creation timestamp into DateTime object
    result['createdOn'] = datetime.fromtimestamp(c_time)

    return result

#get last backup
data = [getFileInfo('/mnt/data/db/{}'.format(item)) for item in os.listdir("/mnt/data/db") if item.endswith(".sql")]
sorted_data=sorted(data, key=lambda i: i['modifiedOn'],reverse=True)
last_backup = sorted_data[0]['path']
print(f"Last backup: {last_backup}")

#Connecto to the database
db_string = 'postgres://{}:{}@{}:{}/{}'.format(db_user, db_pass, db_host, db_port, db_name)
print(db_string)
db = psycopg2.connect(db_string)
cursor = db.cursor()
cursor.execute("SELECT version();")
version = cursor.fetchone()
print(version)

cursor.execute("SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'public'")
count = cursor.fetchone()

if len(count) > 0 and count[0] == 0:
    print("*** RECREATE DB from Backup! ****")
    print('loading {}'.format(last_backup))
    loadBackup(last_backup)

else:
    print("continue....")

db.close()

while True:
    tz = timezone('EST')
    x = datetime.now(tz)
    new_backup=f'/mnt/data/db/postgres-backup-{x.strftime("%m%d%y%H%M%S")}.sql'
    print('Backing up %s database to %s' % (db_name, new_backup))
    ps = subprocess.Popen(
        ['pg_dump', '-h', db_host, '-U', db_user, '-d', db_name, '-f', new_backup],
        stdout=subprocess.PIPE)
    output = ps.communicate()[0]

    for line in output.splitlines():
        print(line)
    if KEEP_ALL == 0:
        purgeBackup(last_backup,new_backup)
    last_backup = new_backup

    sleep(3600)


#!/usr/bin/env python3

import re
import collections
import vcf
import sys
import argparse
import copy

def parse_my_args():
    parser = argparse.ArgumentParser("Combines a VCF of individual calls into one large VCF for population.")
    parser.add_argument("vcf", nargs="?", help="Input VCF file; default stdin.")
    parser.add_argument("-n", "--name", help="name of combined population.", required=True)

    args = parser.parse_args()
    return(args)

def get_arg_vars(args):
    if args.vcf:
        inconn = open(args.vcf, "r")
    else:
        inconn = sys.stdin
    name = args.name
    return(inconn, name)

def combine_vcf(vcfin, name, outwriter):
    regex = re.compile(r"[/|]")
    for record in vcfin:
        calls = [call for call in record]
        ad1 = 0
        ad2 = 0
        for call in calls:
            callgt = regex.split(call["GT"])
            if len(callgt) == 1:
                inc = 2
            else:
                inc = 1
            for i in callgt:
                if i=="0":
                    ad1 += inc
                if i=="1":
                    ad2 += inc
        if ad1 >0 and ad2 > 0:
            gt = "0|1"
        elif ad2 > 0:
            gt = "1"
        else:
            gt = "0"
        writeout(gt, ad1, ad2, name, record, outwriter)
        #print(cmh.summary())

        #control_ad0, control_ad1 = [calls.AD[:2] for i in control]
        #test_ad0, test_ad1 = [calls.AD[:2] for i in test]

def writeout(gt, ad1, ad2, name, record, writer):
    newrecord = copy.deepcopy(record)
    newrecord.samples = []
    CallData = collections.namedtuple("CallData", ["GT", "AD"])
    mycalldat = CallData(GT = str(gt), AD = [str(ad1), str(ad2)])
    newrecord.samples.append(vcf.model._Call(newrecord, name, mycalldat))
    writer.write_record(newrecord)

def main():
    args = parse_my_args()

    inconn, name = get_arg_vars(args)

    vcfin = vcf.Reader(inconn)
    outwriter = vcf.Writer(sys.stdout, vcfin)
    combine_vcf(vcfin, name, outwriter)

    inconn.close()

if __name__ == "__main__":
    main()

#>>> for i in b:
#...     for j in i:
#...         try:print(j.data.AD)


from typing import List

class WordFilter:

    def __init__(self, words: List[str]):
        self.a ={}
        for ind, i in enumerate(words):
            for j in range(len(i) + 1):
                for k in range(len(i) + 1):
                    now = i[:j] + '$' + i[k:]
                    self.a[now] = ind

    def f(self, prefix: str, suffix: str) -> int:
        k = prefix + '$' + suffix
        return self.a.get(k, -1)

if __name__ == '__main__':
    words = ["apple"]
    prefix = "a"
    suffix = "e"
    obj = WordFilter(words)
    print(obj.f(prefix, suffix))
# Your WordFilter object will be instantiated and called as such:
# obj = WordFilter(words)
# param_1 = obj.f(prefix,suffix)


#!/usr/bin/env python3
import os
import sys
import rzpipe

curdir = os.path.dirname(os.path.realpath(__file__))

rz = rzpipe.open(curdir + "/ls", ["-2"])

# print(rzpipe.__file__)
# print(rzpipe.VERSION)

rz.cmd("aa")

sys.stdout.write("/bin/ls    ")

pi1 = rz.cmd("pi 1 @e:scr.color=0").strip()
if pi1 == "push rbp":
    print("OK")
else:
    print("FAIL")
# print(pi1)
# print(rz.cmd("pd 10"));
rz.quit()


import xml.etree.ElementTree as ET
from datetime import datetime
from bs4 import BeautifulSoup

class XMLParser:
    def __init__(self, file: str):
        self.file = file
        self.parameters = {'INPUT':{},
                        'DISCRIMINATOR':{},
                        'QDC':{},
                        'SPECTRA':{},
                        'REJECTIONS':{},
                        'ENERGY_CALIBRATION':{},
                        'SYNC':{},
                        'HARDWARE_COINCIDENCE':{},
                        'MISC':{}}
        self.groups = list(self.parameters.keys())
        self.reformatted = ['SRV_PARAM_CH_POLARITY','SRV_PARAM_CH_BLINE_NSMEAN','HARDWARE_COINCIDENCE','SRV_PARAM_START_MODE','SRV_PARAM_CH_SPECTRUM_NBINS','SRV_PARAM_CH_INDYN','SRV_PARAM_CH_CFD_FRACTION','SRV_PARAM_CH_DISCR_MODE','SRV_PARAM_CH_ENERGY_COARSE_GAIN','SRV_PARAM_TRGOUT_MODE']
        self.reformatted_keys = ['polarity', 'baseline', 'coincidence', 'start', 'ebins', 'input_range', 'cfd', 'discriminator', 'coarse_gain', 'trig_out']
        self.formatted = 0
        self.board_formatted = False

    def get_board_properties(self):
        root = ET.parse(self.file).getroot()
        name = root.find('board/label').text
        id = root.find('board/id').text
        model = root.find('board/modelName').text
        adc_bits = root.find('board/adcBitCount').text
        sample_rate = int(root.find('board/sampleTime').text)*10**6
        dpp_type = root.find('board/dppType').text
        roc = root.find('board/rocFirmware/major').text + '.' + root.find('board/rocFirmware/minor').text + ' build ' + str(hex(int(root.find('board/rocFirmware/build').text))).split('0x')[-1].zfill(4)
        amc = root.find('board/amcFirmware/major').text + '.' + root.find('board/amcFirmware/minor').text + ' build ' + str(hex(int(root.find('board/amcFirmware/build').text))).split('0x')[-1].zfill(4)
        link = root.find('board/connectionType').text + ' link #' + root.find('board/address').text
        status = root.find('board/active').text
        if status == 'true':
            status = True
        if status == 'false':
            status = False
        return name, id, model, adc_bits, sample_rate, dpp_type, roc, amc, link, status

    def reformat(self, list_of_params):
        """
        Reformats the values of the XML file.

        Args:
            list_of_params (list): Contains `polarity`, `baseline`, `coincidence`, `start`, `ebins`, `input_range`, `cfd`, `discriminator`, `coarse_gain`, 'trig_out`.
        """
        if 'all' in list_of_params:
            list_of_params.clear()
            list_of_params = ['polarity', 'baseline', 'coincidence', 'start', 'ebins', 'input_range', 'cfd', 'discriminator', 'coarse_gain', 'trig_out']

        #Formatting of the text values
        if 'polarity' in list_of_params:
            pol = self.parameters['INPUT']['SRV_PARAM_CH_POLARITY']
            real_pol = pol.split('_')[-1]
            self.parameters['INPUT']['SRV_PARAM_CH_POLARITY'] = real_pol

        if 'baseline' in list_of_params:
            bline = self.parameters['INPUT']['SRV_PARAM_CH_BLINE_NSMEAN']
            real_bline = bline.split('_')[-1]
            self.parameters['INPUT']['SRV_PARAM_CH_BLINE_NSMEAN'] = real_bline

        if 'coincidence' in list_of_params:
            coinc_mode = self.parameters['HARDWARE_COINCIDENCE']['SRV_PARAM_COINC_MODE']
            real_coinc_mode = coinc_mode[11:]
            self.parameters['HARDWARE_COINCIDENCE']['SRV_PARAM_COINC_MODE'] = real_coinc_mode

        if 'start' in list_of_params:
            start_mode = self.parameters['SYNC']['SRV_PARAM_START_MODE']
            real_start_mode = start_mode[11:]
            self.parameters['SYNC']['SRV_PARAM_START_MODE'] = real_start_mode

        if 'ebins' in list_of_params:
            energy_bins = self.parameters['SPECTRA']['SRV_PARAM_CH_SPECTRUM_NBINS']
            real_energy_bins = energy_bins[5:] +'.0'
            self.parameters['SPECTRA']['SRV_PARAM_CH_SPECTRUM_NBINS'] = real_energy_bins

        if 'input_range' in list_of_params:
            input_range = self.parameters['INPUT']['SRV_PARAM_CH_INDYN']
            input_range = input_range.split('_')[1:]
            real_input_range = input_range[0] + '.' + input_range[1]
            self.parameters['INPUT']['SRV_PARAM_CH_INDYN'] = real_input_range

        if 'cfd' in list_of_params:
            cfd_frac = self.parameters['DISCRIMINATOR']['SRV_PARAM_CH_CFD_FRACTION']
            real_cfd_frac = cfd_frac.split('_')[-1]
            self.parameters['DISCRIMINATOR']['SRV_PARAM_CH_CFD_FRACTION'] = real_cfd_frac

        if 'discriminator' in list_of_params:
            disc_mode = self.parameters['DISCRIMINATOR']['SRV_PARAM_CH_DISCR_MODE']
            disc_mode = disc_mode.split('_')[-1]
            if disc_mode == "LED":
                real_disc_mode = "Leading Edge Discriminator"
            if disc_mode == "CFD":
                real_disc_mode = "Constant Fraction Discriminator"
            self.parameters['DISCRIMINATOR']['SRV_PARAM_CH_DISCR_MODE'] = real_disc_mode

        if 'coarse_gain' in list_of_params:
            coarse_gain = self.parameters['QDC']['SRV_PARAM_CH_ENERGY_COARSE_GAIN']
            self.parameters['QDC']['SRV_PARAM_CH_ENERGY_COARSE_GAIN'] = coarse_gain.split('_')[1]

        if 'trig_out' in list_of_params:
            trig_out = self.parameters['SYNC']['SRV_PARAM_TRGOUT_MODE'].split('_')[2:]
            real_trig_out =""
            for elem in trig_out:
                real_trig_out += elem + ' '
            self.parameters['SYNC']['SRV_PARAM_TRGOUT_MODE'] = real_trig_out

        #FIXING THE VALUES THAT ARE IN NANOSECONDS TO SECONDS FOR SPINMOB AUTOSCALING
        if self.formatted == 0:
            self.parameters['INPUT']['SRV_PARAM_RECLEN'] = float(self.parameters['INPUT']['SRV_PARAM_RECLEN'])*10**(-9)
            self.parameters['INPUT']['SRV_PARAM_CH_PRETRG'] = float(self.parameters['INPUT']['SRV_PARAM_CH_PRETRG'])*10**(-9)
            self.parameters['DISCRIMINATOR']['SRV_PARAM_CH_TRG_HOLDOFF'] = float(self.parameters['DISCRIMINATOR']['SRV_PARAM_CH_TRG_HOLDOFF'])*10**(-9)
            self.parameters['DISCRIMINATOR']['SRV_PARAM_CH_CFD_DELAY'] = float(self.parameters['DISCRIMINATOR']['SRV_PARAM_CH_CFD_DELAY'])*10**(-9)
            self.parameters['QDC']['SRV_PARAM_CH_GATE'] = float(self.parameters['QDC']['SRV_PARAM_CH_GATE'])*10**(-9)
            self.parameters['QDC']['SRV_PARAM_CH_GATESHORT'] = float(self.parameters['QDC']['SRV_PARAM_CH_GATESHORT'])*10**(-9)
            self.parameters['QDC']['SRV_PARAM_CH_GATEPRE'] = float(self.parameters['QDC']['SRV_PARAM_CH_GATEPRE'])*10**(-9)
            self.parameters['SPECTRA']['SW_PARAMETER_TIME_DISTRIBUTION_CH_T0'] = float(self.parameters['SPECTRA']['SW_PARAMETER_TIME_DISTRIBUTION_CH_T0'])*10**(-9)
            self.parameters['SPECTRA']['SW_PARAMETER_TIME_DISTRIBUTION_CH_T1'] = float(self.parameters['SPECTRA']['SW_PARAMETER_TIME_DISTRIBUTION_CH_T1'])*10**(-9)
            self.parameters['SPECTRA']['SW_PARAMETER_TIME_DIFFERENCE_CH_T0'] = float(self.parameters['SPECTRA']['SW_PARAMETER_TIME_DIFFERENCE_CH_T0'])*10**(-9)
            self.parameters['SPECTRA']['SW_PARAMETER_TIME_DIFFERENCE_CH_T1'] = float(self.parameters['SPECTRA']['SW_PARAMETER_TIME_DIFFERENCE_CH_T1'])*10**(-9)
            self.parameters['HARDWARE_COINCIDENCE']['SRV_PARAM_COINC_TRGOUT'] = float(self.parameters['HARDWARE_COINCIDENCE']['SRV_PARAM_COINC_TRGOUT'])*10**(-9)
            self.parameters['REJECTIONS']['SW_PARAMETER_CH_ENERGYCUTENABLE'] = (True if self.parameters['REJECTIONS']['SW_PARAMETER_CH_ENERGYCUTENABLE'] == 'true' else False)
        self.formatted += 1

    def get_parameters(self):
        """
        Gets the board parameters (shared parameters for all channels).
        """
        root = ET.parse(self.file).getroot()
        board_parameters = root.find('board/parameters')
        for entry in board_parameters:
            key = entry.find('key').text
            value = entry.find('value/value').text
            group = entry.find('value/descriptor/group').text

            if value == 'true':
                value = True
            if value == 'false':
                value = False

            for tab in self.groups:
                if group == tab:
                    #if units is None:
                    self.parameters[tab][key] = value
                    #else:
                        #self.parameters[tab][key] = [value, units]
        self.formatted = 0
        self.reformat(['all'])

        return self.parameters

    def get_chn_parameters(self, chn_number: str):
        root = ET.parse(self.file).getroot()
        channels = root.findall('board/channel')
        channel_in_use = channels[chn_number]
        # keys = channel_in_use.findall('values/entry/key')
        # values = channel_in_use.findall('values/entry/value')
        entries = channel_in_use.findall('values/entry')
        entries_with_vals = []
        for index, entry in enumerate(entries):
            if entry.find('value') is not None:
                entries_with_vals.append(entries[index])

        keys = []
        values = []
        for entry in entries_with_vals:
            keys.append(entry.find('key'))
            values.append(entry.find('value'))

        list_format = []
        for key in keys:
            if key.text in self.reformatted:
                list_format.append(self.reformatted_keys[self.reformatted.index(key.text)])

        for group in self.parameters:
            for index, key in enumerate(keys):
                if key.text in self.parameters[group]:
                    if 'true' in values[index].text or 'false' in values[index].text:
                        values[index].text = (True if values[index].text == 'true' else False)
                    else:
                        self.parameters[group][key.text] = values[index].text

        # self.formatted = 0
        # self.reformat(list_format)
        return self.parameters

    def get_ch_label(self, chn_number: str):
        root = ET.parse(self.file).getroot()
        channels = root.findall('board/channel')
        channel_to_check = channels[chn_number]
        index = channel_to_check.find('index').text
        entries = channel_to_check.findall('values/entry')
        for entry in entries:
            if entry.find('key').text == "SW_PARAMETER_CH_LABEL":
                if entry.find('value') is not None:
                    label = entry.find('value').text
                    break
        else: # I'm a genius. Don't mind me using disgusting functions in python.
            label = "CH" #This is executed if the loop ends normally (so without encountering the break above.)
        return (index, label)

class InfoParser:
    def __init__(self, file: str):
        self.file = file

    def get_run_info(self):
        with open(self.file) as f:
            informations = f.readlines()[0:4]
        self.id = informations[0].split('=')[-1][:-1]
        self.time_start = datetime.strptime(informations[1].split('=')[-1].split('.')[0], "%Y/%m/%d %H:%M:%S")
        self.time_stop = datetime.strptime(informations[2].split('=')[-1].split('.')[0], "%Y/%m/%d %H:%M:%S")
        self.time_real = self.time_stop - self.time_start
        return self.id, self.time_start, self.time_stop, self.time_real

if __name__ == '__main__':
    file = "C:\\Users\\chloe\\OneDrive - McGill University\\Coincidence Testing\\Co60 Spectrums with different settings\\DAQ\\4096Chns-20lsb(LE)-80Gain-(300.80.50)-150s\\settings.xml"
    test = XMLParser(file)
    test.get_parameters()
    print(test.get_ch_label(2))


import socket
udpsocket = socket.socket(family=socket.AF_INET, type=socket.SOCK_DGRAM)
udpsocket.bind(("localhost",7777))
print("UDP server is up and listening")

pair = udpsocket.recvfrom(1024)
res = pair[0]
add = pair[1]
print("Response from client :")
print("Message Received: ", res.decode())

print("Address of client: ", add, end = '\n\n')

udpsocket.sendto(str.encode("Connected to Server!"), add)

while(True):
    pair = udpsocket.recvfrom(1024)
    res = pair[0]
    add = pair[1]
    print("Client: ", res.decode())

    message = input("ME: ")
    udpsocket.sendto(str.encode(message), add)


import sys
sys.path.append('../')
# sys.path.insert(0, '/path/to/application/app/folder')
# from flask import Flask
#from flask_testing import TestCase
import unittest
from unittest import TestCase
import read_ini

# class MyTest(TestCase):
#     def create_app(self):
#         app = Flask(__name__)
#         app.config['TESTING'] = True
#         return app
#

class TestDBcon(TestCase):
    def setUp(self):
        TestCase.setUp(self)

    def test_settings_ini_has_all_required_keys(self):
        expected_keys = ['name', 'age' , 'height']
        filename = '../settings.ini'
        ini_to_dict = read_ini.convert_settings_ini_to_dict(filename)
        keys_retrived_from_ini_file = list(ini_to_dict) # or [*ini_to_dict]
        for k in expected_keys:
            self.assertIn(k,
                      keys_retrived_from_ini_file,
                      'we got %s' %
                             keys_retrived_from_ini_file)
#         self.assertListEqual(keys_retrived_from_ini_file,
#                              expected_keys, 'we got %s' %
#                              keys_retrived_from_ini_file)

    def test_get_value_from_settings_ini_file(self):
        expected_result = 'bhujay'
        self.assertEqual(read_ini.get_value_from_settings_ini_file('name'),
                         expected_result, 'Test result is % s' %
                         read_ini.get_value_from_settings_ini_file('name'))

if __name__ == '__main__':
    unittest.main()


#!/usr/bin/env python3
"""antipatibot, discord server."""
import asyncio
import logging
import os
import secrets
from dataclasses import dataclass

import discord
from discord.ext import commands
import yt_dlp as youtube_dl

youtube_dl.utils.bug_reports_message = lambda: ''

ytdl_format_options = {
    'format': 'bestaudio/best',
    'outtmpl': '%(extractor)s-%(id)s-%(title)s.%(ext)s',
    'restrictfilenames': True,
    'noplaylist': False,
    'extract_flat': 'in_playlist',  # don't extract stream urls / thumbnails # ... for playlists.
    'nocheckcertificate': True,
    'ignoreerrors': False,
    'logtostderr': False,
    'quiet': True,
    'no_warnings': True,
    'default_search': 'auto',
    'source_address': '0.0.0.0'  # bind to ipv4 since ipv6 addresses cause issues sometimes
}

ffmpeg_options = {
    'options': '-vn'
}

ytdl = youtube_dl.YoutubeDL(ytdl_format_options)

class YTDLSource(discord.PCMVolumeTransformer):
    """Youtube source class, which allows the bot to play youtube videos"""

    def __init__(self, source, *, data, volume=0.5):
        super().__init__(source, volume)
        self.data = data
        self.title = data.get('title')
        self.url = data.get('url')

    @classmethod
    async def from_url(cls, url, *, loop=None, stream=False):
        """Returns an audio from a youtube link."""
        loop = loop or asyncio.get_event_loop()
        data = await loop.run_in_executor(None, lambda: ytdl.extract_info(url, download=not stream))
        if 'entries' in data:
            # Take first item from a playlist.
            # This should never happen, since we handle this in the play command,
            # but better safe than sorry.
            data = data['entries'][0]
        filename = data['url'] if stream else ytdl.prepare_filename(data)
        return cls(discord.FFmpegPCMAudio(filename, **ffmpeg_options), data=data)

@dataclass()
class BotSettings:
    """Bot settings associated to each antipatibot instance."""

    discord_token: str = os.getenv("ANTIPATIBOT_DISCORD_TOKEN", "")
    command_prefix: str = os.getenv("ANTIPATIBOT_COMMAND_PREFIX", "!")
    max_queue_size: int = int(os.getenv("ANTIPATIBOT_MAX_QUEUE_SIZE", "1000"))
    idle_timeout: int = int(os.getenv("ANTIPATIBOT_IDLE_TIMEOUT", "300"))

@dataclass()
class GuildData:
    """Data associated to each guild: song queue, music task and lock."""

    guild_id: int = None
    lock: asyncio.Lock = asyncio.Lock()
    queue: asyncio.Queue = None
    task: asyncio.Task = None
    loop: bool = False

    def __init__(self, guild_id: int, max_queue_size: int, task: asyncio.Task = None):
        self.guild_id = guild_id
        self.task = task
        self.queue = asyncio.Queue(max_queue_size)

# pylint: disable=R0201,R0904
class AntipatiBot(commands.Cog):
    """AntipatiBot's collection of command."""

    def __init__(self, bot, log, settings: BotSettings):
        self.bot = bot
        self.log = log
        self.settings = settings
        self.guild_data = {}

    async def cog_command_error(self, ctx, error):
        message = ctx.message.content
        self.log.error(f"command_error:{ctx.guild.id}:{self.log.sanitize(ctx.author)}" +
                       f":{ctx.command}:{ctx.author.id}:{message}:{error}")
        await ctx.message.reply("Invalid command.")

    async def cog_before_invoke(self, ctx):
        self.log.info(
            f"command:{ctx.guild.id}:{self.log.sanitize(ctx.author)}:{ctx.author.id}:{ctx.command}")

    @commands.Cog.listener()
    async def on_ready(self):
        """Triggers when the bot is ready to run, used to log basic information."""
        self.log.info("login:%s", self.bot.user)
        for guild in self.bot.guilds:
            self.log.info("joined_guild:%d:%s", guild.id, self.log.sanitize(guild.name))
            self.guild_data[guild.id] = GuildData(guild.id, self.settings.max_queue_size)

    async def ensure_guild_thread(self, guild_id: int):
        """Ensure the music_player_loop thread is running for the given guild_id"""
        guild_data = self.guild_data[guild_id]
        async with guild_data.lock:
            if guild_data.task is None:
                guild_data.task = asyncio.create_task(self.music_player_loop(guild_data))

    async def terminate_guild_thread(self, guild_id: int):
        """Kill the music_player_loop thread for the given guild_id"""
        guild_data = self.guild_data[guild_id]
        async with guild_data.lock:
            if guild_data.task is not None:
                guild_data.task.cancel()
                guild_data.queue = asyncio.Queue(self.settings.max_queue_size)
                guild_data.loop = False
                guild_data.task = None

    @commands.Cog.listener()
    async def on_voice_state_update(self, member: discord.Member, before: discord.VoiceState,
                                    after: discord.VoiceState):
        """Triggers when the bot joins or leaves a voice channel.
           Starts the music_player_loop for the given guild."""
        guild_id = member.guild.id

        if self.bot.user == member:
            """Ensure the music_player_loop is alive/dead"""
            if after.channel is not None:
                await self.ensure_guild_thread(guild_id)
            elif after.channel is None:
                await self.terminate_guild_thread(guild_id)
        else:
            """Terminate when you are left alone on a channel"""
            voice = discord.utils.get(self.bot.voice_clients, guild__id=guild_id)
            if voice is not None and len(voice.channel.members) < 2:
                # FIXME: possible race condition on idle timeout
                await voice.disconnect()
                await self.terminate_guild_thread(guild_id)

    async def music_player_loop(self, guild_data: GuildData):
        """Task which handles the queue list, cross-guild in theory (wip)."""
        self.log.info(f"music_player_loop:{guild_data.guild_id}:start")
        while True:
            try:
                (song_request, ctx) = \
                    await asyncio.wait_for(guild_data.queue.get(), self.settings.idle_timeout)
                self.log.info("song request: " + str(song_request))
                player = await YTDLSource.from_url(song_request, loop=self.bot.loop, stream=True)
                playing_current_song = asyncio.Event()

                def on_song_end(error):
                    if error is not None:
                        self.log.error("Player error: %s", error)
                    playing_current_song.set()

                ctx.voice_client.play(player, after=on_song_end)
                await ctx.send(f"Now playing: {player.title}")
                await playing_current_song.wait()
                if guild_data.loop:
                    try:
                        guild_data.queue.put_nowait((song_request, ctx))
                    except asyncio.QueueFull:
                        pass
            except asyncio.CancelledError:
                self.log.info(f"music_player_loop:{guild_data.guild_id}:cancelled")
                return
            except asyncio.TimeoutError:
                self.log.info(f"music_player_loop:{guild_data.guild_id}:timeout")
                voice = discord.utils.get(self.bot.voice_clients, guild__id=guild_data.guild_id)
                if voice is not None:
                    await voice.disconnect()
                return
            except Exception as exception:  # pylint: disable=W0703
                self.log.warning(f"music_player_loop:{guild_data.guild_id}:uncaught exception: {exception}")

    @commands.command()
    async def join(self, ctx, *, channel: discord.VoiceChannel = None):
        """
        Either join a given voice channel or move to the author voice channel.
        If no channel is specified, connect to the user's current voice channel.
        """""
        if channel is None:
            if ctx.author.voice is None:
                return await ctx.message.reply("You are not connected to a voice channel.")
            channel = ctx.author.voice.channel
        if ctx.voice_client is not None:
            await ctx.voice_client.move_to(channel)
        else:
            await channel.connect()
        await self.ensure_guild_thread(ctx.guild.id)

    @commands.command(aliases=["cicca"])
    async def cichero(self, ctx):
        """Great classic."""
        return await self.play(ctx, song_request="https://www.youtube.com/watch?v=DAuPe14li4g")

    @commands.command(aliases=["jhon"])
    async def john(self, ctx):
        """He truly is."""
        return await self.play(ctx, song_request="https://www.youtube.com/watch?v=dALcFSyFcXs")

    @commands.command()
    async def bastardo(self, ctx):
        """Mia mamma è una brava donna, napoletana ma brava"""
        return await self.play(ctx, song_request="https://www.youtube.com/watch?v=12gmyUCqLxA")

    @commands.command(aliases=["p", "youtube", "yt"])
    async def play(self, ctx, *, song_request: str):
        """Add youtube song to playlist."""
        async with ctx.typing():
            guild_data = self.guild_data[ctx.guild.id]
            songs = [ytdl.extract_info(song_request, download=False)]
            if "entries" in songs[0]:
                # YouTube playlist
                songs = list(song for song in songs[0]["entries"])
            for song in songs:
                try:
                    guild_data.queue.put_nowait((song["url"], ctx))
                except asyncio.QueueFull:
                    await ctx.message.reply(
                        f"Song queue is full (Max size: {guild_data.queue.maxsize})")
                    return
            await ctx.message.reply("Song added to the queue" if len(songs) == 1
                                    else f"Added {len(songs)} songs to the queue.")

    @commands.command(aliases=["clear", "clean", "hairottoilcazzo"])
    async def stop(self, ctx, *, reply=True):
        """Clear the queue and stop playing music"""
        guild_data = self.guild_data[ctx.guild.id]
        try:
            while True:
                guild_data.queue.get_nowait()
        except asyncio.QueueEmpty:
            await self.skip(ctx)
            if reply:
                await ctx.message.reply("Song queue cleared and music stopped")

    @commands.command(aliases=["kill", "terminate", "harakiri"])
    async def disconnect(self, ctx):
        """Clear the queue, stop playing music and disconnect from the channel"""
        await self.stop(ctx, reply=False)
        if ctx.voice_client is not None:
            await ctx.voice_client.disconnect()

    @commands.command(aliases=["next"])
    async def skip(self, ctx):
        """Skip the song that is currently playing."""
        if ctx.voice_client is not None and ctx.voice_client.is_playing():
            ctx.voice_client.stop()

    @commands.command()
    async def loop(self, ctx):
        """Toggle the loop functionality"""
        async with ctx.typing():
            guild_data = self.guild_data[ctx.guild.id]
            guild_data.loop = not guild_data.loop
        await ctx.message.reply(f"Loop {'activated' if guild_data.loop else 'deactivated'}")

    @commands.command(aliases=["die", "roll"])
    async def dice(self, ctx, num: int = 1, sides: int = 20, show_sides: bool = True):
        """Roll an n sided dice"""
        if sides < 1 or sides > 0x1337 or num < 1 or num > 40:
            return await ctx.message.reply("You have been added to a list.")
        if num == 1:
            return await ctx.message.reply((f"[d{sides}] " if show_sides else "") +
                                           f"You rolled a {secrets.randbelow(sides) + 1}")
        rolls = [secrets.randbelow(sides) + 1 for _ in range(num)]
        return await ctx.message.reply(
            f"[{num}d{sides}] You rolled {'+'.join([str(r) for r in rolls])} = {sum(rolls)}")

    # pylint: disable=C0103
    @commands.command()
    async def d4(self, ctx, n=1):
        """Roll a 4-sided dice"""
        await self.dice(ctx, sides=4, num=n, show_sides=False)

    @commands.command()
    async def d6(self, ctx, n=1):
        """Roll a 6-sided dice"""
        await self.dice(ctx, sides=6, num=n, show_sides=False)

    @commands.command()
    async def d8(self, ctx, n=1):
        """Roll a 8-sided dice"""
        await self.dice(ctx, sides=8, num=n, show_sides=False)

    @commands.command()
    async def d10(self, ctx, n=1):
        """Roll a 10-sided dice"""
        await self.dice(ctx, sides=10, num=n, show_sides=False)

    @commands.command()
    async def d12(self, ctx, n=1):
        """Roll a 10-sided dice"""
        await self.dice(ctx, sides=12, num=n, show_sides=False)

    @commands.command()
    async def d20(self, ctx, n=1):
        """Roll a 20-sided dice"""
        await self.dice(ctx, sides=20, num=n, show_sides=False)

    @commands.command()
    async def d100(self, ctx, n=1):
        """Roll a 100-sided dice"""
        await self.dice(ctx, sides=100, num=n, show_sides=False)

    @play.before_invoke
    @cichero.before_invoke
    @john.before_invoke
    @bastardo.before_invoke
    async def ensure_voice(self, ctx):
        """Pre-hook used to ensure you the bot is connected to a voice channel before starting to
        play music."""
        return await self.join(ctx)

def main():
    """Entrypoint for antipatibot program"""
    logging.basicConfig(level=logging.INFO)
    logging.getLogger("discord").setLevel(logging.WARNING)
    log = logging.getLogger("antipatibot")
    #    log.setLevel(logging.DEBUG)
    settings = BotSettings()
    bot = commands.Bot(command_prefix=commands.when_mentioned_or(settings.command_prefix),
                       description="AntipatiBot")

    log.sanitize = lambda message: str(message).replace(":", "_") \
        .replace("\r", "\\r") \
        .replace("\n", "\\n") \
        .replace("\t", "\\t")

    bot.add_cog(AntipatiBot(bot, log, settings))
    try:
        discord_api_file = "/antipatibot/discord_token.txt"
        if os.path.exists(discord_api_file) and os.path.isfile(discord_api_file):
            with open(discord_api_file, encoding='utf8') as file:
                settings.discord_token = file.read().strip("\n\r\t ")
        bot.run(settings.discord_token)
    except discord.errors.LoginFailure:
        log.error("invalid_discord_token:Please set a valid discord bot API token.")

if __name__ == "__main__":
    main()


import pytest
import common

@pytest.mark.parametrize(
    "data,start,end",
    [
        ("0-0", 0, 0),
        ("11-22", 11, 22),
    ],
)
def test_parse(data: str, start: int, end: int):
    assert common.SectionRange.parse(data) == common.SectionRange(start, end)

@pytest.mark.parametrize(
    "range1,range2,res",
    [
        ("1-2", "3-4", False),
        ("1-2", "2-3", False),
        ("1-2", "2-2", True),
        ("1-3", "2-2", True),
        ("2-2", "2-2", True),
        ("2-3", "2-2", True),
        ("2-3", "1-2", False),
        ("2-3", "1-1", False),
    ],
)
def test_contains(range1: str, range2: str, res: bool):
    sr1 = common.SectionRange.parse(range1)
    sr2 = common.SectionRange.parse(range2)
    assert sr1.contains(sr2) == res

@pytest.mark.parametrize(
    "range1,range2,res",
    [
        ("1-2", "3-4", False),
        ("1-2", "2-3", True),
        ("1-2", "2-2", True),
        ("1-3", "2-2", True),
        ("2-2", "2-2", True),
        ("2-3", "2-2", True),
        ("2-3", "1-2", True),
        ("2-3", "1-1", False),
    ],
)
def test_overlaps(range1: str, range2: str, res: bool):
    sr1 = common.SectionRange.parse(range1)
    sr2 = common.SectionRange.parse(range2)
    assert sr1.overlaps(sr2) == res

2022 - expedition/04 - camp cleanup/test_common.py


"""
Data import
https://github.com/tategallery/collection
"""
import json
import os
import sys
import time
import pandas as pd

csv_file = '/media/joji/DATA/workspace/data/artwork_data.csv'
selected_columns = ['id', 'artist', 'title', 'medium', 'year',
                    'acquisitionYear', 'height', 'width', 'units']

def main():
    df = pd.read_csv(csv_file, index_col='id', usecols=selected_columns)
    print(df)
    print(f"Unique artists: {len(df['artist'].unique())}")
    # print(len(set(df['artist'])))
    tf_series = df['artist'] == 'Bacon, Francis'
    print(tf_series.value_counts())
    print(type(tf_series.value_counts()))
    print(df['artist'].value_counts()['Bacon, Francis'])

if __name__ == '__main__':
    t01 = time.time()
    main()
    t02 = time.time()
    print(f'{sys.argv[0]} took {t02 - t01:.0f} seconds')


import os
from twilio.rest import Client
from urllib.request import urlopen

import re
import time
import smtplib

#need twilio credientials to run
account_sid = os.environ["TWILIO_ACCOUNT_SID"]
auth_token = os.environ["TWILIO_AUTH_TOKEN"]
twilio_number = os.environ["TWILIO_NUMBER"]
ubc_url = "https://courses.students.ubc.ca/cs/main?pname=subjarea&tname=subjareas&req=5&dept="

WAIT_TIME = 20

def sendMessage(message, phonenumber):
    client = Client(account_sid, auth_token)
    client.messages.create(
        to=phonenumber,
        from_=twilio_number,
        body=message
    )

def check_seats(url, user_info, regex_objects):
    web_page_text = urlopen(url).read()
    htmlText = web_page_text.decode("utf8")

    general = re.search(regex_objects["general_seats"], htmlText)
    restricted = re.search(regex_objects["restricted_seats"], htmlText)
    temp_unavailable = htmlText.find("Note: this section is temp. unavailable")

    print("Still looking...")
    print("Restricted Seats: ", restricted.group(1))
    print("General Seats: ", general.group(1))
    if temp_unavailable != -1:
        return 3
    if not general or not restricted:
        print("Something went wrong, maybe you put the wrong url in or lost internet connection, try restarting")
        return 0
    if general.group(1) != '0':
        return 1
    if restricted.group(1) != '0':
        return 2
    else:
        return 0

def compile_regex():
    regexs = {}
    regexs["general_seats"] = re.compile("General Seats Remaining:	(.*?)	")
    regexs["restricted_seats"] = re.compile("Restricted Seats Remaining\*:	(.*?)	")
    return regexs

#gathers nessesary user info
def gather_user_info():
    user_info = {}
    user_info['department'] = input("Enter department:")
    user_info["course_number"] = input("Enter course number: ")
    user_info["section"] = input("Enter section number: ")

    user_info["phone_number"] = input("Enter phone number:(in format +xxxxxxxxxxx) ")
    user_info["restricted"] = input("Are restricted seats okay?(yes/no)")
    return user_info

def main():
    user_info = gather_user_info()
    print(user_info['department'])
    defined_url = ubc_url + user_info["department"] + "&course=" + user_info["course_number"] + "&section=" + user_info["section"]
    regex_objects = compile_regex()
    compile_regex()
    while True:
        status = check_seats(defined_url, user_info, regex_objects)

        if status == 1:
            print("GENERAL SEAT AVAILABLE SENDING MESSAGE")
            sendMessage('There is a general seat available in ' + user_info["department"] + ' ' + user_info["course_number"] + '! Grab it here: ' + defined_url, user_info["phone_number"])
            break
        if status == 2:
            if user_info["restricted"] == "yes":
                print("RESTRICTED SEAT AVAILABLE")
                sendMessage('There is a restricted seat available in ' + user_info["department"] + ' ' + user_info["course_number"] + '! Grab it here: ' + defined_url, user_info["phone_number"])
            break
        if status == 3:
            print("The course is temporarily unavailable")
            time.sleep(WAIT_TIME)
        else:
            time.sleep(WAIT_TIME)

if __name__ == "__main__":
    main()


import boto3
from operator import itemgetter

ecr_client = boto3.client('ecr')
repositories = ecr_client.describe_repositories()['repositories']

if len(repositories) == 0:
    print("Repository is empty!")

for repo in repositories:
    print(f"Repository name: {repo['repositoryName']}")

query_repository_name = "java-maven-app"

images = ecr_client.describe_images(
        repositoryName=query_repository_name
    )

image_tags = []

for image in images['imageDetails']:
    image_tags.append({
        'tag': image['imageTags'],
        'date': image['imagePushedAt']
    })

sorted_images_based_on_date = sorted(image_tags, key=itemgetter("date"), reverse=True)
for image in sorted_images_based_on_date:
    print(image)


#encoding=utf8
import sys
reload(sys)
sys.setdefaultencoding('utf8')

import StringIO
import json
import logging
import random
import urllib
import urllib2

# functions
import responseHandler

# standard app engine imports
from google.appengine.api import urlfetch
from google.appengine.ext import ndb
import webapp2

# global variables
from config import TOKEN
BASE_URL = 'https://api.telegram.org/bot' + TOKEN + '/'

# ================================

class EnableStatus(ndb.Model):
    # key name: str(chat_id)
    enabled = ndb.BooleanProperty(indexed=False, default=False)

def setEnabled(chat_id, yes):
    es = EnableStatus.get_or_insert(str(chat_id))
    es.enabled = yes
    es.put()

def getEnabled(chat_id):
    es = EnableStatus.get_by_id(str(chat_id))
    if es:
        return es.enabled
    return False

class MeHandler(webapp2.RequestHandler):
    def get(self):
        urlfetch.set_default_fetch_deadline(60)
        self.response.write(json.dumps(json.load(urllib2.urlopen(BASE_URL + 'getMe'))))

class GetUpdatesHandler(webapp2.RequestHandler):
    def get(self):
        urlfetch.set_default_fetch_deadline(60)
        self.response.write(json.dumps(json.load(urllib2.urlopen(BASE_URL + 'getUpdates'))))

class SetWebhookHandler(webapp2.RequestHandler):
    def get(self):
        urlfetch.set_default_fetch_deadline(60)
        url = self.request.get('url')
        if url:
            self.response.write(json.dumps(json.load(urllib2.urlopen(BASE_URL + 'setWebhook', urllib.urlencode({'url': url})))))

class WebhookHandler(webapp2.RequestHandler):
    def post(self):
        urlfetch.set_default_fetch_deadline(60)
        body = json.loads(self.request.body)
        logging.info('request body:')
        logging.info(body)
        self.response.write(json.dumps(body))

        update_id = body['update_id']
        message = body['message']
        message_id = message.get('message_id')
        location = message.get('location')
        date = message.get('date')
        text = message.get('text')
        fr = message.get('from')
        chat = message['chat']
        chat_id = chat['id']

        # location - weather

        if location:
            lat = location.get('latitude')
            lon = location.get('longitude')
            weather = responseHandler.locationInput(lat, lon)
            responseHandler.sendWeather(chat_id, weather)
            return

        # command

        if text:
            if text.startswith('/'):
                if text.lower() == '/start':
                    responseHandler.sendTextMessage(chat_id, '幸会。')
                    setEnabled(chat_id, True)
                # 待办：最好把这里做成每晚定时
                elif text.lower() == '/weathertmr':
                    LAT = 57.63
                    LON = 18.31
                    weather = responseHandler.locationInput(LAT, LON)
                    responseHandler.forecastWeather(chat_id, weather)
                elif text.lower() == '/stop':
                    responseHandler.sendTextMessage(chat_id, '好，下次再说。')
                    setEnabled(chat_id, False)
                else:
                    responseHandler.sendTextMessage(chat_id, '什么？')
            else:
                responseHandler.replyMessage(chat_id, message_id, '嗯')

app = webapp2.WSGIApplication([
    ('/me', MeHandler),
    ('/updates', GetUpdatesHandler),
    ('/set_webhook', SetWebhookHandler),
    ('/webhook', WebhookHandler),
], debug=True)


from nuitka.nodes.CallNodes import makeExpressionCall
from nuitka.nodes.ConstantRefNodes import makeConstantRefNode
from nuitka.nodes.ContainerMakingNodes import (
    makeExpressionMakeTuple,
    makeExpressionMakeTupleOrConstant,
)
from nuitka.nodes.DictionaryNodes import makeExpressionMakeDictOrConstant
from nuitka.nodes.FunctionNodes import (
    ExpressionFunctionRef,
    makeExpressionFunctionCall,
    makeExpressionFunctionCreation,
)
from nuitka.nodes.KeyValuePairNodes import makeExpressionPairs
from nuitka.nodes.OutlineNodes import ExpressionOutlineBody
from nuitka.nodes.ReturnNodes import StatementReturn
from nuitka.nodes.VariableAssignNodes import makeStatementAssignmentVariable
from nuitka.nodes.VariableRefNodes import ExpressionTempVariableRef
from nuitka.PythonVersions import python_version

from .ComplexCallHelperFunctions import (
    getFunctionCallHelperDictionaryUnpacking,
    getFunctionCallHelperKeywordsStarDict,
    getFunctionCallHelperKeywordsStarList,
    getFunctionCallHelperKeywordsStarListStarDict,
    getFunctionCallHelperPosKeywordsStarDict,
    getFunctionCallHelperPosKeywordsStarList,
    getFunctionCallHelperPosKeywordsStarListStarDict,
    getFunctionCallHelperPosStarDict,
    getFunctionCallHelperPosStarList,
    getFunctionCallHelperPosStarListStarDict,
    getFunctionCallHelperStarDict,
    getFunctionCallHelperStarList,
    getFunctionCallHelperStarListStarDict,
)
from .ReformulationDictionaryCreation import buildDictionaryUnpackingArgs
from .ReformulationSequenceCreation import buildListUnpacking
from .TreeHelpers import (
    buildNode,
    buildNodeTuple,
    getKind,
    makeStatementsSequenceFromStatements,
)

def buildCallNode(provider, node, source_ref):
    called = buildNode(provider, node.func, source_ref)

    if python_version >= 0x350:
        list_star_arg = None
        dict_star_arg = None

    positional_args = []

    # For Python3.5 compatibility, the error handling with star argument last
    # is the old one, only with a starred argument before that, things use the
    # new unpacking code.
    for node_arg in node.args[:-1]:
        if getKind(node_arg) == "Starred":
            assert python_version >= 0x350
            list_star_arg = buildListUnpacking(provider, node.args, source_ref)
            positional_args = ()
            break
    else:
        if node.args and getKind(node.args[-1]) == "Starred":
            assert python_version >= 0x350

            list_star_arg = buildNode(provider, node.args[-1].value, source_ref)
            positional_args = buildNodeTuple(provider, node.args[:-1], source_ref)
        else:
            positional_args = buildNodeTuple(provider, node.args, source_ref)

    # Only the values of keyword pairs have a real source ref, and those only
    # really matter, so that makes sense.
    keys = []
    values = []

    for keyword in node.keywords[:-1]:
        if keyword.arg is None:
            assert python_version >= 0x350

            outline_body = ExpressionOutlineBody(
                provider=provider, name="dict_unpacking_call", source_ref=source_ref
            )

            tmp_called = outline_body.allocateTempVariable(
                temp_scope=None, name="called"
            )

            helper_args = [
                ExpressionTempVariableRef(variable=tmp_called, source_ref=source_ref),
                makeExpressionMakeTuple(
                    elements=buildDictionaryUnpackingArgs(
                        provider=provider,
                        keys=(keyword.arg for keyword in node.keywords),
                        values=(keyword.value for keyword in node.keywords),
                        source_ref=source_ref,
                    ),
                    source_ref=source_ref,
                ),
            ]

            dict_star_arg = makeExpressionFunctionCall(
                function=makeExpressionFunctionCreation(
                    function_ref=ExpressionFunctionRef(
                        function_body=getFunctionCallHelperDictionaryUnpacking(),
                        source_ref=source_ref,
                    ),
                    defaults=(),
                    kw_defaults=None,
                    annotations=None,
                    source_ref=source_ref,
                ),
                values=helper_args,
                source_ref=source_ref,
            )

            outline_body.setChildBody(
                makeStatementsSequenceFromStatements(
                    makeStatementAssignmentVariable(
                        variable=tmp_called, source=called, source_ref=source_ref
                    ),
                    StatementReturn(
                        expression=_makeCallNode(
                            called=ExpressionTempVariableRef(
                                variable=tmp_called, source_ref=source_ref
                            ),
                            positional_args=positional_args,
                            keys=keys,
                            values=values,
                            list_star_arg=list_star_arg,
                            dict_star_arg=dict_star_arg,
                            source_ref=source_ref,
                        ),
                        source_ref=source_ref,
                    ),
                )
            )

            return outline_body

    # For Python3.5 compatibility, the error handling with star argument last
    # is the old one, only with a starred argument before that, things use the
    # new unpacking code.

    if node.keywords and node.keywords[-1].arg is None:
        assert python_version >= 0x350

        dict_star_arg = buildNode(provider, node.keywords[-1].value, source_ref)
        keywords = node.keywords[:-1]
    else:
        keywords = node.keywords

    for keyword in keywords:
        keys.append(
            makeConstantRefNode(
                constant=keyword.arg, source_ref=source_ref, user_provided=True
            )
        )
        values.append(buildNode(provider, keyword.value, source_ref))

    if python_version < 0x350:
        list_star_arg = buildNode(provider, node.starargs, source_ref, True)
        dict_star_arg = buildNode(provider, node.kwargs, source_ref, True)

    return _makeCallNode(
        called=called,
        positional_args=positional_args,
        keys=keys,
        values=values,
        list_star_arg=list_star_arg,
        dict_star_arg=dict_star_arg,
        source_ref=source_ref,
    )

def _makeCallNode(
    called, positional_args, keys, values, list_star_arg, dict_star_arg, source_ref
):
    # Many variables, but only to cover the many complex call cases.

    if list_star_arg is None and dict_star_arg is None:
        result = makeExpressionCall(
            called=called,
            args=makeExpressionMakeTupleOrConstant(
                elements=positional_args,
                user_provided=True,
                source_ref=source_ref,
            ),
            kw=makeExpressionMakeDictOrConstant(
                makeExpressionPairs(keys=keys, values=values),
                user_provided=True,
                source_ref=source_ref,
            ),
            source_ref=source_ref,
        )

        # Bug compatible line numbers before Python 3.8
        if python_version < 0x380:
            if values:
                result.setCompatibleSourceReference(
                    source_ref=values[-1].getCompatibleSourceReference()
                )
            elif positional_args:
                result.setCompatibleSourceReference(
                    source_ref=positional_args[-1].getCompatibleSourceReference()
                )

        return result
    else:
        # Dispatch to complex helper function for each case. These do
        # re-formulation of complex calls according to Developer Manual.

        key = (
            bool(positional_args),
            bool(keys),
            list_star_arg is not None,
            dict_star_arg is not None,
        )

        table = {
            (True, True, True, False): getFunctionCallHelperPosKeywordsStarList,
            (True, False, True, False): getFunctionCallHelperPosStarList,
            (False, True, True, False): getFunctionCallHelperKeywordsStarList,
            (False, False, True, False): getFunctionCallHelperStarList,
            (True, True, False, True): getFunctionCallHelperPosKeywordsStarDict,
            (True, False, False, True): getFunctionCallHelperPosStarDict,
            (False, True, False, True): getFunctionCallHelperKeywordsStarDict,
            (False, False, False, True): getFunctionCallHelperStarDict,
            (True, True, True, True): getFunctionCallHelperPosKeywordsStarListStarDict,
            (True, False, True, True): getFunctionCallHelperPosStarListStarDict,
            (False, True, True, True): getFunctionCallHelperKeywordsStarListStarDict,
            (False, False, True, True): getFunctionCallHelperStarListStarDict,
        }

        get_helper = table[key]

        helper_args = [called]

        if positional_args:
            helper_args.append(
                makeExpressionMakeTupleOrConstant(
                    elements=positional_args,
                    user_provided=True,
                    source_ref=source_ref,
                )
            )

        # Order of evaluation changed in Python3.5.
        if python_version >= 0x350 and list_star_arg is not None:
            helper_args.append(list_star_arg)

        if keys:
            helper_args.append(
                makeExpressionMakeDictOrConstant(
                    pairs=makeExpressionPairs(keys=keys, values=values),
                    user_provided=True,
                    source_ref=source_ref,
                )
            )

        # Order of evaluation changed in Python3.5.
        if python_version < 0x350 and list_star_arg is not None:
            helper_args.append(list_star_arg)

        if dict_star_arg is not None:
            helper_args.append(dict_star_arg)

        result = makeExpressionFunctionCall(
            function=makeExpressionFunctionCreation(
                function_ref=ExpressionFunctionRef(
                    function_body=get_helper(), source_ref=source_ref
                ),
                defaults=(),
                kw_defaults=None,
                annotations=None,
                source_ref=source_ref,
            ),
            values=helper_args,
            source_ref=source_ref,
        )

        # Bug compatible line numbers before Python 3.8
        if python_version < 0x380:
            result.setCompatibleSourceReference(
                source_ref=helper_args[-1].getCompatibleSourceReference()
            )

        return result


'''S3 uploader module'''

import os
import time
import signal
import sys

import boto3
# This module seems to have some issues. pylint ignore them
from setproctitle import setproctitle, getproctitle  # pylint: disable=E0611

from kafkatos3.ThreadPool import ThreadPool

def upload_file(self, filename):
    '''horrible callback function outside the class because it needs to be pickable'''
    self.upload_file_to_s3(filename)

class S3Uploader(object):
    '''class for uploading files to s3'''

    def __init__(self, config, logger):
        '''constructor'''
        self.config = config
        self.logger = logger
        self.pool = None

    def upload_file_to_s3(self, filename):
        '''upload file to s3'''
        self.logger.info("Uploading file: " + filename + " to s3")

        working_dir = self.config.get("main", "working_directory")

        s3_key = "kafkatos3" + filename.replace(working_dir + "/tos3", "")

        self.logger.info("S3 key is " + s3_key)

        if self.config.get("s3", "s3_access_key") != "":
            access_key = self.config.get("s3", "s3_access_key")
            secret_key = self.config.get("s3", "s3_secret_key")
            s3client = boto3.client("s3", aws_access_key_id=access_key,
                                    aws_secret_access_key=secret_key)
        else:
            s3client = boto3.client("s3")

        bucket = self.config.get("s3", "s3_bucket_name")

        s3client.upload_file(filename, bucket, s3_key)

        os.remove(filename)

    def run(self):
        '''main executor'''
        def cb_exit_gracefully(signum, frame):
            '''callback to exit gracefully for a pool thread'''
            self.logger.info("Shutting down S3Uploader, signum %d"% (signum))
            sys.exit(0)

        self.logger.info("S3Uploader process starting up")
        self.pool = ThreadPool(int(self.config.get("s3", "s3uploader_workers")))

        setproctitle("[s3upload] " + getproctitle())

        signal.signal(signal.SIGINT, cb_exit_gracefully)
        signal.signal(signal.SIGTERM, cb_exit_gracefully)

        while True:
            tos3_dir = os.path.join(self.config.get(
                "main", "working_directory"), "tos3")
            files = self.get_files(tos3_dir, ".gz")

            self.pool.map(self.upload_file_to_s3, files)

            time.sleep(float(self.config.get(
                "s3", "s3upload_check_interval")))
        sys.exit(0)

    def get_files(self, directory, extension):
        ''' return a list of files in a directory recusively based on extension'''
        file_list = []
        for dirpath, _, files in os.walk(directory):
            for filename in files:
                fname = os.path.join(dirpath, filename)
                filename, file_extension = os.path.splitext(fname)
                if file_extension == extension:
                    file_list.append(fname)
        return file_list