import re
import json
import ftfy
from functools import lru_cache
from malaya.text.rules import rules_normalizer
from malaya.text.regex import _expressions
from malaya.text.english.words import words as _english_words
from malaya.tokenizer import Tokenizer
from malaya.path import PATH_PREPROCESSING, S3_PATH_PREPROCESSING
from malaya.function import check_file, validator
from typing import List

_annotate = [
    'hashtag',
    'allcaps',
    'elongated',
    'repeated',
    'emphasis',
    'censored',
]

_normalize = list(_expressions.keys())

def get_normalize():
    return _normalize

def get_annotate():
    return _annotate

def _case_of(text):
    return (
        str.upper
        if text.isupper()
        else str.lower
        if text.islower()
        else str.title
        if text.istitle()
        else str
    )

def unpack_english_contractions(text):
    """
    Replace *English* contractions in ``text`` str with their unshortened forms.
    N.B. The "'d" and "'s" forms are ambiguous (had/would, is/has/possessive),
    so are left as-is.
    Important Note: The function is taken from textacy (https://github.com/chartbeat-labs/textacy).
    """

    text = re.sub(
        r"(\b)([Aa]re|[Cc]ould|[Dd]id|[Dd]oes|[Dd]o|[Hh]ad|[Hh]as|[Hh]ave|[Ii]s|[Mm]ight|[Mm]ust|[Ss]hould|[Ww]ere|[Ww]ould)n't",
        r'\1\2 not',
        text,
    )
    text = re.sub(
        r"(\b)([Hh]e|[Ii]|[Ss]he|[Tt]hey|[Ww]e|[Ww]hat|[Ww]ho|[Yy]ou)'ll",
        r'\1\2 will',
        text,
    )
    text = re.sub(
        r"(\b)([Tt]hey|[Ww]e|[Ww]hat|[Ww]ho|[Yy]ou)'re", r'\1\2 are', text
    )
    text = re.sub(
        r"(\b)([Ii]|[Ss]hould|[Tt]hey|[Ww]e|[Ww]hat|[Ww]ho|[Ww]ould|[Yy]ou)'ve",
        r'\1\2 have',
        text,
    )
    text = re.sub(r"(\b)([Cc]a)n't", r'\1\2n not', text)
    text = re.sub(r"(\b)([Ii])'m", r'\1\2 am', text)
    text = re.sub(r"(\b)([Ll]et)'s", r'\1\2 us', text)
    text = re.sub(r"(\b)([Ww])on't", r'\1\2ill not', text)
    text = re.sub(r"(\b)([Ss])han't", r'\1\2hall not', text)
    text = re.sub(r"(\b)([Yy])(?:'all|a'll)", r'\1\2ou all', text)
    return text

def _get_expression_dict():
    return {
        k.lower(): re.compile(_expressions[k]) for k, v in _expressions.items()
    }

class Preprocessing:
    def __init__(
        self,
        normalize=[
            'url',
            'email',
            'percent',
            'money',
            'phone',
            'user',
            'time',
            'date',
            'number',
        ],
        annotate=[
            'allcaps',
            'elongated',
            'repeated',
            'emphasis',
            'censored',
            'hashtag',
        ],
        lowercase=True,
        fix_unidecode=True,
        expand_english_contractions=True,
        translator=None,
        speller=None,
        segmenter=None,
        stemmer=None,
    ):
        self._fix_unidecode = fix_unidecode
        self._normalize = normalize
        self._annotate = annotate
        self._regexes = _get_expression_dict()
        self._tokenizer = Tokenizer(lowercase=lowercase).tokenize
        self._expand_contractions = expand_english_contractions
        self._all_caps_tag = 'wrap'
        self._translator = translator
        self._speller = speller
        self._segmenter = segmenter
        if self._segmenter:
            self._expand_hashtags = True
        else:
            self._expand_hashtags = False
        self._stemmer = stemmer

    def _add_special_tag(self, m, tag, mode='single'):

        if isinstance(m, str):
            text = m
        else:
            text = m.group()

        if mode == 'single':
            return ' {} <{}> '.format(text, tag)
        elif mode == 'wrap':
            return ' '.join([' <{}> {} </{}> '.format(tag, text, tag)]) + ' '
        elif mode == 'every':
            tokens = text.split()
            processed = ' '.join([' {} <{}> '.format(t, tag) for t in tokens])
            return ' ' + processed + ' '

    @lru_cache(maxsize=65536)
    def _handle_hashtag_match(self, m):
        expanded = m.group()[1:]
        if self._expand_hashtags:
            expanded = self._segmenter.segment([expanded])[0]
            expanded = ' '.join(expanded.split('-'))
            expanded = ' '.join(expanded.split('_'))

        if 'hashtag' in self._annotate:
            expanded = self._add_special_tag(expanded, 'hashtag', mode='wrap')

        return expanded

    @lru_cache(maxsize=65536)
    def _handle_repeated_puncts(self, m):
        text = m.group()
        text = ''.join(sorted(set(text), reverse=True))

        if 'repeated' in self._annotate:
            text = self._add_special_tag(text, 'repeated')

        return text

    @lru_cache(maxsize=65536)
    def _handle_generic_match(self, m, tag, mode='every'):
        text = m.group()
        text = self._add_special_tag(text, tag, mode=mode)

        return text

    def _handle_elongated_match(self, m):
        text = m.group()
        text = self._regexes['normalize_elong'].sub(r'\1\1', text)
        if self._speller and text.lower() not in _english_words:
            if hasattr(self._speller, 'normalize_elongated'):
                text = _case_of(text)(
                    self._speller.normalize_elongated(text.lower())
                )
            else:
                text = _case_of(text)(self._speller.correct(text.lower()))
        if 'elongated' in self._annotate:
            text = self._add_special_tag(text, 'elongated')
        return text

    @lru_cache(maxsize=65536)
    def _handle_emphasis_match(self, m):
        text = m.group().replace('*', '')
        if 'emphasis' in self._annotate:
            text = self._add_special_tag(text, 'emphasis')

        return text

    def _dict_replace(self, wordlist, _dict):
        return [_dict.get(w, w) for w in wordlist]

    @staticmethod
    def text(wordlist):
        in_hashtag = False
        _words = []
        for word in wordlist:

            if word == '<hashtag>':
                in_hashtag = True
            elif word == '</hashtag>':
                in_hashtag = False
            elif word in {'<allcaps>', '</allcaps>'} and in_hashtag:
                continue

            _words.append(word)

        return _words

    def process(self, text):
        text = re.sub(r' +', ' ', text)
        if self._fix_unidecode:
            text = ftfy.fix_text(text)

        for item in self._normalize:
            text = self._regexes[item].sub(
                lambda m: ' ' + '<' + item + '>' + ' ', text
            )

        text = self._regexes['hashtag'].sub(
            lambda w: self._handle_hashtag_match(w), text
        )

        if 'allcaps' in self._annotate:
            text = self._regexes['allcaps'].sub(
                lambda w: self._handle_generic_match(
                    w, 'allcaps', mode=self._all_caps_tag
                ),
                text,
            )
        if 'elongated' in self._annotate:
            text = self._regexes['elongated'].sub(
                lambda w: self._handle_elongated_match(w), text
            )
        if 'repeated' in self._annotate:
            text = self._regexes['repeat_puncts'].sub(
                lambda w: self._handle_repeated_puncts(w), text
            )
        if 'emphasis' in self._annotate:
            text = self._regexes['emphasis'].sub(
                lambda w: self._handle_emphasis_match(w), text
            )
        if 'censored' in self._annotate:
            text = self._regexes['censored'].sub(
                lambda w: self._handle_generic_match(w, 'censored'), text
            )
        if self._expand_contractions:
            text = unpack_english_contractions(text)

        text = re.sub(r' +', ' ', text)
        text = self.text(text.split())
        text = ' '.join(text)
        text = self._tokenizer(text)
        text = self._dict_replace(text, rules_normalizer)
        if self._translator:
            text = self._dict_replace(text, self._translator)
        if self._stemmer:
            rejected = ['<', '</', '>', '>']
            text = [
                self._stemmer.stem(w)
                if (
                    w not in _english_words
                    and all([r not in w for r in rejected])
                )
                else w
                for w in text
            ]

        return text

def preprocessing(
    normalize: List[str] = [
        'url',
        'email',
        'percent',
        'money',
        'phone',
        'user',
        'time',
        'date',
        'number',
    ],
    annotate: List[str] = [
        'allcaps',
        'elongated',
        'repeated',
        'emphasis',
        'censored',
        'hashtag',
    ],
    lowercase: bool = True,
    fix_unidecode: bool = True,
    expand_english_contractions: bool = True,
    translate_english_to_bm: bool = True,
    speller=None,
    segmenter=None,
    stemmer=None,
    **kwargs,
):
    """
    Load Preprocessing class.

    Parameters
    ----------
    normalize: list
        normalizing tokens, can check all supported normalizing at `malaya.preprocessing.get_normalize()`.
    annotate: list
        annonate tokens <open></open>,
        only accept ['hashtag', 'allcaps', 'elongated', 'repeated', 'emphasis', 'censored'].
    lowercase: bool
    fix_unidecode: bool
    expand_english_contractions: bool
        expand english contractions
    translate_english_to_bm: bool
        translate english words to bahasa malaysia words
    speller: object
        spelling correction object, need to have a method `correct`
    segmenter: object
        segmentation object, need to have a method `segment`.
        If provide, it will expand hashtags, #mondayblues == monday blues
    stemmer: object
        stemmer object, need to have a method `stem`.
        If provide, it will stem or lemmatize the string.

    Returns
    -------
    result : malaya.preprocessing.Preprocessing class
    """

    if any([e not in _normalize for e in normalize]):
        raise ValueError(
            'normalize element not able to recognize, supported normalization can check at get_normalize()'
        )
    if any([e not in _annotate for e in annotate]):
        raise ValueError(
            "annotate only accept ['hashtag', 'allcaps', 'elongated', 'repeated', 'emphasis', 'censored']"
        )
    validator.validate_object_methods(
        speller, ['correct', 'normalize_elongated'], 'speller'
    )
    validator.validate_object(segmenter, 'segment', 'segmenter')
    validator.validate_object(stemmer, 'stem', 'stemmer')

    if translate_english_to_bm:
        path = check_file(
            PATH_PREPROCESSING['english-malay'],
            S3_PATH_PREPROCESSING['english-malay'],
            **kwargs,
        )
        try:
            with open(path['model']) as fopen:
                translator = json.load(fopen)
        except BaseException:
            raise Exception(
                "failed to load english-malay vocab, please run `malaya.utils.delete_cache('preprocessing/english-malay')`")
    else:
        translator = None

    return Preprocessing(
        normalize=normalize,
        annotate=annotate,
        lowercase=lowercase,
        fix_unidecode=fix_unidecode,
        expand_english_contractions=expand_english_contractions,
        translator=translator,
        speller=speller,
        segmenter=segmenter,
        stemmer=stemmer,
    )


import datetime
import sys

import unittest
from unittest import TestCase

from botocore.compat import six
from mock import call, patch

from ebcli.lib import utils
from ebcli.objects.exceptions import CommandError, InvalidOptionsError
StringIO = six.moves.StringIO

HELLO_WORLD_MSG = 'Hello, world!'
HAPPY_ARGS = ['echo', HELLO_WORLD_MSG]
OS_ERROR_ARGS = ['this_is_not_a_command']
COMMAND_ERROR_ARGS = ['date', 'illegal_argument']
MOCK_FILES = ['/a', '/b', '/c', '/d']

class TestUtils(TestCase):
    @unittest.skipIf(sys.platform.startswith('win'), 'Test is not equipped to run on Windows')
    @patch('ebcli.lib.utils.LOG')
    @patch('ebcli.lib.utils.sys.stdout', new_callable=StringIO)
    def test_exec_cmd_live_output_happy_case(self, out, LOG):
        expected_output = HELLO_WORLD_MSG + "\n"

        self.assertEqual(utils.exec_cmd(HAPPY_ARGS), expected_output)
        LOG.debug.assert_called_once_with(' '.join(HAPPY_ARGS))
        self.assertEqual(out.getvalue(), expected_output)

    @patch('ebcli.lib.utils.LOG')
    def test_exec_cmd_live_output_oserror_case(self, LOG):
        self.assertRaises(OSError, utils.exec_cmd, OS_ERROR_ARGS)
        LOG.debug.assert_called_once_with(' '.join(OS_ERROR_ARGS))

    @unittest.skipIf(sys.platform.startswith('win'), 'Test is not equipped to run on Windows')
    @patch('ebcli.lib.utils.LOG')
    def test_exec_cmd_live_output_commanderror_case(self, LOG):
        self.assertRaises(CommandError, utils.exec_cmd, COMMAND_ERROR_ARGS)
        LOG.debug.assert_called_once_with(' '.join(COMMAND_ERROR_ARGS))

    def test_flatten_empty_list(self):
        self.assertEqual([], utils.flatten([]))

    def test_flatten_nested_single_list(self):
        self.assertEqual([0], utils.flatten([[0]]))

    def test_flatten_nested_long_list(self):
        self.assertEqual([0, 1, 2], utils.flatten([[0], [1], [2]]))

    def test_flatten__string_list(self):
        self.assertEqual(
            [
                '2018-09-15 19:41:43',
                'CREATE_IN_PROGRESS       ',
                'my-cfn-stack (AWS::CloudFormation::Stack)\nUser Initiated         '
            ],
            utils.flatten(
                [
                    ['2018-09-15 19:41:43'],
                    ['CREATE_IN_PROGRESS       '],
                    ['my-cfn-stack (AWS::CloudFormation::Stack)\nUser Initiated         ']
                ]
            )
        )

    @patch('os.path.getmtime')
    def test_last_modified_file(self, getmtime):
        getmtime.side_effect = [9.01, 200.0, 300.0, 5000.0]
        expected_last_mod_file = MOCK_FILES[-1]
        self.assertEqual(expected_last_mod_file,
                          utils.last_modified_file(MOCK_FILES))

    def test_anykey(self):
        key0, key1 = 'a', 'b'
        self.assertEqual(utils.anykey({key0: None}), key0)
        self.assertIn(utils.anykey({key0: None, key1: None}), {key0, key1})

    def test_merge_dicts_both_empty(self):
        self.assertDictEqual({}, utils.merge_dicts({}, {}))

    def test_merge_dicts_one_empty(self):
        mock_dict = {'abc': 123, 'bcd': 555}
        self.assertDictEqual(mock_dict, utils.merge_dicts(mock_dict, {}))
        self.assertDictEqual(mock_dict, utils.merge_dicts({}, mock_dict))

    def test_merge_dicts_not_overlapping(self):
        low_priority = {1: 1, 2: 2, 3: 3}
        high_priority = {'a': 'high_a', 'b': 'high_b', 'e': 'high_e'}
        expected = {1: 1, 2: 2, 3: 3, 'a': 'high_a', 'b': 'high_b', 'e': 'high_e'}

        self.assertDictEqual(expected, utils.merge_dicts(low_priority, high_priority))

    def test_merge_dicts_overlapping(self):
        low_priority = {'a': 'low_a', 'b': 'low_b', 'd': 'low_d'}
        high_priority = {'a': 'high_a', 'b': 'high_b', 'e': 'high_e'}
        expected = {'a': 'high_a', 'b': 'high_b', 'd': 'low_d', 'e': 'high_e'}

        self.assertDictEqual(expected, utils.merge_dicts(low_priority, high_priority))

    def test_parse_source__source_is_blank(self):
        self.assertEqual(
            (None, None, None),
            utils.parse_source('')
        )

    def test_parse_source__source_is_not_codecommit(self):
        with self.assertRaises(InvalidOptionsError) as context_manager:
            utils.parse_source('github')
        self.assertEqual(
            'Source location "github" is not supported by the EBCLI',
            str(context_manager.exception)
        )

    def test_parse_source__source_location_is_specified__repository_and_branch_are_not(self):
        self.assertEqual(
            ('codecommit', None, None),
            utils.parse_source('codecommit')
        )

    def test_parse_source__source_location_repository_and_branch_are_specified(self):
        self.assertEqual(
            ('codecommit', 'repository', 'branch'),
            utils.parse_source('codecommit/repository/branch')
        )

    def test_parse_source__branch_name_has_forward_slash(self):
        self.assertEqual(
            ('codecommit', 'repository', 'my/branch/name'),
            utils.parse_source('codecommit/repository/my/branch/name')
        )

    def test_pad_list(self):
        self.assertEqual(['text'], utils.padded_list(['text'], [1]))
        self.assertEqual(['text', '', '', ''], utils.padded_list(['text'], [1, 2, 3, 4]))
        self.assertEqual(
            ['text', 'hello, world', '', ''],
            utils.padded_list(['text', 'hello, world'], [1, 2, 3, 4])
        )
        self.assertEqual(
            ['text', 'hello, world'],
            utils.padded_list(['text', 'hello, world'], [1])
        )
        with self.assertRaises(AttributeError) as context_manager:
            utils.padded_list(['text'], [])

        self.assertEqual(
            'The reference_list argument must be non-empty.',
            str(context_manager.exception)
        )

    def test_pad_line(self):
        self.assertEqual('hello, world!\n', utils.padded_line('hello, world!'))
        self.assertEqual('  hello, world!\n', utils.padded_line('hello, world!', padding=2))
        self.assertEqual('hello, world!\n', utils.padded_line('hello, world!', padding=-1))
        self.assertEqual('hello, world!\n', utils.padded_line('hello, world!', padding='a'))

    def test_left_padded_string(self):
        self.assertEqual('  hello, world', utils.left_padded_string('hello, world', padding=2))

    def test_right_padded_string(self):
        self.assertEqual('hello, world  ', utils.right_padded_string('hello, world', padding=2))

    def test_longest_string(self):
        self.assertEqual(
            'abcde',
            utils.longest_string(['a', 'abc', 'abcde'])
        )

    def test_row_wrapper(self):
        string_width_mappings = [
            {
                'string': '2018-08-12 18:36:42',
                'width': 19
            },
            {
                'string': 'CREATE_COMPLETE',
                'width': 35
            },
            {
                'string': 'ServerlessRestApiDeployment47fc2d5f9d (AWS::ApiGateway::Deployment)\n'
                          'The API gateway, ServerlessRestApiDeployment47fc2d5f9d, was successfully built',
                'width': 67
            },
        ]
        self.assertEqual(
            [
                '2018-08-12 18:36:42   CREATE_COMPLETE                       ServerlessRestApiDeployment47fc2d5f9d (AWS::ApiGateway::Deployment)',
                '                                                            The API gateway, ServerlessRestApiDeployment47fc2d5f9d, was        ',
                '                                                            successfully built                                                 '
            ],
            utils.row_wrapper(string_width_mappings)
        )

    def test_row_wrapper__lengthy_resource_status(self):
        string_width_mappings = [
            {
                'string': '2018-08-12 18:36:42',
                'width': 19
            },
            {
                'string': 'UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS',
                'width': 35
            },
            {
                'string': 'ServerlessRestApiDeployment47fc2d5f9d (AWS::ApiGateway::Deployment)\n'
                          'The API gateway, ServerlessRestApiDeployment47fc2d5f9d, was successfully built',
                'width': 67
            },
        ]
        self.assertEqual(
            [
                '2018-08-12 18:36:42   UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN   ServerlessRestApiDeployment47fc2d5f9d (AWS::ApiGateway::Deployment)',
                '                      _PROGRESS                             The API gateway, ServerlessRestApiDeployment47fc2d5f9d, was        ',
                '                                                            successfully built                                                 '
            ],
            utils.row_wrapper(string_width_mappings)
        )

    def test_random_string(self):
        utils.random_string()
        utils.random_string(length=10)

    def test_sleep(self):
        utils.sleep(sleep_time=0)

    def test_datetime_utcnow(self):
        now = datetime.datetime.utcnow()
        a_little_later = utils.datetime_utcnow()
        very_small_difference = datetime.timedelta(seconds=0.001)
        self.assertTrue(
            a_little_later - now < very_small_difference
        )


import pygame

WHITE = 255, 255, 255

class Paddle(object):
    def __init__(self, pos=[0,0], dimensions=(20,100), speed=5,
            up=pygame.K_UP, dwn=pygame.K_DOWN, direc=0):
        self.surface = pygame.Surface(dimensions)
        self.surface.fill(WHITE)
        self.rect = self.surface.get_rect().move(pos)
        self.up = up
        self.dwn = dwn
        self.speed = speed
        self.direc = direc

    def move(self, frect):
        if self.direc == 1 and self.rect.bottom + self.speed > frect.bottom:
            self.direc = 0
        elif self.direc == -1 and self.rect.top + self.speed < frect.top:
            self.direc = 0
        self.rect = self.rect.move([0, self.speed * self.direc])


# -*- coding: utf-8 -*-
import os
from http.server import HTTPServer, SimpleHTTPRequestHandler
from pathlib import Path

import click

from behappy.core.conf import settings
from behappy.core.main import BeHappy, BeHappyFile, BeHappySync
from behappy.core.utils import timeit

@click.group()
def main():
    pass

@main.command()
@click.option('--target', default='target', help='Path to build folder')
@click.option('--conf', default='behappy.ini', help='Path to config')
@click.option('--tags', default='', help='Filter albums by tags')
@timeit
def build(target, conf, tags):
    """
    Build static site
    """
    settings.load(conf)

    tags = set([i.strip() for i in tags.split(',') if i.strip()])
    blog = BeHappy(target, tags)
    blog.build()

@main.command()
@click.argument('videos', nargs=-1)
@click.option('--preview', default='0', help='Preview time in seconds')
@timeit
def convert_video(videos, preview):
    """
    Convert videos to mp4 and generate preview
    """
    for video in videos:
        print('# Starting {}'.format(video))
        src = Path(video)
        dst_video = src.with_suffix('.mp4')
        dst_preview = src.with_suffix('.jpg')

        convert_template = 'ffmpeg -i {0} -vcodec libx264 -preset slow -crf 28 -movflags faststart {1}'
        preview_template = 'ffmpeg -i {0} -vframes 1 -an -ss {1} {2}'
        exif_template = 'exiftool -overwrite_original -TagsFromFile {0} "-EXIF:all>EXIF:all" {1}'

        if not dst_video.exists():
            os.system(convert_template.format(src.as_posix(), dst_video.as_posix()))
        if dst_preview.exists():
            dst_preview.unlink()
        os.system(preview_template.format(src.as_posix(), int(preview), dst_preview))
        os.system(exif_template.format(src.as_posix(), dst_preview.as_posix()))
        print('# End {}\n\n'.format(video))

@main.command()
@click.option('--target', default='target', help='Path to build folder')
@click.option('--port', default='8000', help='Path to build folder')
def server(target, port):
    """
    Run test web server
    """
    if not os.path.exists(target):
        os.mkdir(target)
    os.chdir(target)

    httpd = HTTPServer(('0.0.0.0', int(port)), SimpleHTTPRequestHandler)
    print('# server at http://127.0.0.1:{0}'.format(port))
    httpd.serve_forever()

@main.command()
def new():
    """
    Create new album.
    """
    folder = Path('.').absolute()
    file = BeHappyFile(folder)
    file.new()

@main.command()
@click.option('--target', default='target', help='Path to build folder')
@click.option('--profile', default=None, help='AWS profile')
@click.option('--endpoint', default=None, help='S3 endpoint url')
@click.option('--bucket', help='S3 bucket name')
@click.option('--cloudfront', default=None, help='AWS cloudfront distribution id')
@timeit
def sync(target, profile, endpoint, bucket, cloudfront):
    """
    Run test web server
    """
    folder = Path(target)
    be_sync = BeHappySync(folder, profile, endpoint, bucket)
    print('Sync S3')
    be_sync.s3()
    if cloudfront:
        print('Invalidate CloudFront')
        be_sync.cloudfront_invalidate(cloudfront)

if __name__ == '__main__':
    main()


import logging
from collections import OrderedDict
from dordis.primitives.differential_privacy import (
    ddgauss, dskellam
)
from dordis.config import Config

registered_dict = OrderedDict([
    ('ddgauss', ddgauss.Handler),
    ('dskellam', dskellam.Handler)
])

def get():
    handler_type = Config().agg\
        .differential_privacy.mechanism

    if handler_type in registered_dict:
        logging.info("Differential privacy: %s", handler_type)
        registered_handler = registered_dict[handler_type]()
    else:
        raise ValueError(
            f"No such DP mechanism: {handler_type}")

    return registered_handler


import sys
import os
BIN = '../../'
sys.path.append(BIN)
from my_nn_modules import AE_basic, AE_bn_LeakyReLU
from utils import plot_activations
from my_nn_modules import get_data, RMSELoss
from fastai import train as tr
from fastai.callbacks import ActivationStats
from fastai import basic_train, basic_data
from fastai.callbacks.tracker import SaveModelCallback
from torch.utils.data import TensorDataset
import torch.utils.data
import torch.nn as nn
import torch
import utils
from scipy import stats
import my_matplotlib_style as ms
import datetime
import pickle
import matplotlib as mpl
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np

plt.close('all')
# import torch.optim as optim

mpl.rc_file(BIN + 'my_matplotlib_rcparams')

# Load AOD data
# Smaller dataset fits in memory on Kebnekaise
train = pd.read_pickle(BIN + 'processed_data/aod/custom_normalized_train_10percent.pkl')
test = pd.read_pickle(BIN + 'processed_data/aod/custom_normalized_test_10percent.pkl')
train = train[train['m'] > 1e-3]
test = test[test['m'] > 1e-3]

bs = 1024
# Create TensorDatasets
train_ds = TensorDataset(torch.tensor(train.values, dtype=torch.float),
                         torch.tensor(train.values, dtype=torch.float))
valid_ds = TensorDataset(torch.tensor(test.values, dtype=torch.float),
                         torch.tensor(test.values, dtype=torch.float))
# Create DataLoaders
train_dl, valid_dl = get_data(train_ds, valid_ds, bs=bs)
# Return DataBunch
db = basic_data.DataBunch(train_dl, valid_dl)

module_name = 'AE_bn_LeakyReLU'
module = AE_bn_LeakyReLU
latent_dim = 14
grid_search_folder = module_name + '_AOD_grid_search_custom_normalization_1500epochs/'
grid_search_folder = module_name + '_AOD_grid_search_custom_normalization_1500epochs_12D10D8D/'
model_folder = 'AE_27_200_200_200_%d_200_200_200_27' % latent_dim
train_folder = 'AE_bn_LeakyReLU_bs4096_lr1e-02_wd1e-02_ppNA'    # 14D 18D 20D
# train_folder = 'AE_bn_LeakyReLU_bs4096_lr3e-02_wd1e-04_ppNA'  # 16D
# train_folder = 'AE_bn_LeakyReLU_bs4096_lr1e-03_wd1e-01_ppNA'  # 12D
# train_folder = 'AE_bn_LeakyReLU_bs4096_lr1e-03_wd1e-02_ppNA'  # 10D
# train_folder = 'AE_bn_LeakyReLU_bs4096_lr1e-02_wd1e-04_ppNA'  # 8D
save = True

loss_func = nn.MSELoss()

plt.close('all')
tmp = train_folder.split('bs')[1]
param_string = 'bs' + tmp
save_dict_fname = 'save_dict' + param_string + '.pkl'
path_to_save_dict = grid_search_folder + model_folder + '/' + train_folder + '/' + save_dict_fname
saved_model_fname = 'best_' + module_name + '_' + param_string.split('_pp')[0]
path_to_saved_model = grid_search_folder + model_folder + '/' + 'models/' + saved_model_fname
curr_save_folder = grid_search_folder + model_folder + '/' + train_folder + '/'

with open(path_to_save_dict, 'rb') as f:
    curr_save_dict = pickle.load(f)

train_losses = curr_save_dict[module_name][param_string]['train_losses']
val_losses = curr_save_dict[module_name][param_string]['val_losses']

# # Plot losses
# batches = len(train_losses)
# epochs = len(val_losses)
# val_iter = (batches / epochs) * np.arange(1, epochs + 1, 1)
# # loss_name = str(loss_func).split("(")[0]
# plt.figure()
# plt.plot(train_losses, label='Train')
# plt.plot(val_iter, val_losses, label='Validation', color='orange')
# plt.yscale(value='log')
# plt.legend()
# plt.ylabel('MSE')
# plt.xlabel('Batches processed')
# if save:
#     fig_name = 'new_losses.png'
#     plt.savefig(curr_save_folder + fig_name)

nodes = model_folder.split('AE_')[1].split('_')
nodes = [int(x) for x in nodes]
model = module(nodes)
learn = basic_train.Learner(data=db, model=model, loss_func=loss_func, true_wd=True, bn_wd=False,)
learn.model_dir = grid_search_folder + model_folder + '/' + 'models/'
learn.load(saved_model_fname)
learn.model.eval()

plt.close('all')
unit_list = ['[GeV]', '[rad]', '[rad]', '[GeV]']
variable_list = [r'$p_T$', r'$\eta$', r'$\phi$', r'$E$']
line_style = ['--', '-']
colors = ['orange', 'c']
markers = ['*', 's']

idxs = (0, -1)  # Pick events to compare
data = torch.tensor(test[idxs[0]:idxs[1]].values, dtype=torch.float)
pred = model(data).detach().numpy()
data = data.detach().numpy()

data_df = pd.DataFrame(data, columns=test.columns)
pred_df = pd.DataFrame(pred, columns=test.columns)

# Unnormalize
unnormalized_data_df = utils.custom_unnormalize(data_df)
# unnormalized_data_df = data_df
unnormalized_pred_df = utils.custom_unnormalize(pred_df)

# Handle variables with discrete distributions
unnormalized_pred_df['N90Constituents'] = unnormalized_pred_df['N90Constituents'].round()
uniques = unnormalized_data_df['ActiveArea'].unique()
utils.round_to_input(unnormalized_pred_df, uniques, 'ActiveArea')

hist_groups = [
    ['pt', 'eta', 'phi', 'm', 'LeadingClusterPt', 'LeadingClusterCenterLambda', 'LeadingClusterSecondLambda', 'LeadingClusterSecondR'],
    ['ActiveArea', 'ActiveArea4vec_eta', 'ActiveArea4vec_phi', 'ActiveArea4vec_pt', 'ActiveArea4vec_m', 'Timing', 'OotFracClusters5', 'OotFracClusters10'],
    ['AverageLArQF', 'NegativeE', 'CentroidR', 'DetectorEta', 'EMFrac', 'HECFrac', 'HECQuality', 'LArQuality'],
    ['Width', 'WidthPhi', 'N90Constituents'],
]

mpl.rcParams['xtick.labelsize'] = 16
mpl.rcParams['ytick.labelsize'] = 16
mpl.rcParams['axes.labelsize'] = 18

group = hist_groups[0]

for i_group, group in enumerate(hist_groups):
    group_data_df = unnormalized_data_df[group]
    group_pred_df = unnormalized_pred_df[group]
    data = group_data_df.values
    pred = group_pred_df.values

    nrows = 3
    ncols = 3
    if i_group == 3:
        nrows = 1
    fig, axs = plt.subplots(nrows=nrows, ncols=ncols)

    # Histograms
    alph = 0.8
    n_bins = 200
    for ii in np.arange(nrows):
        for kk in np.arange(ncols):
            pp = nrows * ii + kk
            if i_group == 3:
                fig.set_size_inches(12, 3, forward=True)
                ax = axs[kk]
            else:
                ax = axs[ii, kk]
            if i_group != 3:
                if pp == ncols * nrows - 1:
                    fig.delaxes(ax)
                    break
            if pp == 0:
                n_hist_data, bin_edges, _ = ax.hist(
                    data[:, pp], color=colors[1], label='Input', alpha=1, bins=n_bins, density=False)
                n_hist_pred, _, _ = ax.hist(pred[:, pp], color=colors[0],
                                            label='Output', alpha=alph, bins=n_bins, density=False)
            else:
                n_hist_data, bin_edges, _ = ax.hist(
                    data[:, pp], color=colors[1], alpha=1, bins=n_bins, density=False)
                n_hist_pred, _, _ = ax.hist(pred[:, pp], color=colors[0],
                                            alpha=alph, bins=n_bins, density=False)
            ax.set_xlabel(group[pp])
            # plt.xlabel(test.columns[kk])
            # plt.ylabel('Number of jets')
            # ax.ticklabel_format(style='sci', scilimits=(0, 0), axis='y')
            ax.set_yscale('log')
    plt.tight_layout()
    if i_group != 3:
        fig.legend(bbox_to_anchor=(.76, 0.08, 0.2, 0.2), fontsize=30)

    fig_name = 'hist_%d_group%d' % (latent_dim, i_group)
    if save:
        plt.savefig(curr_save_folder + fig_name)

if not save:
    plt.show()


#!/usr/bin/env python from __future__ import division, print_function
import sys
import random, math, copy
import pdb
import csv

sys.path.insert(0, '..')
sys.path.append('.')

from tbpsite import settings
from django.core.management import setup_environ

setup_environ(settings)

from tutoring.models import Tutoring
from constants import TUTORING_DAY_CHOICES, TUTORING_HOUR_CHOICES
csvFileName = sys.argv[1]

MAX_TUTORS_PER_HOUR = 5
MIN_TUTORS_PER_HOUR = 2  # Want this
ENFORCED_MIN_TUTORS_PER_HOUR = 1  # Enforce this

TUTORING_START = 10  # 10AM
TUTORING_END = 16  # 1 hr before 5pm

tutoringHours = {}
for d in TUTORING_DAY_CHOICES:
    for h in TUTORING_HOUR_CHOICES:
        tutoringHours[( int(d[0]), int(h[0]) + TUTORING_START )] = set()

for t in Tutoring.current.all():
    if not t.hidden:
        tutoringHours[(int(t.day_1), int(t.hour_1) + 10)] |= set(map(lambda x: str(x), list(t.profile.classes.all())))
        tutoringHours[(int(t.day_2), int(t.hour_2) + 10)] |= set(map(lambda x: str(x), list(t.profile.classes.all())))

for key in tutoringHours:
    tutoringHours[key] = list(tutoringHours[key])

data = tutoringHours

days = {0: "Monday", 1: "Tuesday", 2: "Wednesday", 3: "Thursday", 4: "Friday"}

subjects = set()

for ls in data.values():
  for item in ls:
    subject = item.split(' ')[0]
    if subject not in subjects:
      subjects.add(subject)

subjects = sorted(list(subjects))

def format_classes(classes_lst):
    formatted_classes_lst = [','.join([class_str.split(' ')[1] for class_str in classes_lst if subject in class_str]) for subject in subjects]
    return formatted_classes_lst

with open(csvFileName, 'wb') as csvFile:
    writer = csv.writer(csvFile)
    writer.writerow(['Hour', 'Class'] + days.values())
    for hour in range(TUTORING_START, TUTORING_END + 1):
        classes = []
        for day in days:
            classes.append(format_classes(data[(day, hour)]))
        rows = zip(*classes)
        for index, row in enumerate(rows):
            writer.writerow([str(hour) if not index else '', subjects[index]] + list(row))


import yfinance as yf
import streamlit as st
import pandas as pd

st.write(
    '''
    # Simple stock price app
    Shown are the stock closing price and volume of Google!
    '''
)

trickerSymbol = 'GOOGL'

trickerData = yf.Ticker(trickerSymbol)

trickerDf = trickerData.history(period = 'id', start = '2011-5-31', end = '2021-5-31')

st.line_chart(trickerDf.Close)

st.line_chart(trickerDf.Volume)


import json
from bson import json_util

from flask import render_template, request, Response

import query
from app import app
from utils import datetime_range

@app.route('/', methods=['GET'])
def index():
    return render_template('index.html')

@app.route('/api/player', methods=['GET'])
def player():
    """
    The start of an API route, returns a player's data using ?name='x'.
    """
    name = request.args.get('name')
    player_dict = query.query_specific_player(name)
    player_dict.pop("_id", None)

    return Response(json.dumps(player_dict, default=json_util.default),
                    mimetype='application/json')

@app.route('/api/gamelogs', methods=['GET'])
def gamelogs():
    gamelogs_query = {}

    name = request.args.get('name')
    if name:
        gamelogs_query.update({'Player': name})
    start = request.args.get('start')
    end = request.args.get('end')
    if start:
        gamelogs_query.update(datetime_range(start, end))
    active = request.args.get('active')

    if not active:
        gamelogs_list = query.query_games(gamelogs_query)
    else:
        gamelogs_list = query.query_games(gamelogs_query, active=True)

    if gamelogs_list:
        for gamelog_dict in gamelogs_list:
            gamelog_dict.pop("_id", None)

    return Response(json.dumps(gamelogs_list, default=json_util.default),
                    mimetype='application/json')


import os
import csv
import pandas as pd
import numpy as np
from scipy import stats
from pathlib import Path
from cliffs_delta import cliffs_delta

def read_data(dataset, dataset_path, metric):
    resample_balancers = ['SMOTE', 'ROS', 'ADASYN', 'BSMOTE', 'RUS', 'NM', 'CNN', 'TL', 'ENN']

    datas = {}
    for balancer in os.listdir(dataset_path):
        if '_result.csv' in balancer:
            continue

        data_path = ''
        if balancer in resample_balancers:
            if dataset == 'DataClass':
                data_path = dataset_path + '%s/NB/metrics_result.csv' % balancer
            if dataset == 'FeatureEnvy':
                data_path = dataset_path + '%s/LogisticRegression/metrics_result.csv' % balancer
            if dataset == 'GodClass':
                data_path = dataset_path + '%s/NB/metrics_result.csv' % balancer
            if dataset == 'LongMethod':
                data_path = dataset_path + '%s/LogisticRegression/metrics_result.csv' % balancer
        else:
            continue

        raw_datas = pd.read_csv(data_path)
        raw_datas = raw_datas[metric].values

        datas[balancer] = raw_datas

    return datas

def process_data(datas):
    metric_datas = []
    balancers = []

    baseline_data = datas['SMOTE']
    metric_datas.append(baseline_data)

    for key, value in datas.items():
        if key == 'SMOTE':
            continue
        metric_datas.append(value)
        balancers.append(key)

    return metric_datas, balancers

def wilcoxon(l1, l2):
    w, p_value = stats.wilcoxon(l1, l2, correction=False)
    return p_value

def Wilcoxon_signed_rank_test(metric_datas, balancers, dataset, metric):
    pvalues = []
    sortpvalues = []
    bhpvalues = []
    cliffs = []
    medians = []
    labels = []

    for i in range(1, len(metric_datas)):
        print(balancers[i-1])
        pvalue = wilcoxon(metric_datas[0], metric_datas[i])
        pvalues.append(pvalue)
        sortpvalues.append(pvalue)

        cliff, _ = cliffs_delta(metric_datas[i], metric_datas[0])
        cliffs.append(abs(cliff))

        median = np.median(metric_datas[i]) - np.median(metric_datas[0])
        medians.append(median)

    sortpvalues.sort()

    for i in range(len(pvalues)):
        bhpvalue = pvalues[i]*(len(pvalues))/(sortpvalues.index(pvalues[i])+1)
        bhpvalues.append(bhpvalue)

    for i in range(len(balancers)):
        if bhpvalues[i] < 0.05 and cliffs[i] > 0.147:
            if medians[i] > 0:
                labels.append('significant improvement')
            if medians[i] < 0:
                labels.append('significant decrease')
            if medians[i] == 0:
                labels.append('non-significant')
        else:
            labels.append('non-significant')

    output_path = 'results/resampling_significant/'
    Path(output_path).mkdir(parents=True, exist_ok=True)
    file_path = output_path + '{}_{}.csv'.format(dataset, metric)

    csv_file = open(file_path, 'w', newline='', encoding='utf-8')
    csv_write = csv.writer(csv_file)
    csv_write.writerow(['balancer', 'pvalue', 'cliff', 'significant'])

    for i in range(len(balancers)):
        csv_write.writerow([balancers[i], bhpvalues[i], cliffs[i], labels[i]])

    csv_file.close()

if __name__ == '__main__':
    metrics = ['Accuracy', 'Precision', 'Recall', 'F1', 'Kappa', 'AUC', 'MCC']

    result_path = 'results/OurDataset/'
    for dataset in os.listdir(result_path):
        if dataset in ['LongParameterList', 'SwitchStatements']:
            continue
        dataset_path = result_path + '%s/' % dataset
        for metric in metrics:
            print("Doing Wilcoxon signed rank test in %s_%s ..." % (dataset, metric))
            datas = read_data(dataset, dataset_path, metric)
            metric_datas, balancers = process_data(datas)
            Wilcoxon_signed_rank_test(metric_datas, balancers, dataset, metric)


# System library
import os

import psycopg2
# Local imports
from .tables import queries

def init_database():
    """ This method is used initialize database """
    url = os.environ.get('URL')
    conn = psycopg2.connect(url)
    cursor = conn.cursor()
    # print("Connected")
    try:
        for query in queries:
            cursor.execute(query)
        conn.commit()
        return conn
    except Exception as e:
        print(e)

def init_test_db():
    """ This method is used to initialize test database """
    url = os.environ.get("TEST_URL")
    conn = psycopg2.connect(url)
    cursor = conn.cursor()
    dismantle()
    try:
        for query in queries:
            cursor.execute(query)
        conn.commit()
        return conn
    except Exception as e:
        print(e)

def tear_down():
    """ Destroy all data in table in the test database """
    url = os.getenv('DATABASE_TEST_URL')
    conn = psycopg2.connect(url)
    cursor = conn.cursor()
    users = "DROP TABLE IF EXISTS users CASCADE"
    meals = "DROP TABLE IF EXISTS meals CASCADE"
    orders = "DROP TABLE IF EXISTS orders CASCADE"
    category = "DROP TABLE IF EXISTS category CASCADE"
    blacklist = "DROP TABLE IF EXISTS blacklist CASCADE"

    queries = [orders, meals, users, category]
    try:
        for query in queries:
            cursor.execute(query)
        conn.commit()
    except Exception as e:
        print(e)

# dbconn_init()


import numpy as np
import pandas as pd

from utils.read_data import read_data
from utils.plot import plot_var
from utils.var import var

transform = ["RGDP", "RGOV", "RCONS", "RINV"]
to = ["Y", "G", "CONS", "INV"]

name_map = dict(zip(transform, to))

def parse_data(df: pd.DataFrame)->pd.DataFrame:

    new_df = df.copy()
    pop = new_df["POP"]

    for transf_var in transform:
        trans_data = np.log(new_df[transf_var] / pop)
        new_df[name_map[transf_var]] = trans_data

    return new_df.drop(transform, axis = 1)

if __name__ == "__main__":
    df = read_data("data/ramey.xls")
    parsed_df = parse_data(df)["1960Q1":]

    restr_var = ["G", "T", "Y"]
    restr_df = parsed_df[restr_var]
    res = var(restr_df, trend="ctt", lags = 4)
    plot_var(res, impulse="G", folder="ramey/base")

    aug_var = restr_var + ["CONS", "INV"]
    aug_df = parsed_df[aug_var]
    res = var(aug_df, trend="ctt", lags = 4)
    plot_var(res, impulse="G", folder="ramey/aug")

    war_var = ["WAR"] + aug_var
    war_df = parsed_df[war_var]
    res = var(war_df, trend="ctt", lags = 4)
    plot_var(res, impulse="WAR", folder="ramey/war")

    mil_var = ["MIL"] + aug_var
    mil_df = parsed_df[mil_var]
    res = var(mil_df, trend="ctt", lags = 4)
    plot_var(res, impulse="MIL", folder="ramey/mil")


from unittest.mock import MagicMock

import pytest
from django.urls import reverse
from store_kit.http import status

from nest.recipes.ingredients.endpoints import (
    recipe_ingredient_create_api,
    recipe_ingredient_delete_api,
    recipe_ingredient_groups_create_api,
    recipe_ingredient_groups_list_api,
    recipe_ingredient_list_api,
)

from ..factories.endpoints import Endpoint, EndpointFactory, FactoryMock, Request
from ..factories.records import (
    RecipeIngredientItemGroupRecordFactory,
    RecipeIngredientRecordFactory,
)
from ..helpers.clients import authenticated_client, authenticated_staff_client

recipe_ingredient_create_api_factory = EndpointFactory(
    endpoint=Endpoint(
        url=reverse("api-1.0.0:recipe_ingredient_create_api"),
        method="POST",
        view_func=recipe_ingredient_create_api,
        mocks=[FactoryMock("create_recipe_ingredient", None)],
        payload={"title": "Ingredient 1", "product": 1},
    ),
    requests={
        "authenticated_request": Request(
            help="Test that normal users are unable to create recipe ingredients",
            client=authenticated_client,
            expected_status_code=status.HTTP_401_UNAUTHORIZED,
            expected_mock_calls={"create_recipe_ingredient": 0},
        ),
        "staff_request": Request(
            help="Test that staff users are able to create recipe ingredients",
            client=authenticated_staff_client,
            expected_status_code=status.HTTP_201_CREATED,
            expected_mock_calls={"create_recipe_ingredient": 1},
        ),
    },
)

recipe_ingredient_list_api_factory = EndpointFactory(
    endpoint=Endpoint(
        url=reverse("api-1.0.0:recipe_ingredient_list_api"),
        view_func=recipe_ingredient_list_api,
        mocks=[
            FactoryMock(
                "get_recipe_ingredients", [RecipeIngredientRecordFactory.build()]
            )
        ],
    ),
    requests={
        "authenticated_request": Request(
            help="Test that normal users are able to retrieve a list of ingredients",
            client=authenticated_staff_client,
            expected_status_code=status.HTTP_200_OK,
            expected_mock_calls={"get_recipe_ingredients": 1},
        ),
    },
)

recipe_ingredient_delete_api_factory = EndpointFactory(
    endpoint=Endpoint(
        url=reverse("api-1.0.0:recipe_ingredient_delete_api"),
        method="DELETE",
        view_func=recipe_ingredient_delete_api,
        mocks=[FactoryMock("delete_recipe_ingredient", None)],
        payload={"ingredient_id": 1},
    ),
    requests={
        "authenticated_request": Request(
            help="Test that normal users are unable to delete recipe ingredients",
            client=authenticated_client,
            expected_status_code=status.HTTP_401_UNAUTHORIZED,
            expected_mock_calls={"delete_recipe_ingredient": 0},
        ),
        "staff_request": Request(
            help="Test that staff users are able to delete recipe ingredients",
            client=authenticated_staff_client,
            expected_status_code=status.HTTP_200_OK,
            expected_mock_calls={"delete_recipe_ingredient": 1},
        ),
    },
)

recipe_ingredient_groups_create_api_factory = EndpointFactory(
    endpoint=Endpoint(
        url=reverse("api-1.0.0:recipe_ingredient_groups_create_api", args=[1]),
        method="POST",
        view_func=recipe_ingredient_groups_create_api,
        mocks=[FactoryMock("create_recipe_ingredient_item_groups", None)],
        payload=[
            {
                "title": "Group 1",
                "ordering": 1,
                "ingredients": [
                    {
                        "ingredient": "1",
                        "portion_quantity": "1",
                        "unit": "1",
                        "additional_info": None,
                    }
                ],
            }
        ],
    ),
    requests={
        "authenticated_request": Request(
            help="Test that normal users are unable to create recipe ingredient item groups",
            client=authenticated_client,
            expected_status_code=status.HTTP_401_UNAUTHORIZED,
            expected_mock_calls={"create_recipe_ingredient_item_groups": 0},
        ),
        "staff_request": Request(
            help="Test that staff users are able to create recipe ingredient item groups",
            client=authenticated_staff_client,
            expected_status_code=status.HTTP_201_CREATED,
            expected_mock_calls={"create_recipe_ingredient_item_groups": 1},
        ),
    },
)

recipe_ingredient_groups_list_api_factory = EndpointFactory(
    endpoint=Endpoint(
        url=reverse("api-1.0.0:recipe_ingredient_groups_list_api", args=[1]),
        view_func=recipe_ingredient_groups_list_api,
        mocks=[
            FactoryMock(
                "get_recipe_ingredient_item_groups_for_recipe",
                [RecipeIngredientItemGroupRecordFactory.build()],
            )
        ],
    ),
    requests={
        "authenticated_request": Request(
            help=(
                "Test that normal users are unable to retrieve a list of recipe "
                "ingredient item groups"
            ),
            client=authenticated_client,
            expected_status_code=status.HTTP_401_UNAUTHORIZED,
            expected_mock_calls={"get_recipe_ingredient_item_groups_for_recipe": 0},
        ),
        "staff_request": Request(
            help=(
                "Test that staff users are able to retrieve a list of recipe "
                "ingredient item groups"
            ),
            client=authenticated_staff_client,
            expected_status_code=status.HTTP_200_OK,
            expected_mock_calls={"get_recipe_ingredient_item_groups_for_recipe": 1},
        ),
    },
)

request_factories = [
    recipe_ingredient_create_api_factory,
    recipe_ingredient_list_api_factory,
    recipe_ingredient_delete_api_factory,
    recipe_ingredient_groups_create_api_factory,
    recipe_ingredient_groups_list_api_factory,
]

@pytest.mark.parametrize("factory", request_factories)
@pytest.mark.django_db
def test_recipes_ingredients_endpoints(
    factory: EndpointFactory, mocker: MagicMock
) -> None:
    factory.make_requests_and_assert(mocker)


import nltk
from nltk.corpus import brown

news_text = brown.words(categories='romance')
#Desoliegue de propiedades de un corpus en particular, en este caso brown
#print('Numero de categorias: ', str(len(brown.categories())))
#print('Numero de archivos: ',  str(len(brown.fileids())))
#print('Numero de caracteres: ',str(len(brown.raw('cr09'))))
#print('Numero de palabras: ', str(len(brown.words('cr09'))))
#print('Numero de oraciones: ', str(len(brown.sents('cr09'))))

#Conteo de conicidencias
#fdist =  nltk.FreqDist([w.lower() for w in news_text]) #texto
#modals = ['love', 'hate'] #Palabras a contar frecuencia
#for m in modals:
#    print(m + ':', fdist[m]) #Buscamos y mostramos la frecuencia

#Conteo de conincidencias en diferentes categorias
cdf = nltk.ConditionalFreqDist(
    (genre,word)
    for genre in brown.categories()
    for word in brown.words(categories=genre))

genres = ['news','romance', 'humor'] #Generos donde se buscara
words = ['hate','love'] #Palabras a buscar en genero

cdf.tabulate(conditios=genres, samples=words)
cdf.plot()


import discord
import typing

from bot import BasicCog
from discord.ext import commands
from utils.checks import access_level, AccessLevel

class Database(BasicCog):
    def __init__(self, bot):
        self.bot = bot

    @commands.group()
    @access_level(AccessLevel.TRUSTED)
    async def cache(self, ctx):
        if not ctx.invoked_subcommand:
            await ctx.send('You must provide a valid subcommand.')

    @cache.command(name='clear')
    async def cache_clear(self, ctx):
        """Clears the cache."""
        self.bot.db.clear_cache()
        await ctx.send('Cache cleared.')

    @cache.command(name='get')
    async def cache_get(self, ctx, guild_id: typing.Optional[int] = None):
        """Gets the cache for a guild."""
        if guild_id is None:
            guild_id = ctx.guild.id

        cache = self.bot.db.guild_cache.get(guild_id)
        if cache is None:
            await ctx.send('No cache found.')
            return

        await ctx.send(f'Cache for {guild_id}: ```json\n{cache}```')

    @commands.group()
    @access_level(AccessLevel.TRUSTED)
    async def db(self, ctx):
        if not ctx.invoked_subcommand:
            await ctx.send('You must provide a valid subcommand.')

    @db.command(name='guild')
    async def db_guild(self, ctx, guild_id: typing.Optional[int] = None):
        """Gets the database entry for a guild."""
        if guild_id is None:
            guild_id = ctx.guild.id

        entry = await self.bot.db.get_guild(guild_id)
        if entry is None:
            await ctx.send('No entry found.')
            return

        await ctx.send(f'Entry for {guild_id}: ```json\n{dict(entry)}```')

    @db.command(name='user')
    async def db_user(self, ctx, user: typing.Optional[discord.User] = None):
        """Gets the database entry for a user."""
        if user is None:
            user = ctx.author

        entry = await self.bot.db.get_user(user.id)
        if entry is None:
            await ctx.send('No entry found.')
            return

        await ctx.send(f'Entry for {user.id}: ```json\n{dict(entry)}```')

def setup(bot):
    bot.add_cog(Database(bot))


from django.urls import path

from core.views import (
    all_movies,
    add_movie,
    edit_movie,
    delete_movie
)

urlpatterns = [
    path('all/', all_movies, name="all"),
    path('new/', add_movie, name="new"),
    path('edit/<int:id>/', edit_movie, name="edit"),
    path('delete/<int:id>/', delete_movie, name="delete"),
]


"""
zot capture - capture reads matching bait sequences

Usage:
    zot capture [options] <sequences> <input>...

Options:
    -b NUM          number of reads to buffer per bait sequence [default: 4096]
    -k K            value of K to use - must be even. [default: 24]
    -P PREFIX       prefix for output files [default: .]
    -p              treat inputs as paired reads
    -v              produce verbose output
    -z              produce compressed output
"""

import sys
import zipfile

import docopt
import tqdm
import yaml

from zotmer.library.basics import kmersList
from zotmer.library.file import openFile, readFasta, readFastq
from zotmer.library.reads import reads

class ReadCache(object):
    def __init__(self, pfx, nm, paired, B, z):
        self.B = B
        self.nr = 0
        suff = ''
        if z:
            suff = '.gz'
        if paired:
            nm1 = '%s/%s_1.fastq%s' % (pfx, nm, suff)
            nm2 = '%s/%s_2.fastq%s' % (pfx, nm, suff)
            self.names = [nm1, nm2]
            self.buffers = [[], []]
            self.N = 2
        else:
            nm = '%s/%s.fastq%s' % (pfx, nm, suff)
            self.names = [nm]
            self.buffers = [[]]
            self.N = 1

    def add(self, rds):
        assert len(rds) == self.N
        self.nr += 1
        for i in xrange(self.N):
            self.buffers[i].append(rds[i])
        if len(self.buffers[0]) >= self.B:
            self.flush()

    def flush(self):
        for i in range(self.N):
            if len(self.buffers[i]) == 0:
                continue
            with openFile(self.names[i], 'a') as f:
                for rd in self.buffers[i]:
                    print >> f, rd[0]
                    print >> f, rd[1]
                    print >> f, rd[2]
                    print >> f, rd[3]
            self.buffers[i] = []

    def end(self):
        self.flush()
        print >> sys.stderr, '%s: %d' % (self.names[0], self.nr)

def main(argv):
    opts = docopt.docopt(__doc__, argv)

    K = int(opts['-k'])

    B = int(opts['-b'])

    paired = opts['-p']

    verbose = opts['-v']

    Z = opts['-z']

    names = []
    seqs = []
    baits = {}
    with openFile(opts['<sequences>']) as f:
        for (nm, seq) in readFasta(f):
            n = len(names)
            names.append(nm)
            seqs.append(seq)
            for x in kmersList(K, seq, True):
                if x not in baits:
                    baits[x] = set([])
                baits[x].add(n)

    N = len(names)

    caches = [ReadCache(opts['-P'], names[n], paired, B, Z)  for n in range(N)]

    nr = 0
    nh = 0
    for itm in reads(opts['<input>'], reads=True, kmers=True, fwdOnly=True, paired=paired, verbose=verbose):
        nr += 1
        E = len(itm.kmers)
        hits = set([])
        for i in xrange(E):
            fwd = itm.kmers[i]
            for x in fwd:
                if x in baits:
                    hits |= baits[x]
        for n in hits:
            caches[n].add(itm.reads)

        if len(hits) > 0:
            nh += 1

    for n in xrange(N):
        caches[n].end()

if __name__ == '__main__':
    main(sys.argv[1:])


from selenium import webdriver
import time
import os

link = "http://suninjuly.github.io/file_input.html"

try:
    browser = webdriver.Chrome()
    browser.get(link)

    input1 = browser.find_element_by_name('firstname')
    input1.send_keys("Oleg")

    input2 = browser.find_element_by_name('lastname')
    input2.send_keys("Zharov")

    input3 = browser.find_element_by_name('email')
    input3.send_keys("Oleg@example.com")

    input4 = browser.find_element_by_id('file')
    current_dir = os.path.abspath(os.path.dirname(__file__))
    file_path = os.path.join(current_dir, 'file.txt')
    input4.send_keys(file_path)

    button = browser.find_element_by_class_name("btn.btn-primary")
    button.click()

finally:
    # успеваем скопировать код за 30 секунд
    time.sleep(10)
    # закрываем браузер после всех манипуляций
    browser.quit()

# another
# from selenium import webdriver
# from selenium.webdriver.common.by import By
# from time import sleep
# import os
#
# try:
#     browser = webdriver.Chrome()
#     browser.get('http://suninjuly.github.io/file_input.html')
#     current_dir = os.path.abspath(os.path.dirname(__file__))
#     file_path = os.path.join(current_dir, 'file.txt')
#
#     selector = ["[name = 'firstname']", "[name='lastname']", "[name='email']", "[name='file']"]
#     answers = ['first_name', 'second_name', 'e_mail@mail.ru',file_path]
#
#     for k, v in enumerate(selector):
#         browser.find_element(By.CSS_SELECTOR, v).send_keys(answers[k])
#
#     submit_button=browser.find_element(By.TAG_NAME, 'button').click()
# finally:
#     sleep(5)
#     browser.quit()

#one more
# import os
# import time
# from selenium import webdriver
# from selenium.webdriver.common.by import By
#
# driver = webdriver.Chrome()
# url = "http://suninjuly.github.io/file_input.html"
#
# try:
#     driver.get(url)
#
#     current_dir = os.path.abspath(os.path.dirname(__file__))
#     file_path = os.path.join(current_dir, "filetest.txt")
#
#     driver.find_element(By.NAME, "firstname").send_keys("** Name **")
#     driver.find_element(By.NAME, "lastname").send_keys("** Last name **")
#     driver.find_element(By.NAME, "email").send_keys("** This is email **")
#     time.sleep(1)
#     driver.find_element(By.ID, "file").send_keys(file_path)
#     driver.find_element(By.CSS_SELECTOR, "[type='submit']").click()
#
# except Exception as er:
#     print("*** ОШИБКА ***", er)
# finally:
#     time.sleep(7)
#     driver.quit()


#!/usr/bin/env python
#
# Reads an HTML file, dumps the rows of the first table it finds.
#
import sys
from bs4 import BeautifulSoup
from bs4util.spantable import TableFrame

filename = sys.argv[1]

html = "\n".join(open(filename,"rt").readlines())
soup = BeautifulSoup(html)
tables = soup.find_all('table')
print("Document has %d table(s)." % len(tables))
table = tables[0]

frame = TableFrame(table)
for row in frame.rows():
    print(row)


import dash
from dash import dcc, callback, Input, Output
import pandas as pd
import plotly.express as px
from modules.utils import get_reg_country_dict, get_layout, get_data_from_api

page_name = "Cereal Yield"

dash.register_page(__name__, path='/cereal-yield', name=page_name, suppress_callback_exceptions=True)

reg_country_dict = get_reg_country_dict()
suffix = "yield"
layout = get_layout(suffix)

@callback(
    Output("countries-drop-yield","options"),
    Output("countries-drop-yield","value"),
    Input("region-drop-yield","value")
)
def get_countries_by_region(region):
    countries = reg_country_dict[0][region]
    first_country_in_reg = list(countries.keys())[0]
    return countries,first_country_in_reg

@callback(
    Output("container-div-yield","children"),
    Output("store-dataframe-yield","data"),
    Input("countries-drop-yield", "value")
)
def update_line_graph_by_country(selected_country):
    data_frame_update_yield = get_data_from_api(suffix,selected_country)
    max_year = data_frame_update_yield["date_year"].max()
    min_year = data_frame_update_yield["date_year"].min()
    range_slider = dcc.RangeSlider(
        id="year-range-slider-yield",
        min=min_year,
        max=max_year,
        step=1,
        value=[min_year,max_year],
        marks={i:{'label':f"{i}"} for i in range(min_year,max_year,5)}
        )
    return range_slider, data_frame_update_yield.to_json()

@callback(
    Output("line-graph-yield", "figure"),
    Input("store-dataframe-yield","data"),
    Input("year-range-slider-yield","value"),
    Input("log-options-yield","value")
)

def get_data(stored_dataframe, years,graph_log):
    data_frame_from_store_yield = pd.read_json(stored_dataframe)
    data_frame_from_store_yield = data_frame_from_store_yield[data_frame_from_store_yield["date_year"].between(years[0],years[1])]
    log_x = False
    log_y = False
    if graph_log:
        log_x = "x" in graph_log
        log_y = "y" in graph_log
    fig = px.line(data_frame_from_store_yield, x="date_year", y="cereal_yield",
        title=page_name,
        markers=True,
        log_y=log_y,
        log_x=log_x,
        #text="date"
    )
    fig.update_layout(transition_duration=500,
        xaxis_title='Years',
        yaxis_title=page_name),
    return fig


from collections import Counter
import sys

n = int(input())
arr = []

for i in range(n):
    arr.append(int(sys.stdin.readline().strip()))

arr.sort()#오름차순 정렬
dic = Counter(arr).most_common() #빈도수 체크

print("%d\n%d\n%d\n%d" % (round(sum(arr) / n), #평균(반올림)
                          arr[n // 2], #중앙값
                          dic[1][0] if len(arr) > 1 and dic[0][1] == dic[1][1] else dic[0][0], #최빈값 여러개면 최빈값 중 두번째로 작은 수
                          max(arr) - min(arr))) #최대값-최소값                                    (오름차순 정렬 후 빈도수 체크했기 때문에 다음 값이 두번째로 작은 값)


import torch
from matplotlib import pyplot as plt
from torch import nn, optim
from tqdm import tqdm

from my_data_set import MyDataset
from my_model import MyModel

data_file = './data.txt'
label_file = './label.txt'
dataset = MyDataset(data_file, label_file, train_frac=0.7, val_frac=0.15, test_frac=0.15, batch_size=32)

train_dataloader = dataset.get_train_dataloader()
val_dataloader = dataset.get_val_dataloader()
test_dataloader = dataset.get_test_dataloader()

train_loss_list = []
val_loss_list = []
test_loss_list = []

# 定义训练函数
def train(the_model, train_data, val_data, num_epochs, lr):
    criterion = nn.MSELoss()
    optimizer = optim.Adam(the_model.parameters(), lr=lr)
    best_val_loss = float('inf')
    for epoch in range(num_epochs):
        the_model.train()
        train_loss = 0.0
        for X, Y in tqdm(train_data, desc=f'Epoch {epoch + 1} (train)'):
            optimizer.zero_grad()
            pred_y = the_model(X)
            loss = criterion(pred_y, Y)
            loss.backward()
            optimizer.step()
            train_loss += loss.item()
        train_loss /= len(train_data.dataset)
        train_loss_list.append(train_loss)
        the_model.eval()
        val_loss = 0.0
        with torch.no_grad():
            for X, Y in tqdm(val_data, desc=f'Epoch {epoch + 1} (val)'):
                pred_y = the_model(X)
                loss = criterion(pred_y, Y)
                val_loss += loss.item()
            val_loss /= len(val_data.dataset)
        val_loss_list.append(val_loss)

        if val_loss < best_val_loss:
            best_val_loss = val_loss
            torch.save(the_model.state_dict(), 'best_model.pt')

        print(f'Epoch {epoch + 1} | Train loss: {train_loss:.4f} | Val loss: {val_loss:.4f}')

    print('Training finished!')

# 定义测试函数
def test(the_model, test_data):
    criterion = nn.MSELoss()
    the_model.load_state_dict(torch.load('best_model.pt'))
    the_model.eval()
    test_loss = 0.0
    with torch.no_grad():
        for X, Y in tqdm(test_data, desc='Testing'):
            pred_y = the_model(X)
            loss = criterion(pred_y, Y)
            test_loss += loss.item()
        test_loss /= len(test_data.dataset)
    print(f'Test loss: {test_loss:.4f}')

input_size, output_size = dataset.get_data_size()
# 定义模型、训练和测试
model = MyModel(input_size, output_size)
train(model, train_dataloader, val_dataloader, num_epochs=1000, lr=0.001)
test(model, test_dataloader)

fig, ax = plt.subplots()
# 绘制折线图
ax.plot(list(range(100, len(train_loss_list))), train_loss_list[100:], label='train loss')
ax.plot(list(range(100, len(val_loss_list))), val_loss_list[100:], label='val loss')

# 设置图表标题和轴标签
ax.set_title('Train and val loss lines on the model')
ax.set_xlabel('Epoch')
ax.set_ylabel('Loss')

# 添加图例
ax.legend()

# 显示图表
plt.show()

Solving equations using deep learning/train_the_model.py


import requests
import sys
from rich.console import Console
from rich.table import Table
from rich.progress import track

banner = '''

                \033[32;1m_______ _______  _____  _     _ _______ _____ __   _\033[0m
                \033[31;1m|_____| |       |   __| |     | |______   |   | \  |\033[0m
                \033[32;1m|     | |_____  |____\| |_____| |       __|__ |  \_|\033[0m

                    \033[33;1mAqacquisition Domain Finder\033[0m   \033[34;1m-Author: [0xAgun]\033[0m

'''

print(banner)

table = Table(title="Enumerating domain for "+sys.argv[1])

table.add_column("Type", justify="right", style="cyan", no_wrap=False)
table.add_column("Urls", style="magenta", justify="center")
# table.add_column("Box Office", justify="right", style="green")

mainurl = "https://crt.sh/?q="
domain = sys.argv[1]
req = requests.get(mainurl+domain+'&output=json')
js = req.json()

for url in js:
    j = url['common_name']
    save = open("urls.txt","a")
    save.write(str(j)+"\n")
    save.close()
    table.add_row("domain", j)

console = Console()
console.print(table)

print("\033[31mDone grabbing check urls.txt\033[0m")


import json
import boto3

# Create SQS client
sqs = boto3.client('sqs')

# Get uri from terraform
with open('uri.txt') as f:
    queue_url = f.readline()

def receive_message():
    # Receive message from SQS queue
    response = sqs.receive_message(
        QueueUrl=queue_url,
        AttributeNames=[
            'All'
        ],
        MaxNumberOfMessages=1,
        VisibilityTimeout=10,
        WaitTimeSeconds=10
    )

    message = response['Messages'][0]

    print("Received Message:")
    print(json.dumps(message, indent=4, sort_keys=True))
    return message['ReceiptHandle']

def delete_message(receipt_handle):
    # Delete received message from queue
    sqs.delete_message(
        QueueUrl=queue_url,
        ReceiptHandle=receipt_handle
    )

msg1 = receive_message()    # MessageGroupId 1
msg2 = receive_message()    # MessageGroupId 2
delete_message(msg1)
delete_message(msg2)
msg3 = receive_message()    # MessageGroupId 1
delete_message(msg3)


import asyncio
import gc
import threading
from itertools import count
from time import perf_counter, sleep

import click
import cv2
import numpy as np

from jina import Client, Document

_loop = None

def fire_and_forget(coro):
    global _loop
    if _loop is None:
        _loop = asyncio.new_event_loop()
        threading.Thread(target=_loop.run_forever, daemon=True).start()
    _loop.call_soon_threadsafe(asyncio.create_task, coro)

class JinaClient:
    def __init__(self, host: str = "0.0.0.0", port: int = 4091, use_async=False):
        self.use_async = use_async
        self.client = Client(host=host, port=port, asyncio=use_async)

    @staticmethod
    def read_frames(cap: cv2.VideoCapture, video_path, fps: int = 30):
        try:
            for frame_count in count():
                success, frame = cap.read()
                frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
                yield Document(
                    tensor=np.array(frame),
                    tags={
                        "frame_id": frame_count,
                        "video_path": video_path,
                        "output_stream": "test2",
                    },
                )
        finally:
            cap.release()
        return

    @staticmethod
    async def send_async(frame: Document, client: Client):
        async for _ in client.post(
            on="/infer", inputs=frame, request_size=1, return_responses=True
        ):
            continue

    @staticmethod
    def send_sync(frame: Document, client: Client):
        start = perf_counter()
        client.post(on="/infer", inputs=frame, request_size=1, return_responses=True)
        print(f"Time: {perf_counter() - start}s")

    def infer(self, video_path: str):
        cap = cv2.VideoCapture(video_path)
        for frame in self.read_frames(cap, video_path):
            if self.use_async:
                fire_and_forget(self.send_async(frame, self.client))
                gc.collect()

            else:
                self.send_sync(frame, self.client)
            print("Sent frame!")
            sleep(1 / 30)

@click.command()
@click.option("--video", "-v", type=click.Path(exists=True))
def main(video):
    client = JinaClient(use_async=True)
    client.infer(video)

if __name__ == "__main__":
    main()


# Create your views here.
from django.shortcuts import render_to_response
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.template.context import RequestContext
from models import *
from itertools import chain
from collections import defaultdict

def mzml(request):
    allexp = Experiment.objects.all()
    paginator = Paginator(allexp, 1)  # show 10 mzml per page

    page = request.GET.get('page')
    try:
        contacts = paginator.page(page)
    except PageNotAnInteger:
        # If page is not an integer, deliver first page.
        contacts = paginator.page(1)
    except EmptyPage:
        # If page is out of range (e.g. 9999), deliver last page of results.
        contacts = paginator.page(paginator.num_pages)

    return render_to_response('mzml.html', {"exp": contacts},
                                context_instance=RequestContext(request))

def pepxml(request):
    allpepxml = pepXML.objects.all()
    paginator = Paginator(allpepxml, 1)  # show 10 mzml per page

    page = request.GET.get('page')
    try:
        contacts = paginator.page(page)
    except PageNotAnInteger:
        # If page is not an integer, deliver first page.
        contacts = paginator.page(1)
    except EmptyPage:
        # If page is out of range (e.g. 9999), deliver last page of results.
        contacts = paginator.page(paginator.num_pages)

    return render_to_response('pepxml.html', {"pepxml": contacts},
                                context_instance=RequestContext(request))

def index(request):
    expcount = Experiment.objects.count()
    allmass = SelectedIon.objects.count()
    allpep = pepXML.objects.count()
    lastexp = Experiment.objects.latest('id')
    #lastpep = pepXML.objects.latest('id')
    #cid = request.POST['cnum']
    #fid = request.FILES['myfile']
    errors = []

    """
    1) check whether post is from input text and it's not blank.
    2) get all forms request values mz, searchdate, search space, charge state
    and ppm.
    3) if charge state is none (default) then calculate lower range and upper
    ppm range.
    4) if charge state is provided in the request then protonated it by
    multiplying with 1.00727638 and caculate lower and upper ppm mass range.
    """
    if 'textmz' in request.POST and request.POST['textmz'] != '' and \
    'myfile' not in request.FILES:
        cid = request.POST['textmz']
        searchdate = request.POST['fdate1']
        sspace = request.POST['r']
        cstate = request.POST['ch']
        ppm = request.POST.__getitem__('ppm')

        if request.POST['ch'] == 'none':
            defmass = float(ppm) / int(1000000)
            massvalue = float(cid)
            masscon = defmass * massvalue
            highrange = massvalue + masscon
            lowrange = massvalue - masscon
            prec = request.POST['prec']
        elif request.POST['ch'] == 'neutral':
            prec = request.POST['prec']
            masscon, mass_with_charges = [], []

            for charge in range(1, 6):
                mass_with_charges.append((float(cid) + (charge * 1.00727638)) / charge)
            defmass = float(5) / int(1000000)
            [masscon.append(defmass * item) for item in mass_with_charges]
            upper_mass_range = [sum(pair) for pair in zip(mass_with_charges, masscon)]
            lower_mass_range = [a - b for a, b in zip(mass_with_charges, masscon)]
        else:
            defmass = float(ppm) / 1000000
            charge = int(cstate) * float(1.00727638)
            cm = float(cid)
            massvalue = (cm + charge) / int(cstate)
            masscon = defmass * massvalue
            highrange = massvalue + masscon
            lowrange = massvalue - masscon
            prec = request.POST['prec']
        # check all requests/input and perform query
        if searchdate == "" and sspace == 'all':
            if request.POST['ch'] == 'none':
                myquery = SelectedIon.objects.select_related(). \
                            filter(monoiso__range=(lowrange, highrange)). \
                            order_by('monoiso')
                querycount = myquery.count()
            elif request.POST['ch'] == 'neutral':
                myquery, queryresult = [], []
                for x, y in zip(lower_mass_range, upper_mass_range):
                    result = SelectedIon.objects.select_related() \
                                .filter(monoiso__range=(x, y)) \
                                .order_by('monoiso')
                    queryresult.append(result)
                # remove empty item list from the myquery list
                myquery = [x for x in queryresult if x]
                # make a flat list from a list of lists
                myquery = [item for sublist in myquery for item in sublist]
                querycount = len(myquery)
                myquerylist = list(myquery)
                chargecount, masses= [], []
                mass_with_charge = []
                for chargemass in myquerylist:
                    charge = str(chargemass.chargestate)
                    chargecount.append(chargemass.chargestate)
                    masses.append(chargemass.monoiso)
                    mass_with_charge.append([int(chargemass.chargestate), chargemass.monoiso],)
                #mass_dict = {{t[0]:t[1:],} for t in mass_with_charge}
                mass_dict = dict(mass_with_charge)
                d1 = defaultdict(list)
                for k, v in mass_with_charge:
                    d1[k].append(v)
                d = dict((k, tuple(v)) for k ,v in d1.iteritems())
                mytype = type(d)

            else:
                myquery = SelectedIon.objects.select_related() \
                            .filter(chargestate__iexact=cstate) \
                            .filter(monoiso__range=(lowrange, highrange)) \
                            .order_by('monoiso')
                querycount = myquery.count()

        elif searchdate and sspace == 'all':
            if request.POST['ch'] == 'none':
                myquery = SelectedIon.objects.select_related() \
                            .filter(monoiso__range=(lowrange, highrange)) \
                            .filter(sele_fk__exp__uploaddate__iexact=searchdate) \
                            .order_by('monoiso')
                querycount = myquery.count()
            else:
                myquery = SelectedIon.objects.select_related() \
                            .filter(chargestate__iexact=cstate) \
                            .filter(monoiso__range=(lowrange, highrange)) \
                            .filter(sele_fk__exp__uploaddate__iexact=searchdate) \
                            .order_by('monoiso')
                querycount = myquery.count()
        elif searchdate and sspace == 'recent':
            if request.POST['ch'] == 'none':
                myquery1 = SelectedIon.objects.select_related() \
                            .filter(monoiso__range=(lowrange, highrange)) \
                            .filter(sele_fk__exp__uploaddate__iexact=searchdate) \
                            .reverse()[:5]
                myquery = sorted(myquery1, key=lambda o: o.monoiso)
                querycount = sum(1 for result in myquery)
            else:
                myquery1 = SelectedIon.objects.select_related() \
                .filter(chargestate__iexact=cstate) \
                .filter(monoiso__range=(lowrange, highrange)) \
                .filter(sele_fk__exp__uploaddate__iexact=searchdate) \
                .reverse()[:5]
                myquery = sorted(myquery1, key=lambda o: o.monoiso)
                querycount = sum(1 for result in myquery)

        elif searchdate == "" and sspace == 'recent':
            if request.POST['ch'] == 'none':
                myquery1 = SelectedIon.objects.select_related() \
                            .filter(monoiso__range=(lowrange, highrange))[:5]
                myquery = sorted(myquery1, key=lambda o: o.monoiso)
                querycount = sum(1 for result in myquery)
            else:
                myquery1 = SelectedIon.objects.select_related() \
                            .filter(chargestate__iexact=cstate) \
                            .filter(monoiso__range=(lowrange, highrange))[:5]
                myquery = sorted(myquery1, key=lambda o: o.monoiso)
                querycount = sum(1 for result in myquery)
        if myquery:
            template = 'result.html'
            #'high': highrange,'low': lowrange,'qcount': querycount,
            #'masses': masses,'mass_with_charge': mass_with_charge,
            #'mass_dict':mass_dict,
            #'dict':d, 'mytype': mytype
            return render_to_response(template, {'prec': prec, 'query': cid,
                                                'sections': myquery, 'qcount': querycount,
                                                'cs': cstate, 'ppm': ppm,
                                                'defm': defmass, 'ss': sspace,
                                                'sd': searchdate,
                                                },
                                                context_instance=RequestContext(request))
        else:
            errors.append("*Didn't match with any precursor ion mass in database!!")
            #'pep': lastpep,
            return render_to_response('index.html', {'exp': expcount,
                                        'mass': allmass, 'latest': lastexp, 'pep': allpep,
                                        'errmsg': errors,
                                         },
                                        context_instance=RequestContext(request))
    # check whether post is from UPLOAD FILE and TEXT input is blank.
    # get all forms request values mz, searchdate, search space, charge state
    # and ppm.
    # if charge state is none (default) then calculate lower range and upper
    # ppm range.
    # if charge state is provided in the request then protonate it by multiplying
    # with 1.00727638 and caculate lower and upper ppm mass range
    elif 'myfile' in request.FILES and request.POST['textmz'] == '':
        searchdate = request.POST['fdate1']
        sspace = request.POST['r']
        cstate = request.POST['ch']
        ppm = request.POST.__getitem__('ppm')
        prec = request.POST['prec']
        mfile = request.FILES['myfile']
        # split file name and check if extension is txt
        if mfile.name.split('.')[1] == "txt":
            if request.POST['ch'] == 'none':
                fileid = []
                searchid = []
                masscon = []
                defmass = float(ppm) / int(1000000)
                fread = mfile.readlines()
                [searchid.append(float(item)) for item in fread]
                [fileid.append(float(item)) for item in fread]
                [masscon.append(defmass * item) for item in fileid]
                newlist = [sum(pair) for pair in zip(fileid, masscon)]
                newlist1 = [a - b for a, b in zip(fileid, masscon)]
                #newlist2 = [[x, y] for x, y in zip(newlist, newlist1)]
            else:
                defmass = float(ppm) / int(1000000)
                charge = int(cstate) * float(1.00727638)
                prec = request.POST['prec']
                fileid = []
                searchid = []
                masscon = []
                defmass = float(ppm) / int(1000000)
                fread = mfile.readlines()
                [searchid.append(float(item)) for item in fread]
                [fileid.append(float(item)) for item in fread]
                [masscon.append(defmass * item) for item in fileid]
                newlist = [sum(pair) for pair in zip(fileid, masscon)]
                newlist1 = [a - b for a, b in zip(fileid, masscon)]
                #newlist2 = [[x, y] for x, y in zip(newlist, newlist1)]

            if searchdate == "" and sspace == 'all':
                if request.POST['ch'] == 'none':
                    myresult = []
                    for x, y in zip(newlist1, newlist):
                        myquery = SelectedIon.objects.select_related() \
                                    .filter(monoiso__range=(x, y)) \
                                    .order_by('monoiso')
                        myresult.append(myquery)
                else:
                    myresult = []
                    for x, y in zip(newlist1, newlist):
                        myquery = SelectedIon.objects.select_related() \
                                    .filter(chargestate__iexact=cstate) \
                                    .filter(monoiso__range=(x, y)) \
                                    .order_by('monoiso')
                        myresult.append(myquery)
            elif searchdate and sspace == 'all':
                myresult = []
                if request.POST['ch'] == 'none':
                    for x, y in zip(newlist1, newlist):
                        myquery = SelectedIon.objects.select_related() \
                                    .filter(monoiso__range=(x, y)) \
                                    .filter(sele_fk__exp__uploaddate__iexact=searchdate) \
                                    .order_by('monoiso')
                        myresult.append(myquery)
                else:
                    for x, y in zip(newlist1, newlist):
                        myquery = SelectedIon.objects.select_related() \
                        .filter(chargestate__iexact=cstate) \
                        .filter(monoiso__range=(x, y)) \
                        .filter(sele_fk__exp__uploaddate__iexact=searchdate) \
                        .order_by('monoiso')
                        # end of queryset
                        myresult.append(myquery)
            elif searchdate and sspace == 'recent':
                myresult = []
                if request.POST['ch'] == 'none':
                    for x, y in zip(newlist1, newlist):
                        myquery1 = SelectedIon.objects.select_related() \
                                    .filter(monoiso__range=(x, y)) \
                                    .filter(sele_fk__exp__uploaddate__iexact=searchdate) \
                                    .reverse()[:5]
                        # end of queryset
                        myquery = sorted(myquery1, key=lambda o: o.monoiso)
                        myresult.append(myquery)
                else:
                    for x, y in zip(newlist1, newlist):
                        myquery1 = SelectedIon.objects.select_related(). \
                                    filter(chargestate__iexact=cstate). \
                                    filter(monoiso__range=(x, y)) \
                                    .filter(sele_fk__exp__uploaddate__iexact=searchdate) \
                                    .reverse()[:5]
                        # end of queryset
                        myquery = sorted(myquery1, key=lambda o: o.monoiso)
                        myresult.append(myquery)
            elif searchdate == "" and sspace == 'recent':
                myresult = []
                if request.POST['ch'] == 'none':
                    for x, y in zip(newlist1, newlist):
                        myquery1 = SelectedIon.objects.select_related(). \
                                    filter(monoiso__range=(x, y))[:5]
                        # end of queryset
                        myquery = sorted(myquery1, key=lambda o: o.monoiso)
                        myresult.append(myquery)
                else:
                    for x, y in zip(newlist1, newlist):
                        myquery1 = SelectedIon.objects.select_related() \
                                    .filter(chargestate__iexact=cstate) \
                                    .filter(monoiso__range=(x, y))[:5]
                        # end of queryset
                        myquery = sorted(myquery1, key=lambda o: o.monoiso)
                        myresult.append(myquery)

            # count the length of the myresult queryset list
            querycount = len([num for elem in myresult for num in elem])
            if querycount > 1:
                template = 'result.html'
                return render_to_response(template, {'prec': prec,
                                                    'filesection': myresult,
                                                    'qcount': querycount,
                                                    'range': range(1, (querycount + 1)),
                                                    'cs': cstate, 'ppm': ppm,
                                                    'ss': sspace, 'sd': searchdate,
                                                    'file': mfile,
                                                    'search_id': searchid,
                                                    'fileid': fileid, },
                                                    context_instance=RequestContext(request))
            else:
                errors.append("*Didn't match with any precursor ion masses in database!!!!")
                return render_to_response('index.html', {'exp': expcount,
                                        'mass': allmass, 'latest': lastexp, 'pep': allpep,
                                        'errmsg': errors,
                                         },
                                        context_instance=RequestContext(request))
        else:
            errors.append("*Only text file is suppored!!")
            return render_to_response('index.html', {'exp': expcount, 'mass': allmass,
                                                    'latest': lastexp, 'pep': allpep,
                                                    'errmsg': errors,
                                                    },
                                                     context_instance=RequestContext(request))

    elif 'myfile' in request.FILES and 'textmz' in request.POST:
        errors.append("*Search doesn't accept multiple inputs!!!")
        return render_to_response('index.html', {'exp': expcount, 'mass': allmass, 'pep': allpep,
                                                'latest': lastexp, 'errmsg': errors,
                                                 },
                                                context_instance=RequestContext(request))

    elif ('textmz' in request.POST and request.POST['textmz'] == '') \
        or ('myfile' in request.FILES and request.FILES['myfile'] == ''):
        errors.append("*Search requires one input!!!")
        #'pep': allpep,
        return render_to_response('index.html', {'exp': expcount, 'mass': allmass,
                                                'latest': lastexp, 'errmsg': errors,
                                                 'pep': allpep,},
                                                context_instance=RequestContext(request))

    else:
    	#'pep': allpep,
        return render_to_response('index.html', {'exp': expcount, 'mass': allmass,
                                            'latest': lastexp,
                                            'user': request.user, 'pep': allpep,},
                                             context_instance=RequestContext(request))

def detail(request, match, tab):
    if request.is_ajax():
        spect = Spectrum.objects.filter(spec_fk__id__iexact=match).select_related()
        spect_cv = Spectrum_CV.objects.filter(spec_cv_fk__spec_fk__id__iexact=match)
        scanwin = ScanWindow.objects.filter(scanw_fk__spec_fk__id__iexact=match)
        scan = Scan.objects.filter(scan_fk__spec_fk__id__iexact=match)
        expfile = SelectedIon.objects.get(id=match)
        expname = expfile.sele_fk.exp.filename
        scanlist = list(scan)
        template = 'default.html'
        return render_to_response(template, {'spect': spect, 'spect_cv': spect_cv,
                                            'expname': expname, 'scanw': scanwin,
                                            'scan': scan,'scanlist':scanlist, },
                                            context_instance=RequestContext(request))

def matchdetail(request, match_id, tab_id):
    if request.is_ajax():
        spect = Spectrum.objects.filter(spec_fk__id__iexact=match_id).select_related()
        spect_cv = Spectrum_CV.objects.filter(spec_cv_fk__spec_fk__id__iexact=match_id)
        scanwin = ScanWindow.objects.filter(scanw_fk__spec_fk__id__iexact=match_id)
        scan = Scan.objects.filter(scan_fk__spec_fk__id__iexact=match_id)
        expfile = SelectedIon.objects.get(id=match_id)
        expname = expfile.sele_fk.exp.filename
        scanlist = list(scan)
        # spectid = Spectrum.objects.get(spec_fk__id__iexact=match_id)
        # searchspect_id = spectid.spec_index + 1
        template = 'tab1.html'
        return render_to_response(template, {'spect': spect, 'spect_cv': spect_cv,
                                            'expname': expname, 'scanw': scanwin,
                                            'scan': scan, 'scanlist':scanlist,},
                                            context_instance=RequestContext(request))

def tabdetail(request, tab_id, match_id):
    if request.is_ajax():
        tabvalue = tab_id
        if tabvalue == '1':
            spect = Spectrum.objects.filter(spec_fk__id__iexact=match_id).select_related()
            spect_cv = Spectrum_CV.objects.filter(spec_cv_fk__spec_fk__id__iexact=match_id)
            scanwin = ScanWindow.objects.filter(scanw_fk__spec_fk__id__iexact=match_id)
            scan = Scan.objects.filter(scan_fk__spec_fk__id__iexact=match_id).values()
            scanlist = list(scan)

            template = 'tab1.html'
            return render_to_response(template, {
                                                'scanlist': scanlist, 'spect': spect, 'spect_cv': spect_cv,
                                                'scan': scan, 'scanw': scanwin,},
                                                context_instance=RequestContext(request))
        # end of tab 1
        elif tabvalue == '2':
            spect = Spectrum.objects.get(spec_fk__id__iexact=match_id)
            searchspect_id = spect.spec_index + 1
            expfile = SelectedIon.objects.get(id=match_id)
            expname = expfile.sele_fk.exp.filename
            result = Searchspectrum_Query.objects.filter(spec_query_fk__filename=expname) \
                        .filter(spectrum=searchspect_id)
            # end of queryset
            for i in result:
                spect_id = i.id
            pepsearch = Search_Hit.objects \
                        .filter(search_hit_fk__spec_query_fk__filename=expname) \
                        .filter(search_hit_fk__spectrum=searchspect_id).order_by('hit_rank')
            # end of queryset
            if pepsearch:
                altp = Alternative_Protein.objects \
                        .filter(altpro_fk__spec_query_fk__filename=expname) \
                        .filter(altpro_fk=spect_id)
                # end of queryset
                template = 'tab2.html'
                return render_to_response(template, {'spectquery': result,
                                                    'pepsearch': pepsearch,
                                                    'alt_p': altp, 'spect': spect, },
                                                     context_instance=RequestContext(request))
            else:
                template = 'tab2.html'
                return render_to_response(template, {'error': 'No match found', },
                                        context_instance=RequestContext(request))
        # end of tab 2
        elif tabvalue == '4':
            data = SelectedIon.objects.filter(id=match_id)
            mzml_query = SelectedIon.objects.get(id=match_id)
            mzml_id = str(mzml_query.sele_fk.id)
            sf = Source_File.objects.filter(mzml_fk__id__iexact=mzml_id)
            source = Source_file_CV.objects.filter(sf_fk__id__iexact=mzml_id)
            soft = Software.objects.filter(soft_fk__id__iexact=mzml_id)
            result_list = list(chain(sf, source))
            software = Software_CV.objects.filter(soft_cv_fk__id__iexact=mzml_id).select_related()
            datap = DataProcessing.objects.filter(data_fk__id__iexact=mzml_id)
            data_p = Data_Pro_CV.objects.filter(pro_meth_fk__data_p_fk__data_fk__id__iexact=mzml_id).select_related()
            inss = InsConSource.objects.filter(inss_fk_id__id__iexact=mzml_id)
            insa = InsConAnalyzer.objects.filter(insa_fk_id__id__iexact=mzml_id)
            insd = InsConDetector.objects.filter(insd_fk_id__id__iexact=mzml_id)
            insid = InstrumentConfiguration.objects.filter(ins_pro_fk__id__iexact=mzml_id)
            instru_list = list(chain(inss, insa, insd))
            template = 'tab4.html'
            return render_to_response(template, {'instrument_list': instru_list,
                                                'ins_id': insid, 'results': data,
                                                'source': result_list, 'sf': sf,
                                                'scv': source, 'software': software,
                                                'datap': datap, 'datapro': data_p,
                                                'inss': inss, 'insa': insa,
                                                'insd': insd, 'soft': soft, },
                                                context_instance=RequestContext(request))
        # end of tab 4
        else:
            spect = Spectrum.objects.get(spec_fk__id__iexact=match_id)
            searchspect_id = spect.spec_index + 1
            expfile = SelectedIon.objects.get(id=match_id)
            expname = expfile.sele_fk.exp.filename
            result = Searchspectrum_Query.objects.filter(spec_query_fk__filename=expname).filter(spectrum=searchspect_id)
            for i in result:
                spect_id = i.id
            setting = Search_Setting.objects.filter(search_fk__filename=expname)
            template = 'tab5.html'
            return render_to_response(template, {'ssetting': setting, },
                                        context_instance=RequestContext(request))


from django import template
register = template.Library()

@register.inclusion_tag('panel/templateTags/link.html')
def link(request, link_name, content):
    return{
        'request':request,
        'link_name':link_name,
        'link':f'panel:{link_name}',
        'content':content
    }