#!/usr/bin/env python3
import os
import re
import sys
import time
import queue
import string
import subprocess
import collections
import urllib.parse
import configparser
import multiprocessing
import multiprocessing.managers

import bs4
import colorama

from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
#from selenium.webdriver.firefox.webdriver import Webdriver as fWebdriver
from selenium.webdriver import Firefox, FirefoxProfile
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary

import download
import commandline
import useful.functions

def CreateTheHtml(TypeOfPage, Data = None,):
    '''
    Creates the html unter the name Webpage.html that will be displayed for the user to enter the url link
    to the exteraction website.
    @param OutputFileName:  Is the preconfigured configfile.
    @return FileName:       Is the name of the temporary webpage.
    '''
    FileName = "Webpage.html"
    FileName = os.path.abspath(FileName)
    HtmlStructure = """
        <!DOCTYPE html>

            <meta charset="utf-8">

        """

    HtmlCode = bs4.BeautifulSoup(HtmlStructure, "html.parser")

    if TypeOfPage.upper() == "input".upper():
        Title = HtmlCode.new_tag("title")
        Title.string = "Input"
        HtmlCode.head.append(Title)

        JavaScriptStructure = ("""function SetFutureUrl() {
                                var currentDiv = document.getElementById("tForm");

                                var mainDiv = document.createElement("div");
                                mainDiv.setAttribute("id", "futureWrapper");
                                var FirstText ="Thank you for entering the Url."+
                                "You will be redirected to:";
                                var TextNode = document.createTextNode(FirstText);
                                mainDiv.appendChild(TextNode);

                                var UrlValue = document.getElementById("Url").value;
                                var TextNode = document.createTextNode(UrlValue);
                                var ValueContainer = document.createElement("p");
                                ValueContainer.appendChild(TextNode);

                                ValueContainer.setAttribute("id", "myFutureUrl");
                                mainDiv.appendChild(ValueContainer);
                                var SecondText = "Please be patient."
                                var TextNode = document.createTextNode(SecondText);
                                mainDiv.appendChild(TextNode);

                                var thirdDiv = document.createElement("div");
                                thirdDiv.classList.add("visible");
                                var ValueContainer = document.createElement("p");
                                ValueContainer.setAttribute("id", "ThirdfutureWrapper");
                                var RadiosValue = document.querySelector('input[name = "DeepScan"]:checked').value;
                                var TextNode = document.createTextNode(RadiosValue);
                                ValueContainer.appendChild(TextNode);
                                thirdDiv.appendChild(ValueContainer);
                                document.body.insertBefore(thirdDiv, currentDiv);

                                var secondaryDiv = document.createElement("div");
                                secondaryDiv.classList.add("visible");
                                var ValueContainer = document.createElement("p");
                                ValueContainer.setAttribute("id", "SecondaryfutureWrapper");
                                var outputFile = document.getElementById("outputFile").value;
                                var TextNode = document.createTextNode(outputFile);
                                ValueContainer.appendChild(TextNode);
                                secondaryDiv.appendChild(ValueContainer);
                                document.body.insertBefore(secondaryDiv, currentDiv);

                                document.body.insertBefore(mainDiv, currentDiv);
                                currentDiv.classList.add("visible");

                            }
                            function startTimer(duration, display) {
                                var timer = duration, minutes, seconds;
                                setInterval(function () {
                                    minutes = parseInt(timer / 60, 10)
                                    seconds = parseInt(timer % 60, 10);
                                    minutes = minutes < 10 ? "0" + minutes : minutes;
                                    seconds = seconds < 10 ? "0" + seconds : seconds;
                                    display.textContent = minutes + ":" + seconds;
                                    if (--timer < 0) {
                                        timer = duration;
                                    }
                                }, 1000);
                            }
                            window.onload = function () {
                                var timerTime = 29,
                                display = document.querySelector('#time');
                                startTimer(timerTime, display);
                            };""")

        #   adding javascript
        Javascript = HtmlCode.new_tag("script")
        Javascript["type"] = "text/javascript"
        Javascript.string = JavaScriptStructure

        HtmlCode.head.append(Javascript)
        Registration = bs4.BeautifulSoup('
Registration closes in 00:30 minutes!
', "html.parser")

        Form = HtmlCode.new_tag("form")
        Form["class"] = "Content"
        Form["id"] = "tForm"
        Form["name"] = "tForm"
        Form["onsubmit"] = "SetFutureUrl(); return false;"
        Form.append(Registration)
        HtmlCode.body.append(Form)

        Description = HtmlCode.new_tag("p")
        Description.string = "Do you wish a deep scan?"
        HtmlCode.body.form.append(Description)

        Options = HtmlCode.new_tag("table")
        Options["class"] = "options"

        for i in ("True", "False"):
            Row = HtmlCode.new_tag("tr")

            Texttd = HtmlCode.new_tag("td")
            Text = HtmlCode.new_tag("label")
            Text["for"] = "radio{}".format(i)
            Text.string = i
            Texttd.append(Text)

            ColumnInput = HtmlCode.new_tag("td")
            RadioButton = HtmlCode.new_tag("input")
            RadioButton["id"] = "radio{}".format(i)
            RadioButton["type"] = "radio"
            RadioButton["name"] = "DeepScan"
            RadioButton["value"] = i
            if i == "True":
                RadioButton["checked"] = "checked"

            ColumnInput.append(RadioButton)

            Row.append(Texttd)
            Row.append(ColumnInput)

            Options.append(Row)

        HtmlCode.body.form.append(Options)

        Description = HtmlCode.new_tag("p")
        Description.string = "Please enter the Url to rip:"

        HtmlCode.body.form.append(Description)

        UrlTag = HtmlCode.new_tag("input")
        UrlTag["id"] = "Url"
        UrlTag["name"] = "Url"
        UrlTag["type"] = "text"
        UrlTag["value"] = ""

        HtmlCode.body.form.append(UrlTag)

        Description = HtmlCode.new_tag("p")
        Description.string = "Please enter the file name for the output:"

        HtmlCode.body.form.append(Description)

        FileOutputTag = HtmlCode.new_tag("input")
        FileOutputTag["id"] = "outputFile"
        FileOutputTag["name"] = "outputFile"
        FileOutputTag["type"] = "text"
        FileOutputTag["value"] = Data

        HtmlCode.body.form.append(FileOutputTag)

        HtmlCode.body.form.append(HtmlCode.new_tag("br"))
        HtmlCode.body.form.append(HtmlCode.new_tag("br"))
        SubmitTag = HtmlCode.new_tag("input")
        SubmitTag["type"] = "submit"
        SubmitTag["value"] = "Submit"

        HtmlCode.body.form.append(SubmitTag)

    with open(FileName, "w", encoding="utf-8") as File:
        File.write(HtmlCode.prettify())

    return FileName

def GetData(Source):
    '''
    Writes the basic configurationfile, that contains the default data and a comment.
    @param Source:      The html from the website to be extracted
    @return Url:        The extracted url
    @return FileName:   The extracted output file name
    '''
    Soup = bs4.BeautifulSoup(Source, "html.parser")
    Url = Soup.find("p", id = "myFutureUrl").get_text()
    FileName = Soup.find("p", id = "SecondaryfutureWrapper").get_text()
    DeepScan = Soup.find("p", id = "ThirdfutureWrapper").get_text()
    return Url, FileName, DeepScan

def LogPrint(Message, Type = "INFO"):
    '''
    Is responsable for all the prints on to the screen
    @param Message:     the message to be printed
    @param Type:        what type of information the message is (=> ERROR, INFO, CRITICAL etc.)
    '''
    Type = Type.upper()
    #   check if the module colorama is installed and if the message type is correct.
    if (Type == "ERROR" or Type == "CRITICAL"):
        Type =  "{Bright}{Color}{TypeText}{ResetBright}".format(
            Bright = colorama.Style.BRIGHT,
            Color = colorama.Fore.RED,
            TypeText = Type,
            #ResetColor = colorama.Fore.WHITE,
            ResetBright =  colorama.Style.RESET_ALL
            )

    mesg = ("[{DateOfMessage}] - [{TypeOfMessage}] - "
              "{Message}".format(
                  DateOfMessage = time.strftime("%d.%m.%Y %H:%M:%S",
                                                time.gmtime()
                                                ),
                  TypeOfMessage = Type,
                  Message = Message
                  )
            )

    print(mesg)

def WriteConfig(Configname):
    '''
    Writes the basic configurationfile, that contains the default data and a comment.
    @param Configname:    Configuration File Name
    '''
    #   This sets the Config instance
    config = configparser.ConfigParser()

    #   This commands sets the parser sensitive to upper- and lowercase.
    config.optionxform = lambda option: option

    config["DEFAULT"] =  collections.OrderedDict(
        (
            ("FileName", "Link.txt"),
            ("LinkDirectory", "Links"),
            ("TimeToWait", 30.0),
            ("BrowserPath", ""),
            ("Adblock", True)
        )
    )

    config["DOWNLOAD"] = collections.OrderedDict(
        (
            ("DirectDownload", True),
            ("WhitelistUrl", r"https://drive.google.com/uc?export=download&id=0B2hE6hBecFgzWkphVm5UVUxXeU0"),
            ("BlacklistUrl", r"https://drive.google.com/uc?export=download&id=0B2hE6hBecFgzY05MeTRRLWJkUGc"),
            ("DownloadDirectory", "Downloads"),
            ("ConnectionsPerDownload", 10),
            ("DownloadThreads", 5),
            ("LinkExtractors", 3)
        )
    )

    #   Adding the comment and the prepared dafault configuation data to the file.
    with open(Configname, "w", encoding="utf8") as configfile:
        config.write(configfile)
        configfile.write("#If the LinkDirectory is empty, it will default to the current directory\n"
                         "#If the DownloadDirectory is empty, it will default to the current directory => whatever/AnimeName\n"
                         "#TimeToWait has to be more than 5 seconds and has to be a float.\n"
                         "#BrowserPath maybe empty if ff was installed into the default location, else please add the path."
                         )

def OpenFileDefaultApp(FileName):
    '''
    Opens the File with the default set program. Depending on the operation system.
    @param FileName:    The file path to be opened.
    '''
    if sys.platform.startswith('darwin'):
        subprocess.call(('open', FileName))
    elif os.name == 'nt':
        os.startfile(FileName)
    elif os.name == 'posix':
        subprocess.call(('xdg-open', FileName))

def GetFile(Url):
    Request = urllib.request.Request(Url, headers={
                                                        "User-Agent": "Mozilla/5.0" ,
                                                        "Accept-Encoding": "gzip,deflate",
                                                        "Content-Type": "text/html;charset=utf-8"}
                                    )
    SiteData = urllib.request.urlopen(Request)
    if SiteData.info().get("Content-Encoding") == "gzip":
        SiteData = gzip.decompress(SiteData.read()).decode('utf-8', 'replace')
    else:
        SiteData = SiteData.read().decode('utf-8', 'replace')

    return SiteData

def main():
    """
    Main entry point in to the program.
    """
    argparse = commandline.CustomParser()
    argparse.run()
    parsed = argparse.get_arguments()

    colorama.init(autoreset=True)
    LogPrint("online")
    Configname = "conf.ini"
    SupportedBrowsers = ["Firefox", "Edge"]
    SupportedBrowsers = [i.upper() for i in SupportedBrowsers]

    if not os.path.isfile(Configname):
        WriteConfig(Configname)
        LogPrint("The configuration file has been written.")
    else:

        DownloadFromFile = parsed.File

        config = configparser.RawConfigParser()
        config.read(Configname)

        FileName = config["DEFAULT"]["FileName"]
        DirectoryName = os.path.abspath(config["DEFAULT"]["LinkDirectory"])
        TimeToWait = float(config["DEFAULT"]["TimeToWait"])
        BrowserPath = config["DEFAULT"]["BrowserPath"]
        Adblock = config.getboolean("DEFAULT", "Adblock", )

        DirectDownload = config.getboolean("DOWNLOAD", "DirectDownload", )
        DownloadDirectory = config["DOWNLOAD"]["DownloadDirectory"]
        ConnectionsPerDownload = int(config["DOWNLOAD"]["ConnectionsPerDownload"])
        DownloadThreads = int(config["DOWNLOAD"]["DownloadThreads"])
        WhitelistUrl = config["DOWNLOAD"]["WhitelistUrl"]
        BlacklistUrl = config["DOWNLOAD"]["BlacklistUrl"]
        LinkExtractors = int(config["DOWNLOAD"]["LinkExtractors"])
        Run = True

        if FileName == "":
            Run = False
            LogPrint("Your FileName is empty", "Error")
        """if Browser not in SupportedBrowsers:
            Run = False
            LogPrint("Your chosen browser is not supported or wrongly written.", "Error")
        """
        start_time = time.time()
        if Run is True:

            Source = ""
            SBrowser = None
            Link_list = {}
            AnimeName = ""
            DoDownload = True

            #binary = FirefoxBinary()#r"Drivers/firefox/bin/firefox.exe")
#           firefox_profile = FirefoxProfile()
#            if Adblock is True:
#                adblock_path = os.path.abspath('Drivers/adblock_plus-2.7.3-sm+tb+fx+an.xpi')
#                firefox_profile.add_extension(extension=adblock_path)

            #SBrowser = Firefox()#firefox_binary=binary, )
            path = FirefoxBinary(BrowserPath)
            SBrowser = Firefox(firefox_binary = path) #executable_path = os.path.abspath("Drivers/geckodriver.exe"))
            try:
                RRun = False
                DeepScan = False
                print(SBrowser)

                if DownloadFromFile == "":
                    print(SBrowser)
                    HtmlFileName = CreateTheHtml("input", FileName)
                    SBrowser.get(HtmlFileName)

                    try:
                        Wait = WebDriverWait(SBrowser, 30)
                        element = Wait.until(expected_conditions.presence_of_element_located((By.CSS_SELECTOR, "#myFutureUrl")))
                        RRun = True
                    except:
                        pass
                    finally:
                        if RRun is False:
                            LogPrint("You have failed to enter an url!", "CRITICAL")
                            #SBrowser.quit()

                    #   cleaning up
                    if os.path.isfile(HtmlFileName):
                        os.remove(HtmlFileName)

                    if RRun is True:

                        # get blacklist from gdrive
                        LogPrint("Getting blacklist")
                        Blacklist = GetFile(BlacklistUrl)
                        Blacklist = "|".join([re.escape(Entry) for Entry in [s.strip() for s in Blacklist.splitlines()]])

                        Source = SBrowser.page_source
                        Url, FileName, DeepScan = GetData(Source)

                        if DirectoryName != "":
                            if not os.path.isdir(DirectoryName):
                                os.mkdir(DirectoryName)

                        FileName = os.path.join(DirectoryName, FileName)

                        SBrowser.get(Url)
                        Wait = WebDriverWait(SBrowser, TimeToWait)
                        element = Wait.until(expected_conditions.presence_of_element_located((By.CSS_SELECTOR, "table.listing")))

                        Source = SBrowser.page_source
                        # getting the links from the page
                        Soup = bs4.BeautifulSoup(Source, "html.parser")
                        Table = Soup.find("table", {"class": "listing"})
                        Contentbar = Soup.find("div", {"class": "bigBarContainer"})
                        Container = Contentbar.find("div", {"class": "barContent"})
                        Title = Container.find("a", {"class":"bigChar"})
                        # checking that the title has no problems with the win filesystem
                        AnimeName = Title.string
                        AnimeName = re.sub(r"(?![\w()\-\s]).", " ", AnimeName)

                        BaseURL = urllib.parse.urlparse(Url)
                        t = "{}://{}".format(BaseURL.scheme, BaseURL.netloc)
                        BaseURL = t

                        for Link in Table.find_all("a"):
                            if BaseURL not in Link["href"]:
                                Link["href"] = "{}{}".format(BaseURL, Link["href"])

                            NameOfAnimeEpisode_old = (Link.string).strip(' \t\n\r')
                            NameOfAnimeEpisode = re.sub(Blacklist, "", NameOfAnimeEpisode_old)

                            # checking that there is no problem with the filesystem
                            NameOfAnimeEpisode = re.sub(r"(?![\w()\-\s]).", " ", NameOfAnimeEpisode)

                            # check if last or second last list entry is a number
                            ChangeNumber = False
                            Number_old = "".join(NameOfAnimeEpisode.split(" ")[-1:])
                            if Number_old.isdigit() is True and ChangeNumber is False:
                                ChangeNumber = True
                            else:
                                Number_old = "".join(NameOfAnimeEpisode.split(" ")[-2:-1])

                            if Number_old.isdigit() is True and ChangeNumber is False:
                                ChangeNumber = True

                            if ChangeNumber is True:
                                #   checking if episode number is bigger than 99
                                if Number_old[0] == "0":
                                    Number_new = Number_old[1:]
                                else:
                                    Number_new = Number_old

                                if "OVA" in NameOfAnimeEpisode_old:
                                    Number_new = "OVA {}".format(Number_new)

                                NameOfAnimeEpisode = re.sub(Number_old, Number_new, NameOfAnimeEpisode)

                            #   fixes multiple spaces
                            NameOfAnimeEpisode = re.sub("\s+", " ", NameOfAnimeEpisode)

                            Link_list[NameOfAnimeEpisode] = Link["href"]

                        DownloadableLinks = []
                        if DeepScan == "False":

                            SBrowser.quit()

                            with open(FileName, "w", encoding="utf8") as File:

                                File.write("\n".join(["\t".join(i) for i in Array]))

                            OpenFileDefaultApp(FileName)

                            if DirectDownload is not True:
                                OpenFileDefaultApp(FileName)

                            LogPrint("The HTML code has been riped and written out...")

                if (DirectDownload is True and RRun is True) or DownloadFromFile != "" or DeepScan == "True":
                    LogPrint("Getting whitelist")
                    Whitelist = GetFile(WhitelistUrl)
                    Whitelist = [s.strip() for s in Whitelist.splitlines()]
                    LogPrint("Starting deep scan and download please be patient.")

                    if DownloadDirectory == "":
                        DownloadDirectory = os.path.abspath(AnimeName)
                    else:
                        DownloadDirectory = os.path.abspath(os.path.join(DownloadDirectory, AnimeName))

                    if not os.path.isdir(DownloadDirectory):
                        os.makedirs(DownloadDirectory, exist_ok = True)
                    work_Link_list = [(i, Link_list[i]) for i in useful.functions.natural_sort(Link_list.keys())]

                    Manager = multiprocessing.managers.SyncManager()
                    Manager.start()

                    ProgressQueue = Manager.Queue()
                    DoneWork = Manager.Event()

                    Downloader = download.ArrayDownloader(
                        extractors = LinkExtractors,
                        array = work_Link_list,
                        directory = DownloadDirectory,
                        worker = DownloadThreads,
                        connections = ConnectionsPerDownload,
                        file_name = FileName,
                        browser_type = Browser,
                        adblock = Adblock,
                        whitelist = Whitelist,
                        progress_queue = ProgressQueue,
                        done_work = DoneWork
                    )

                    HtmlFileName = Downloader.check_list_page()
                    SBrowser.get(HtmlFileName)

                    try:
                        Wait = WebDriverWait(SBrowser, 600)
                        Element = Wait.until(expected_conditions.presence_of_element_located((By.CSS_SELECTOR, "#futureEntrys")))
                        Source = SBrowser.page_source
                        if Downloader.analyse_download_page(Source) is False:
                            DoDownload = False
                    except:
                        pass

                    download.ArrayDownloader.delete_file(HtmlFileName)
                    if DoDownload is True:
                        HtmlFileName = Downloader.download_page()
                        Downloader.start()
                        SBrowser.get(HtmlFileName)

                        while not DoneWork.is_set():
                            Progress = None
                            try:
                                Progress = ProgressQueue.get(True, 1)
                            except queue.Empty:
                                pass

                            if Progress is not None:
                                if len(Progress) == 3:
                                    download.ArrayDownloader.update_progressbar(SBrowser, Progress[0], Progress[1], Progress[2])
                                elif len(Progress) == 2:
                                    download.ArrayDownloader.update_state(SBrowser, Progress[0], Progress[1])

                        Manager.shutdown()
                        Downloader.join()
                        download.ArrayDownloader.delete_file(HtmlFileName)
            finally:
                if SBrowser is not None and DoDownload is not True:
                    SBrowser.quit()

                #   cleaning up
                if os.path.isfile("Webpage.html"):
                    os.remove("Webpage.html")

        end_time = time.time()
        LogPrint("Time: {}s".format(end_time-start_time))

    LogPrint("offline")
    os.system("pause")

if __name__ == "__main__":
    multiprocessing.freeze_support()
    main()


import json
import jydoop
import healthreportutils

setupjob = healthreportutils.setupjob

combine = jydoop.sumreducer

def map(key, value, context):
    try:
        payload = json.loads(value)
    except:
        context.write("Bogus\tBogus\tBogus\tBogus", 1)
        return

    output = []
    try:
        info = payload['geckoAppInfo']
        if type(info) == dict:
            output.append(info['name'])
            output.append(info['os'])
            output.append(info['updateChannel'])
            output.append(info['version'])
    except KeyError:
        pass

    if len(output) == 4:
        outkey = "\t".join(output)
        context.write(outkey, 1)

reduce = jydoop.sumreducer


import QSTK.qstkutil.qsdateutil as du
import QSTK.qstkutil.DataAccess as da
import datetime as dt

def get_data(dt_start, dt_end, ls_symbols):
    ldt_timestamps = du.getNYSEdays(dt_start, dt_end, dt.timedelta(hours=16))

    dataobj = da.DataAccess('Yahoo')

    ls_keys = ['open', 'high', 'low', 'close', 'volume', 'actual_close']

    ldf_data = dataobj.get_data(ldt_timestamps, ls_symbols, ls_keys)
    d_data = dict(zip(ls_keys, ldf_data))

    for s_key in ls_keys:
        d_data[s_key] = d_data[s_key].fillna(method='ffill')
        d_data[s_key] = d_data[s_key].fillna(method='bfill')
        d_data[s_key] = d_data[s_key].fillna(1.0)

    return d_data

def get_symbols_in_year(year):
    dataobj = da.DataAccess('Yahoo')

    ls_symbols = dataobj.get_symbols_from_list('sp500' + str(year))
    ls_symbols.append('SPY')

    return ls_symbols


import pytest

from auction_deploy.core import (
    AuctionOptions,
    DeployedAuctionContracts,
    deploy_auction_contracts,
    initialize_auction_contracts,
    missing_whitelisted_addresses,
    whitelist_addresses,
)

@pytest.fixture
def deployed_contracts(web3, auction_options):

    deployed_contracts: DeployedAuctionContracts = deploy_auction_contracts(
        web3=web3, auction_options=auction_options
    )

    return deployed_contracts

def test_deploy_contracts(web3, auction_options: AuctionOptions):

    deployed_contracts: DeployedAuctionContracts = deploy_auction_contracts(
        web3=web3, auction_options=auction_options
    )

    assert (
        deployed_contracts.auction.functions.startPrice().call()
        == auction_options.start_price
    )
    assert (
        deployed_contracts.auction.functions.auctionDurationInDays().call()
        == auction_options.auction_duration
    )
    assert (
        deployed_contracts.auction.functions.minimalNumberOfParticipants().call()
        == auction_options.minimal_number_of_participants
    )
    assert (
        deployed_contracts.auction.functions.maximalNumberOfParticipants().call()
        == auction_options.maximal_number_of_participants
    )

    assert deployed_contracts.locker.functions.initialized().call() is False

    assert deployed_contracts.slasher.functions.initialized().call() is False

def test_init_contracts(deployed_contracts, web3, release_timestamp):

    initialize_auction_contracts(
        web3=web3, contracts=deployed_contracts, release_timestamp=release_timestamp
    )

    assert deployed_contracts.locker.functions.initialized().call() is True
    assert deployed_contracts.slasher.functions.initialized().call() is True

def test_whitelist_addresses(deployed_contracts, whitelist, web3):
    auction_contract = deployed_contracts.auction

    amount = whitelist_addresses(auction_contract, whitelist, batch_size=10, web3=web3)

    for address in whitelist:
        assert auction_contract.functions.whitelist(address).call() is True

    assert amount == len(whitelist)

def test_whitelist_addresses_with_nonce(
    deployed_contracts, whitelist, web3, default_account, account_keys
):
    auction_contract = deployed_contracts.auction

    nonce = web3.eth.getTransactionCount(default_account)

    whitelist_addresses(
        auction_contract,
        whitelist,
        batch_size=10,
        web3=web3,
        transaction_options={"nonce": nonce},
        private_key=account_keys[0],
    )

    for address in whitelist:
        assert auction_contract.functions.whitelist(address).call() is True

SPLIT_SIZE = 20

def test_whitelist_filter(deployed_contracts, whitelist, web3):

    auction_contract = deployed_contracts.auction

    assert missing_whitelisted_addresses(auction_contract, whitelist) == whitelist

    whitelist_addresses(
        auction_contract, whitelist[:SPLIT_SIZE], batch_size=10, web3=web3
    )

    assert (
        missing_whitelisted_addresses(auction_contract, whitelist)
        == whitelist[SPLIT_SIZE:]
    )

def test_whitelist_only_not_whitelisted(deployed_contracts, whitelist, web3):

    auction_contract = deployed_contracts.auction

    number1 = whitelist_addresses(
        auction_contract, whitelist[:SPLIT_SIZE], batch_size=10, web3=web3
    )
    assert number1 == SPLIT_SIZE

    number2 = whitelist_addresses(auction_contract, whitelist, batch_size=10, web3=web3)

    assert number2 == len(whitelist) - number1


import cv2
import numpy as np
import sqlite3
import os
import socket

class sending_message:
    def __init__(self):
        self.TCP_IP = "192.168.43.75"
        self.TCP_PORT = 50001

    def connect(self):
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.sock.connect((self.TCP_IP,self.TCP_PORT))

    def sendto(self, message):
        self.sock.send(message)
        data = self.sock.recv(1024)

    def close(self):
        self.sock.close()

class face_recog:
    def __init__(self):
        self.conn = sqlite3.connect('database.db')
        self.c = self.conn.cursor()
        fname = "recognizer/trainingData.yml"
        if not os.path.isfile(fname):
            print("Please train the data first")
            exit(0)
        self.face_cascade = cv2.CascadeClassifier('haarcascade_frontalface_default.xml')
        self.cap = cv2.VideoCapture(0)
        self.recognizer = cv2.createLBPHFaceRecognizer()
        self.recognizer.load(fname)

    def main_loop(self):
        while True:
              print(cv2.__version__)
              ret, img = self.cap.read()
              gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
              faces = self.face_cascade.detectMultiScale(gray, 1.3, 5)
              for (x,y,w,h) in faces:
                cv2.rectangle(img,(x,y),(x+w,y+h),(0,255,0),3)
                ids,conf = self.recognizer.predict(gray[y:y+h,x:x+w])
                self.c.execute("select name from users where id = (?);", (ids,))
                result = self.c.fetchall()
                name = result[0][0]

                if conf < 50:
                  print("match")
                  cv2.putText(img, name, (x+2,y+h-5), cv2.FONT_HERSHEY_SIMPLEX, 1, (150,255,0),2)
                  cv2.imwrite("test.png", img)
                  with self.conn:
                    self.c=self.conn.cursor()
                    self.c.execute("UPDATE users SET status='rozpoznany' WHERE id = (?);", (ids,))
                    result = self.c.fetchall()
                    print("Rozpoznano osobe ")

                    """client = sending_message()
                    client.connect()
                    client.sendto(name)
                    client.close()"""
                else:
                  print("NO match")
                  cv2.putText(img, 'No Match', (x+2,y+h-5), cv2.FONT_HERSHEY_SIMPLEX, 1, (0,0,255),2)

              cv2.imshow('Face Recognizer',img)
              k = cv2.waitKey(30) & 0xff
              if k == 27:
                break

    def close(self):
        self.cap.release()
        cv2.destroyAllWindows()

f = face_recog()
f.main_loop()


import json,os,time

path='E:\PycharmProjects\legal_tools\legal_tools\lawlib_tool\law'
files= os.listdir(path)
dict1=''
dictMerged=dict(dict1)
all1=''
for file in files:
    data= {
        'title':file.split('.')[0],
        'status':'exsist',
        'time_tag':time.time(),
        'email_status':''
    }

import sqlite3

# conn =sqlite3.connect('law.db')
# cursor =conn.cursor()
# cursor.execute('create table user(id varchar(20)primary key,name varchar(20))')
# cursor.execute('insert into user(id,name) values(\'1\', \'Michael\')')
# print(cursor.rowcount)
# cursor.close()
# conn.commit()
# conn.close()

conn =sqlite3.connect('law.db')
cursor=conn.cursor()
cursor.execute('select * from user where id=?',('1',))
values=cursor.fetchall()
print(values)
cursor.close()
conn.close()


'''
This script is used to gather information from nik's webiste:
'''
import requests
from bs4 import BeautifulSoup
import xlsxwriter as wx
import sys
import xlrd, xlwt
from datetime import date,datetime
import pandas as pd

def web_info():
    response = requests.get('http://thales.cheme.cmu.edu/dfo/comparison/comp.html')
    soup = BeautifulSoup(response.content, 'lxml')

    title = []
    sol = []
    for index, x in enumerate(soup.find_all('tr')):
        text = x.get_text().split()
        if index == 0:
            title = text[:]
            title.insert(0,"index")
        else:
            sol.append(text)
    return title,sol

def set_style(name,height,bold=False):
	style = xlwt.XFStyle()
	font = xlwt.Font()
	font.name = name
	font.bold = bold
	font.color_index = 4
	font.height = height
	style.font = font
	return style

def write_excel(title,sol):
	f = xlwt.Workbook()
	sheet1 = f.add_sheet('models',cell_overwrite_ok=True)
	row0 = title
	#写第一行
	for i in range(0,len(row0)):
		sheet1.write(0,i,row0[i],set_style('Times New Roman',220,True))
	for index,row in enumerate(sol):
		for i in range(0,len(row)):
			sheet1.write(index+1,i,row[i])
	f.save('ModelList.xls')

def read_excel(smoothness,convexity,variables):
	file = 'ModelList.xlsx'
	sheet = pd.read_excel(file,sheet_name='models',header=0,index_col=1,usecols="A:H")
	filter_sol = sheet[(sheet['smoothness']==smoothness)&(sheet['convexity']==convexity)&(sheet['variables']<variables)]
	return filter_sol

if __name__ == "__main__":
    # title, sol = web_info()
    # write_excel(title,sol)
	sol = read_excel('smooth','convex',10)


from stock import *

keyI = 1

def extractT(tupleT):
	return tupleT[priceI];

#[(date, key)]
class Trend:
	def __init__(self, name, count, tuples):
		self.name = name
		self.count = count
		self.tuples = tuples

	def merge(self, trend):
		name = self.name + " + " + trend.name
		count = self.count + trend.count
		n = len(self.tuples)
		m = len(trend.tuples)
		ptrA = 0
		ptrB = 0
		tuples = []
		while ptrA < n and ptrB < m:
			tupleA = self.tuples[ptrA]
			tupleB = trend.tuples[ptrB]
			if tupleA[dateI] == tupleB[dateI]:
				# merge function HERE!
				key = (self.count * tupleA[keyI] + trend.count * tupleB[keyI]) / float(count);
				tuples.append((tupleA[dateI], key))
				ptrA += 1
				ptrB += 1
			elif tupleA[dateI] < tupleB[dateI]:
				ptrA += 1
			else:
				ptrB += 1
		return Trend(name, count, tuples)

def fromStock(stock):
	tuples = []
	for x in stock.tuples:
		tuples.append((x[dateI], extractT(x)))
	return Trend(stock.name, 1, tuples)

# Trend --> Trend --> [(date, key, key)]
def commonDate(TrendA, TrendB):
	n = len(TrendA.tuples)
	m = len(TrendB.tuples)
	ptrA = 0
	ptrB = 0
	ret = []
	while ptrA < n and ptrB < m:
		tupleA = TrendA.tuples[ptrA]
		tupleB = TrendB.tuples[ptrB]
		if tupleA[dateI] == tupleB[dateI]:
			ret.append((tupleA[dateI], tupleA[keyI], tupleB[keyI]))
			ptrA += 1
			ptrB += 1
		elif tupleA[dateI] < tupleB[dateI]:
			ptrA += 1
		else:
			ptrB += 1
	return ret

# [(date, key, key)] --> True / False
def checkTrend(lst, correlationF, threshold):
	if not lst:
		return False
	count = 0
	for i in range(1, len(lst)):
		deltaA = lst[i][1] - lst[i - 1][1]
		deltaB = lst[i][2] - lst[i - 1][2]
		if correlationF(deltaA, deltaB, 0):
			count += 1
	percent = count * 100.0 / len(lst)
	return (percent >= threshold)


"""
@author: "Dickson Owuor"
@copyright: "Copyright (c) 2018 Université de Montpellier"
@credits: "Anne Laurent, Joseph Orero"
@license: "MIT"
@version: "1.0"
@email: "owuordickson@gmail.com"

This code fetched user preferences and executes algorithm and
returns results to the user

"""

from PyCharm.algorithm.DataTransform import DataTransform
from PyCharm.algorithm.T_GRAANK import *

def algorithm_init(filename,ref_item,minsup,minrep):
    try:
        # 1. Load dataset into program
        dataset = DataTransform(filename)
        #print(dataset)

        # 2. Get maximum transformation step
        max_step = dataset.get_max_step(minrep)
        #print("Transformation Step (max): "+str(step))

        # TRANSFORM DATA (for each step)
        patterns = 0
        for s in range(max_step):
            step = s+1 # because for-loop is not inclusive from range: 0 - max_step
            # 3. Calculate representativity
            chk_rep,rep_info = dataset.get_representativity(step)
            #print(rep_info)

            if chk_rep:
                # 4. Transform data
                data,time_diffs = dataset.transform_data(ref_item, step)
                #print(data)

                # 5. Execute GRAANK for each transformation
                title, D1, S1, T1 = Graank(Trad(list(data)), minsup, time_diffs, eq=False)

                pattern_found = check_for_pattern(ref_item,D1)
                if pattern_found == True:
                    print(rep_info)
                    for line in title:
                        print(line)
                    print('Pattern : Support')
                    for i in range(len(D1)):
                        # D is the Gradual Patterns, S is the support for D and T is time lag
                        if (str(ref_item+1)+'+' in D1[i]) or (str(ref_item+1)+'-' in D1[i]):
                            # select only relevant patterns w.r.t *reference item
                            print(str(D1[i]) + ' : ' + str(S1[i]) + ' | ' + str(T1[i]))
                            patterns = patterns + 1
                    print("---------------------------------------------------------")

        if patterns == 0:
            print("Oops! no relevant pattern was found")
            print("---------------------------------------------------------")

    except Exception as error:
        print(error)

def check_for_pattern(ref_item, R):
    pr = 0
    for i in range(len(R)):
        # D is the Gradual Patterns, S is the support for D and T is time lag
        if (str(ref_item + 1) + '+' in R[i]) or (str(ref_item + 1) + '-' in R[i]):
            # select only relevant patterns w.r.t *reference item
            pr = pr + 1
    if pr > 0:
        return True
    else:
        return False

def main(filename,ref_item,minsup,minrep):
    algorithm_init(filename,ref_item,minsup,minrep)

import time
start_time = time.time()
#main("data/test.csv",0,0.1,0.5)
main("data/rain_temp2013-2015_full.csv",0,0.5,0.5)
#main("data/ndvi_test.csv",0,0.5,0.8)
#main("data/ndvi_kenya.csv", 2, 0.5, 0.8)
#main("data/ndvi_towns_full.csv", 0, 0.5, 0.5)
print("--- %s seconds ---" % (time.time() - start_time))


#!/usr/bin/python3.3

import os
import time
from sys import *
import socket
from getopt import *

action = []
user = os.environ["USER"]

def     makeRequest(action, user, *param):
    req = '&&'
    req += str(action)
    req += "::"
    req += str(user)
    for p in param:
        req += '::' + p
    req += '&&'
    return (str(req))

def     sendRequest(conn, req):
    conn.send(bytes(str(req), "ascii"))
def     usage():
    print("Client usage:")
    print("-c || --correction '/path/your_python_file.py' == corrige l'exercice")
    print("-t || --list-tips == affiche la liste des astuces disponible")
    print("-b || --buy-tips id_of_tips == achete une astuce")
    print("-s || --score == affiche votre score")
    print("-r || --credit == affiche votre total de credit")
    print("-a || --achievement == affiche vos HF")
    print("-h || --history == affiche votre historoque")
    print("-m || --my-tips == affiche la liste de vos tips")
    print("-g || --get-subject == affiche le sujet de l'exo en cours")
    print("-A || --add LANGAGE == ajoute LANGAGE a votre liste de langage")
    print("-S || --select LANGAGE == selectionne LANGAGE")
    print("-L || --list == liste les LANGAGE en cours")
    print("-M || --my-level == affiche votre niveau en cours")
    print("-u || --current-lang == affiche le LANGAGE selectionne")
    print("-G || --get-tips == recupere un tips")
    print("-i || --list-lesson == affiche les cours")

def     main():
    conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    try:
        opts, args = getopt(argv[1:], "c:tb:srahmgl:A:S:LMGui", ["correction=", "list-tips", "buy-tips=", "score", "credit", "achievement", "history", "my-tips", "get-subject", "language=", "add=", "select=", "my-level=", "get-tips=", "current-lang", "list-lesson"])
    except:
        print("option error.")
        exit(1)
    if (len(opts) == 0):
        usage()
        exit(0)

#    try:
    conn.connect(("10.19.252.191", 8582))
#    except:
#        print("Error connection.")
#        exit(1)
    index = 0
    argument = list()
    conn.settimeout(3)
    for o, a in opts:
        if o in ("-c", "--correction"):
            index = 16
            print("Get test...")
            sendRequest(conn, makeRequest(index, user, *argument))
            ret = ""
            print("Testing...")
            index = 1
            ret = ""
            try:
                try:
                    try:
                        msg = conn.recv(4096)
                    except:
                        msg = ""
                    print(msg)
                    try:
                        s = (msg.decode("ascii")).split("::")
                    except:
                        s = [""]
                    cmd = ""
                    print(s)
                    for t in s:
                        cmd = a + " " + t
                        print(cmd)
                        ret += os.popen(cmd, 'r').read()
                        print("-----")
                    ret = ret.replace("\n", "??")
                    print(ret)
                except:
                    ret = ""
                try:
                    conn.close()
                except:
                    print("Error closing connection")
            except:
                print("error...")
                ret = ""
            conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            conn.connect(("10.19.252.191", 8582))
            conn.settimeout(3)
            argument.append(ret)
        elif o in ("-t", "--list-tips"):
            index = 7
        elif o in ("-S", "--select"):
            index = 2
            argument.append(a)
        elif o in ("-A", "--add"):
            index = 11
            argument.append(a)
        elif o in ("-L", "--list"):
            index = 3
        elif o in ("-u", "--current-lang"):
            index = 12
        elif o in ("-m", "--my-tips"):
            index = 6
        elif o in ("-s", "--score"):
            index = 5
        elif o in ("-M", "--my-level"):
            index = 4
        elif o in ("-G", "--get-tips"):
            index = 8
            argument.append(a)
        elif o in ("-r", "--credit"):
            index = 9
        elif o in ("-g", "--get-subject"):
            index = 13
        elif o in ("-i", "--list-lesson"):
            index = 14
        elif o in ("-a", "--achievement"):
            index = -1
        elif o in ("-b", "--buy-tips"):
            index = 10
            try:
                argument.append(a)
            except:
                print("Invalide tips")
                conn.close()
                exit(0)
        elif o in ("-l", "--language"):
            index = 2
            argument.append(a)
    print("sending...")
    sendRequest(conn, makeRequest(index, user, *argument))
    try:
        msg = conn.recv(4096)
        print(msg.decode("ascii"))
    except:
        print("end")
    return (0)

###########
main()


###############################################
## Use Brutal Force!
## Can also use Divid and conquer
## - see 1030-2.py [better than this algorithm]
###############################################

def getNumber(s):
    if s.isdigit():
        return ord(s) - ord('0')
    return ord(s) - ord('A') + 10

if __name__ == "__main__":

    s = [x for x in input()]
    #print(s)
    res = 0
    for i in range(len(s)):
        res = res*36 + getNumber( s[i] )

    print(res)


#!/usr/bin/env python
# -*- coding:utf-8 -*-
# @Author: Niccolò Bonacchi
# @Date: Friday, February 8th 2019, 12:51:51 pm
import datetime
import json
from pathlib import Path

import numpy as np
from ibllib.io import raw_data_loaders as raw

def get_port_events(events: dict, name: str = '') -> list:
    out: list = []
    for k in events:
        if name in k:
            out.extend(events[k])
    out = sorted(out)

    return out

def update_buffer(buffer: list, val) -> list:
    buffer = np.roll(buffer, -1, axis=0)
    buffer[-1] = val
    return buffer.tolist()

def texp(factor: float = 0.35, min_: float = 0.2, max_: float = 0.5) -> float:
    """Truncated exponential
    mean = 0.35
    min = 0.2
    max = 0.5
    """
    x = np.random.exponential(factor)
    if min_ <= x <= max_:
        return x
    else:
        return texp(factor=factor, min_=min_, max_=max_)

def get_biased_probs(n: int, idx: int = -1, prob: float = 0.5) -> list:
    """
    get_biased_probs [summary]

    Calculate the biased probability for all elements of an array so that
    the <idx> value has <prob> probability of being drawn in respect to the
    remaining values.
    https://github.com/int-brain-lab/iblrig/issues/74
    For prob == 0.5
    p = [2 / (2 * len(contrast_set) - 1) for x in contrast_set]
    p[-1] *= 1 / 2
    For arbitrary probs
    p = [1/(n-1 + 0.5)] * (n - 1)

    e.g. get_biased_probs(3, idx=-1, prob=0.5)
    >>> [0.4, 0.4, 0.2]

    :param n: The length of the array, i.e. the num of probas to generate
    :type n: int
    :param idx: The index of the value that has the biased probability,
                defaults to -1
    :type idx: int, optional
    :param prob: The probability of the idxth value relative top the rest,
                 defaults to 0.5
    :type prob: float, optional
    :return: List of biased probabilities
    :rtype: list

    """
    n_1 = n - 1
    z = n_1 + prob
    p = [1 / z] * (n_1 + 1)
    p[idx] *= prob
    return p

def draw_contrast(contrast_set: list,
                  prob_type: str = 'biased',
                  idx: int = -1, idx_prob: float = 0.5) -> float:
    if prob_type == 'biased':
        p = get_biased_probs(len(contrast_set), idx=idx, prob=idx_prob)
        return np.random.choice(contrast_set, p=p)
    elif prob_type == 'uniform':
        return np.random.choice(contrast_set)

def check_stop_criterions(init_datetime, rt_buffer, trial_num) -> int:
    # STOPPING CRITERIONS
    # < than 400 trials in 45 minutes
    time_up = init_datetime + datetime.timedelta(minutes=45)
    if time_up <= datetime.datetime.now() and trial_num <= 400:
        return 1

    # Median response time of latest N = 20 trials > than 5 times
    # the median response time and more than 400 trials performed
    N, T = 20, 400
    if len(rt_buffer) >= N and trial_num > T:
        latest_median = np.median(rt_buffer[-N:])
        all_median = np.median(rt_buffer)

        if latest_median > all_median * 5:
            return 2

    end_time = init_datetime + datetime.timedelta(minutes=90)
    if end_time <= datetime.datetime.now():
        return 3

    return False

def get_trial_rt(behavior_data: dict) -> float:
    return (behavior_data['States timestamps']['closed_loop'][0][1] -
            behavior_data['States timestamps']['stim_on'][0][0])

def create_flags(data_file_path: str, poop_count: bool) -> None:
    flag = Path(data_file_path).parent.parent / 'transfer_me.flag'
    open(flag, 'a').close()
    flag2 = Path(data_file_path).parent.parent / 'create_me.flag'
    open(flag2, 'a').close()
    flag3 = Path(data_file_path).parent.parent / 'poop_count.flag'
    if poop_count:
        open(flag3, 'a').close()

def draw_session_order():
    first = list(range(0, 4))
    second = list(range(4, 8))
    third = list(range(8, 12))
    for x in [first, second, third]:
        np.random.shuffle(x)
    first.extend(second)
    first.extend(third)

    return first

def patch_settings_file(sess_or_file: str, patch: dict) -> None:
    sess_or_file = Path(sess_or_file)
    if sess_or_file.is_file() and sess_or_file.name.endswith('_iblrig_taskSettings.raw.json'):  # noqa
        session = sess_or_file.parent.parent
        file = sess_or_file
    elif sess_or_file.is_dir() and sess_or_file.name.isdecimal():
        file = sess_or_file / 'raw_behavior_data' / '_iblrig_taskSettings.raw.json'  # noqa
        session = sess_or_file
    else:
        print('not a settings file or a session folder')
        return

    settings = raw.load_settings(session)
    settings.update(patch)
    with open(file, 'w') as f:
        f.write(json.dumps(settings, indent=1))
        f.write('\n')

    return

if __name__ == "__main__":
    get_biased_probs(4)
    print(draw_contrast([1, 2, 3]))
    print(draw_contrast([1, 2, 3, 4, 5]))
    print(draw_contrast([1, 2, 3, 4, 5, 6, 7]))
    print(draw_contrast([1, 2, 3], prob=0.3, idx=0))
    print(draw_contrast([1, 2, 3, 4, 5], prob=0.5, idx=0))
    print(draw_contrast([1, 2, 3, 4, 5, 6, 7], prob=0.3, idx=-1))


# Time:  O(n^2)
# Space: O(1)
#
# Given an array S of n integers,
# are there elements a, b, c in S such that a + b + c = 0?
# Find all unique triplets in the array which gives the sum of zero.
#
# Note:
# Elements in a triplet (a,b,c) must be in non-descending order. (ie, a <= b <= c)
# The solution set must not contain duplicate triplets.
#    For example, given array S = {-1 0 1 2 -1 -4},
#
#    A solution set is:
#    (-1, 0, 1)
#    (-1, -1, 2)

class Solution(object):
    def threeSum(self,nums):
        """
        :type nums: List[int]
        :type List[List[int]]
        """

        nums, result, i = sorted(nums), [], 0
        while i < len(nums) - 2:
            #when it's the start point or the first part of continous values are not the same (otherwise jump one index of the start values)
            if i == 0 or nums[i] != nums[i-1]:
                j, k = i+1, len(nums)-1
                while j < k:
                    if nums[i] + nums[j] + nums[k] < 0:
                        j +=1
                    elif nums[i] + nums[j] +nums[k] > 0:
                        k -=1
                    else:
                        #when the sum of triplet = 0
                        result.append([nums[i], nums[j], nums[k]])
                        j, k = j + 1, k - 1
                        #when the continuous values are the same , do extra one jump
                        while j < k and nums[j] == nums[j-1]:
                            j += 1
                        #when the continuous values are the same, do extra one jump
                        while j < k and nums[k] == nums[k+1]:
                            k -= 1
            i += 1
            return result

    def threeSum_method3(self, nums):
        #add nums to new_perm , track the sum,  number <= 3
        res = []
        nums.sort()
        for i in xrange(len(nums)-2):
            #if the nums[i] == nums[i-1], force to start the next round with [i+1]
            if i >0 and nums[i] == nums[i-1]:
                continue
            #two extra pointers in the process
            l, r = i+1, len(nums)-1
            while l < r:
                s = nums[i] + nums[l] + nums[r]

                if s > 0:
                    r -= 1
                elif s < 0:
                    l += 1
                else:
                    res.append([nums[i], nums[l], nums[r]])
                    #if duplicate move one step more to avoid duplicate list in result
                    #it's a while auto loop . until can't meet condition then exit
                    while l < r and nums[l] == nums[l+1]:
                        l +=1
                    while l < r and nums[r] == nums[r-1]:
                        r -=1

                    l += 1
                    r -= 1

        return res


# program to tell types of all characters from a string

def fun(a):
    for i in range(len(a)):
        # using ascii value of characters to check type of character
        if (ord(a[i])>=ord('a') and ord(a[i])<=ord('z')) or (ord(a[i])>=ord('A') and ord(a[i])<=ord('Z')):
            print(a[i],' is a character')

        elif (ord(a[i])>=ord('0') and ord(a[i])<=ord('9')):
            print(a[i],' is a number')

        elif ord(a[i])>20:
            print(a[i],' is a special character.')

# taking input from user as a string.
s = input("Enter a string with all kind of characters : ")
a = list(s)
fun(a)


# 백준 1011번 Fly me to the Alpha Centauri
# 골드 5

N = int(input())

def pyramid(n):
    answer = 0
    if n ==0:
        return 0
    for i in range(n-1):
        answer += 2*(i+1)
    answer += n
    return answer

def isnotzero(n):
    if n == 0:
        return 0
    else:
        return 1

for _ in range(N):
    x, y = map(int, input().split())
    y = y - x

    mx = int(pow(y, 0.5))
    tmp = y - pow(mx, 2)
    if tmp >= mx:
        print(2*mx+ isnotzero(tmp-mx))
    else:
        print(2*mx -1 + isnotzero(tmp))


# -*- coding: utf-8 -*-
"""
创建时间 Wed Jan 23 10:57:12 2019
作者:PM.liugang
review:190307
描述:
1.确定k大小和距离度量
2.对于测试集中的一个样本，找到训练集中和它最近的k个样本
3.将这k个样本的投票结果作为测试样本的类别
通常欧氏距离用于实数域的数据集，此时一定要对特征进行标准化，每一维度特征的重要性等同
遗留：
"""
import sys
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

from sklearn.neighbors import KNeighborsClassifier
from sklearn import datasets
from sklearn.cross_validation import train_test_split
# from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

sys.path.append('C:\\Users\\Administrator\\Desktop\\myself')
from plotClassifierRegions import plot_decision_regions

'''搭建模型'''
knn = KNeighborsClassifier(n_neighbors=5, # 最近的5个样本 default n_neighbors=5
                           p=2, # default p=2退化为欧氏距离 p=1 退化为曼哈顿距离
                           metric='minkowski') # 距离度量方式 default minkowski

def showModel(trainData, testData, y_train, y_test, classifier=None):
    """
    可视化模型

    Parameters
    ----------
    trainData : np.array
        训练数据
    testData : np.array
        测试数据
    y_train : np.array
        训练数据标签
    y_test : np.array
        测试数据集标签
    classifier :
        分类器

    Returns
    -------

    """
    sc = StandardScaler()
    sc.fit(trainData)
    trainDataDf = pd.DataFrame(trainData)
    print('{:*^35}'.format('原始数据概况'))
    print(trainDataDf.describe())
    X_train_std = sc.transform(trainData)
    print('{:*^35}'.format('标准化后数据概况'))
    X_train_std_df = pd.DataFrame(X_train_std)
    print(X_train_std_df.describe())
    X_test_std = sc.transform(testData)
    X_combined_std = np.vstack((X_train_std, X_test_std))
    y_combined = np.hstack((y_train, y_test))

    knn.fit(X_train_std,y_train)

    plot_decision_regions(X_combined_std,y_combined,
                      classifier=knn,
                      test_idx=range(105,150))
    plt.xlabel('Petal Length [Standardized]')
    plt.ylabel('Petal Width [Standardized]')
    plt.show()

if __name__ == "__main__":
    iris = datasets.load_iris()
    X = iris.data[:,[2,3]]
    y = iris.target
    print('数据分为%s类,分别是' % len(np.unique(y)))
    X_train, X_test, y_train, y_test = train_test_split(X,y,
                                                      test_size=0.3,
                                                      random_state=0)
    showModel(X_train, X_test, y_train, y_test, knn)


import pandas as pd
import numpy as np
from src.data.data_manager import DataManager

def split_data(raw_data_path, path_to_save, split_parts=100):
    """
    Args:
        raw_data_path: path to the original csv data
        path_to_save: path to .hdf5 data
        split_parts: number parts to split
    """

    saver = DataManager(path_to_save)

    TIMESTEPS_IN_ALL = 629145480
    counter = 0
    for chunk in pd.read_csv(raw_data_path, chunksize=TIMESTEPS_IN_ALL // split_parts):
        data: np.ndarray = chunk.values
        signal = (data[:, 0]).astype('int16')
        time_left = (data[:, 1]).astype('float32')

        name_group = "0"*(4-len(str(counter))) + str(counter)
        saver.push(name_group + "/signal", signal, 'int16')
        saver.push(name_group + "/time_left", time_left, 'float32')

        counter += 1
        print("{} / {} blocks were processed".format(counter, split_parts))
        if counter == 100:
            break

if __name__ == "__main__":
    ORIGINAL_DATA_PATH = "../../data/raw/train.csv"
    SPLITTED_DATA_PATH = "../../data/interim/train.hdf5"

    split_data(ORIGINAL_DATA_PATH, SPLITTED_DATA_PATH)


#!/usr/bin/env python3
"""
  FileName : FileParser.py
  Version  : v0.0.0.1
  Details  : Script connects to predefined AWS S3 Bucket and reads file content. Script Supported File Type : PDF, DOC, DOCX.
"""
import configparser
import os
import logging
import boto3
import docx2txt
import PyPDF2
import pdfminer
from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter
from pdfminer.converter import TextConverter
from pdfminer.layout import LAParams
from pdfminer.pdfpage import PDFPage
from io import StringIO
from io import BytesIO

#
print('File Parsing Program : Executing')

logging.basicConfig(level = logging.INFO, filemode = 'w', filename = 'FileParser')

os.chdir('/fileparser')
print('Home Path :' + os.environ['HOME'])
#os.chdir(os.environ['HOME'])
Working_Dir=os.environ['HOME']
print('Working Directory :' + Working_Dir + '\n')
logging.info('Working Directory :' + Working_Dir + '\n')

config = configparser.ConfigParser()
config.read(Working_Dir+'/AWS_CONFIG.cfg')
os.environ["AWS_ACCESS_KEY_ID"] = config.get("AWS_CREDENTIALS", "AWS_ACCESS_KEY_ID")
os.environ["AWS_SECRET_ACCESS_KEY"] = config.get("AWS_CREDENTIALS", "AWS_SECRET_ACCESS_KEY")
bucket_name = config.get("S3", "bucket_name")
s3_bucket = config.get("S3", "s3_bucket")
bucket_name="fileparser"
"""
  Declare list for various file format
"""
pdf_file_list = []
doc_file_list = []
audio_video_file_list = []
powerpoint_file_list = []
other_format_file_list = []
unavailable_format_file_list = []
content_not_readable_file_list = []

"""
  Declare  variables
"""
filecount=0
parsed_filecount=0
other_filecount=0
#
print("S3 Bucket Name : "+ bucket_name)

os.chdir('/fileparser')
Working_Dir=os.getcwd()
print('Working Directory :' + Working_Dir + '\n')
logging.info('Working Directory :' + Working_Dir + '\n')

"""
  Read files available in S3 Bucket hosted via mount point.
"""
for root,directory_names,file_name_list in os.walk(os.getcwd()):
  for filename in file_name_list:
    print("\n##########################\nProcessing Record : " + filename)
    logging.info("\n##########################\nProcessing Record : " + filename)
    filecount=filecount+1
    FullFilePath=os.path.join(root, filename)

    if os.path.isfile(FullFilePath):
      if filename.find('.pdf') > 0:
       print("File Type : PDF")
       logging.info("File Type : PDF")

       pdfFileObj = open(FullFilePath,'rb')
       pdfReader = PyPDF2.PdfFileReader(pdfFileObj)
       #final_file_totalpages=pdfReader.numPagesreading
       #print(final_file_totalpages)
       pageObj = pdfReader.getPage(0)
       FileContent=pageObj.extractText()

       filesize=os.path.getsize(FullFilePath)
       filesize=str(filesize/1024/1024)

       pdfFileObj.close()

       if(FileContent.strip()==''):
        print('Retry With PDFMiner')
        logging.info('Retry With PDFMiner')

        pagenums = set()
        output = StringIO()
        manager = PDFResourceManager()
        converter = TextConverter(manager, output, laparams=LAParams())
        interpreter = PDFPageInterpreter(manager, converter)

        infile = open(FullFilePath, 'rb') #open the file for read in binary mode

        for page in PDFPage.get_pages(infile, pagenums):     #iterate with the pdf interpreter through the pdf pages
         interpreter.process_page(page)

        #Close the files and converters
        infile.close()
        converter.close()
        FileContent= output.getvalue()
        output.close()

       """
      	Display Processed Details
       """

       if(FileContent.strip()==''):
        content_not_readable_file_list.append(FullFilePath)
        print("Result : Unable to Read File Content")
        logging.info("Result : Unable to Read File Content")
       else:
        remove_line_breaks = FileContent.replace("\n", "")
        remove_line_breaks = remove_line_breaks.replace("\t", "")
        final_file_content = remove_line_breaks.replace("  ", "")
        pdf_file_list.append(filename)
        parsed_filecount=parsed_filecount+1
        print("File Name : "+ filename)
        print("File Word Count : "+ str(len(final_file_content.strip())))
        print("File Size : "+ filesize +" MB")
        print("File Full Path and Name : "+ FullFilePath)
        print("File Content : \n"+ final_file_content)
        logging.info("File Name : "+ filename)
        logging.info("File Word Count : "+ str(len(final_file_content.strip())))
        logging.info("File Size : "+ filesize +" MB")
        logging.info("File Full Path and Name : "+ FullFilePath)
        logging.info("File Content : \n"+ final_file_content)

      elif filename.find('.doc') > 0:
       print("File Type : DOC")
       logging.info("File Type : DOC")

       FileContent = docx2txt.process(FullFilePath)
       filesize=os.path.getsize(FullFilePath)
       filesize=str(filesize/1024/1024)

       """
        Display Processed Details
       """
       if(FileContent.strip()==''):
        content_not_readable_file_list.append(FullFilePath)
        print("Result : Unable to Read File Content")
        logging.info("Result : Unable to Read File Content")
       else:
        remove_line_breaks = FileContent.replace("\n", " ")
        remove_line_breaks = remove_line_breaks.replace("\t", " ")
        final_file_content = remove_line_breaks.replace("  ", " ")
        doc_file_list.append(filename)
        parsed_filecount=parsed_filecount+1
        print("File Name : "+ filename)
        print("File Word Count : "+ str(len(final_file_content.strip())))
        print("File Size : "+ filesize +" MB")
        print("File Full Path and Name : "+ FullFilePath)
        print("File Content : \n"+ final_file_content)
        logging.info("File Name : "+ filename)
        logging.info("File Word Count : "+ str(len(final_file_content.strip())))
        logging.info("File Size : "+ filesize +" MB")
        logging.info("File Full Path and Name : "+ FullFilePath)
        logging.info("File Content : \n"+ final_file_content)

      else:
       print("Result : File Type Not Supported | File with location :",FullFilePath,"\n")
       other_format_file_list.append(os.path.join(root, filename))
       other_filecount=other_filecount+1
    else:
     unavailable_format_file_list.append(os.path.join(root, filename))
     unavailable_filecount=unavailable_filecount+1

"""
  Display Processing Summary
"""
print("#$#$#$#$#$#$#$#$#$#$#$#$#$#$#$#$#$#$#")
print("#         Processing Summary        #")
print("#$#$#$#$#$#$#$#$#$#$#$#$#$#$#$#$#$#$#")
print('\nTotal File Count : ' + str(filecount))
print('Total File Count Processed : ' + str(parsed_filecount))
print('Total File Count NOT Processed : ' + str(len(content_not_readable_file_list)))
print('All Other File Count Identified : ' + str(other_filecount))
logging.info('Total File Count : ' + str(filecount))
logging.info('Total File Count Processed : ' + str(parsed_filecount))
logging.info('Total File Count NOT Processed : ' + str(len(content_not_readable_file_list)))
logging.info('All Other File Count Identified : ' + str(other_filecount))

pdf_total_file=str(len(pdf_file_list))
print('\nPDF Total File Count Processed : ' + pdf_total_file)
print('File List :')
print(pdf_file_list)
logging.info('PDF Total File Count Processed : ' + pdf_total_file)
logging.info('File List :')
logging.info(pdf_file_list)

doc_total_file=str(len(doc_file_list))
print('\nDOC Total File Count : ' + doc_total_file)
print('File List :')
print(doc_file_list)
logging.info('DOC Total File Count : ' + doc_total_file)
logging.info('File List :')
logging.info(doc_file_list)

other_total_file=str(len(other_format_file_list))
print('\nTotal Other File Count Ignored : ' + other_total_file)
print('File List :')
print(other_format_file_list)
logging.info('Total Other File Count Ignored : ' + other_total_file)
logging.info('File List :')
logging.info(other_format_file_list)

print("\nContent Not Readable File List :")
print(content_not_readable_file_list)
logging.info("Content Not Readble File List :")
logging.info(content_not_readable_file_list)

print('\n File Parsing Program : Completed')
logging.info('File Parsing Program : Completed')


def conv(n):
    if n < 0:
        return ""
    elif n == 0:
        return ""
    elif n <= 19:
        numero = ("uno", "due", "tre", "quattro", "cinque", "sei", "sette", "otto","nove", "dieci","undici","dodici","tredici","quattordici","quindici","sedici","diciassette","diciotto","diciannove")
        return numero[n-1]
    elif n <= 99:
        decina = ("venti", "trenta", "quaranta","cinquanta","sessanta","settanta","ottanta","novanta")
        decinaInLettere = decina[int(n/10)-2]
        if( n%10 == 1 or n%10 == 8):
            return decinaInLettere[:-1] + conv(n%10)
        else:
            return decinaInLettere + conv(n%10)
    elif n <= 999:
        centinaia = ("cento", "duecento","trecento","quattrocento","cinquecento","seicento","settecento","ottocento","novecento")
        centinaiainlettere = centinaia[int(n/100)-1]
        if(n%100 > 79 and  n%100 < 90):
            return centinaiainlettere[:-1] + conv(int(n%100))
        elif( n%100 == 0):
            return centinaiainlettere
        else:
            return centinaiainlettere + conv(int(n%100))
    elif n <= 1999:
        mille = "mille"
        return mille + conv(int(n%1000))
    elif n <= 999999:
        mila = "mila"
        return conv(int(n/1000)) + mila + conv(int(n%1000))
    elif n <= 1999999:
        milione = "unmilione"
        return milione + conv(n%1000000)
    elif n <= 999999999:
        milioni = "milioni"
        return conv(int(n/1000000)) + milioni + conv(int(n%1000000))
    elif n <= 1999999999:
        miliardo = "unmiliardo"
        return miliardo + conv(int(n%1000000000))
    elif n <= 999999999999:
        miliardi = "miliardi"
        return conv(int(n/1000000000)) + miliardi + conv(int(n%1000000000))


# -*- coding: utf-8 -*-

# import libraries
from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
import datetime
import time
import shutil
import os

############### LOGIN #############################

def log_me_in(login_name, login_pass):

    driver.get('https://blogs.fangraphs.com/wp-login.php?redirect_to=https%3a%2f%2fwww.fangraphs.com%2findex.aspx')
    driver.find_element_by_id("user_login").send_keys(login_name)
    time.sleep(2)
    driver.find_element_by_id("user_pass").send_keys(login_pass)
    time.sleep(2)
    driver.find_element_by_id("wp-submit").click()
    print("Logging in, please wait.")
    time.sleep(5)
    print("Resume.")
    get_split_page()

############## SPLIT PAGE #########################

def get_split_page():

    time.sleep(5)
    driver.get('https://www.fangraphs.com/leaders/splits-leaderboards?splitArr=&splitArrPitch=&position=B&autoPt=true&splitTeams=false&statType=player&statgroup=1&startDate=2019-03-01&endDate=2019-11-01&players=&filter=')
    print("Getting split page.")
    time.sleep(5)
    print("Resumed.")
    get_per_game_view()

############## HOVER & CLICK #########################

def get_per_game_view():

    element = driver.find_element_by_xpath('//*[@id="react-drop-test"]/div[1]/div[1]/div[10]/div')
    hover = ActionChains(driver).move_to_element(element)
    hover.perform()

    print("Hover menu changing.")
    time.sleep(2)
    print("Resumed")

    # can expand to feature all possible options such as by week/month/year by changing below to li[4]/li[3]/li[2]

    driver.find_element_by_xpath('//*[@id="react-drop-test"]/div[1]/div[1]/div[10]/ul/li[5]').click()
    time.sleep(1)

def player_type_decision(player_type,game_type):

    if player_type == 'b':

        driver.find_element_by_xpath('//*[@id="react-drop-test"]/div[1]/div[1]/div[6]').click()

        time.sleep(1)

        if game_type == 'h':
            driver.find_element_by_xpath('//*[@id="react-drop-test"]/div[1]/div[3]/div/div[1]/div[2]/div[1]').click()
            time.sleep(1)
        elif game_type == 'a':
            driver.find_element_by_xpath('//*[@id="react-drop-test"]/div[1]/div[3]/div/div[1]/div[2]/div[2]').click()
            time.sleep(1)

    elif player_type == 'p':

        driver.find_element_by_xpath('//*[@id="react-drop-test"]/div[1]/div[1]/div[7]').click()
        time.sleep(1)

        if game_type == 'h':
            driver.find_element_by_xpath('//*[@id="react-drop-test"]/div[1]/div[3]/div/div[2]/div[2]/div[1]').click()
            time.sleep(1)
        elif game_type == 'a':
            driver.find_element_by_xpath('//*[@id="react-drop-test"]/div[1]/div[3]/div/div[2]/div[2]/div[2]').click()
            time.sleep(1)

def get_advanced_view():

    driver.find_element_by_xpath('//*[@id="root-buttons-stats"]/div[2]').click()
    print("Getting advanced view.")
    time.sleep(15)

############# GET DATES AND KEEP UPDATING UNTIL YEAR THRESHOLD ################

def date_updater(start_year, end_year, view, folder):

    start_date = datetime.date(start_year, 3, 15)

    if view == 'a':
        get_advanced_view()

    while start_year <= end_year:

        i = 0

        #if(end_year==start_year):
        #    i = 3

        while i < 4:

            end_date = start_date + datetime.timedelta((2*365/12)+1)

            ############## START DATE #######################

            driver.find_element_by_xpath('//*[@id="root-menu-time-filter"]/div[2]/div[2]/div[1]/div[1]/input').clear()

            driver.find_element_by_xpath('//*[@id="root-menu-time-filter"]/div[2]/div[2]/div[1]/div[1]/input').send_keys(start_date.strftime("%m-%d-%Y"))

            ############## END DATE #########################

            time.sleep(1)

            driver.find_element_by_xpath('//*[@id="root-menu-time-filter"]/div[2]/div[2]/div[1]/div[3]/input').clear()

            driver.find_element_by_xpath('//*[@id="root-menu-time-filter"]/div[2]/div[2]/div[1]/div[3]/input').send_keys(end_date.strftime("%m-%d-%Y"))

            ################ UPDATE #########################
            try:
                driver.find_element_by_xpath('//*[@id="button-update"]').click()

                print("Pausing_Updating: "+start_date.strftime("%m-%d-%Y") + " to " + end_date.strftime("%m-%d-%Y"))
                time.sleep(10)
                print("Resumed")

                driver.find_element_by_xpath('//*[@id="react-drop-test"]/div[2]/a').click()
            except:
                pass

            start_date = end_date

            i += 1

        start_date = start_date - datetime.timedelta((8*365/12)+1) + datetime.timedelta(365)

        start_year += 1

    file_mover(folder, view)

###################### MOVE FILES BASED ON HOME OR AWAY #######################

def file_mover(folder_name, view):

    time.sleep(3)
    source = "/home/bkukov/Downloads/"

    if view == 'a':
        dest = "/home/bkukov/PycharmProjects/MLB_New/raw_advanced/"+folder_name
    else:
        dest = "/home/bkukov/PycharmProjects/MLB_New/raw_basic/"+folder_name

    files = os.listdir(source)

    print(files)

    for f in files:
        if f.startswith("Splits"):
            shutil.move(source+f, dest)
        else:
            continue

################################ START UP #####################################

def main():

    choice = ''

    while choice != 'q':

        load_all = [['b', 'h', 'raw_basic_b_home', 'b'], ['b', 'a', 'raw_basic_b_away', 'b'], ['p', 'h', 'raw_basic_p_home', 'b'], ['p', 'a', 'raw_basic_p_away', 'b'],
                    ['b', 'h', 'raw_adv_b_home', 'a'], ['b', 'a', 'raw_adv_b_away', 'a'], ['p', 'h', 'raw_adv_p_home', 'a'], ['p', 'a', 'raw_adv_p_away', 'a']]

        login_name = "bkukov@gmail.com"
        login_pass = "kolusha11"

        log_me_in(login_name, login_pass)

        for selection in load_all:
            player_type, game_type, folder_name, view = selection[0], selection[1], selection[2], selection[3]

            start_year = 2012
            end_year = 2019

            get_split_page()

            player_type_decision(player_type, game_type)

            date_updater(start_year, end_year, view, folder_name)

        print("Press q to quit")

        choice = input()

driver = webdriver.Chrome(executable_path='/home/bkukov/PycharmProjects/MLB_New/chromedriver')
main()
driver.close()


#!/usr/bin/python
#
# Copyright (C) 2015 Umea Universitet, Sweden
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from setuptools import setup, find_packages

__author__ = 'rohe0002'

setup(
    name="oidctest",
    version="0.5.0",
    description="Test framework for testing OpenID Connect provider and "
                "relaying party implementations",
    author="Roland Hedberg",
    author_email="roland.hedberg@umu.se",
    license="Apache 2.0",
    packages=find_packages('src'),
    package_dir={"": "src"},
    classifiers=[
        "Development Status :: 4 - Beta",
        "License :: OSI Approved :: Apache Software License",
        "Programming Language :: Python :: 2.7",
        "Programming Language :: Python :: 3.4",
        "Topic :: Software Development :: Libraries :: Python Modules"],
    install_requires=[
        "argparse",
        "requests >= 2.0.0", 'future',
    ],
    zip_safe=False,
    scripts=['script/optest.py']
)


import torch.nn as nn
import torch
import torch.nn.functional as F

class LapConv2d(nn.Conv2d):

    def __init__(self, in_channels, out_channels, kernel_size=3, stride=1, padding=1, dilation=1, groups=1, bias=True,
                 padding_mode='zeros'):
        super(LapConv2d,self).__init__(in_channels, out_channels, kernel_size, stride, padding, dilation, groups, bias, padding_mode)
        kernel = [[0, -1, 0],
                  [-1, 4, -1],
                  [0, -1, 0]]

        self.weight=nn.Parameter(torch.Tensor(kernel).expand(out_channels,in_channels//groups,kernel_size,kernel_size),requires_grad=False)
        # self.kernel = torch.Tensor(kernel).expand(out_channels, in_channels // groups, 3)
        # self.weight=nn.Parameter(self.kernel,requires_grad=False)

    def forward(self, input):
        # print(self.weight)
        # print(input.size())
        return F.conv2d(input, self.weight, self.bias, self.stride,
                        self.padding, self.dilation, self.groups)


import matplotlib.pyplot as plt
import pylab
import numpy
import math
a1x = [0, 1, 3]
a1y = [2, 0, 1]
a2x = [2, 3, 5]
a2y = [2, -2, 1]

def normal (k, mx, my, start, stop):
    x = []
    y = []

    for i in numpy.arange(start, stop, 0.01):
        x.append(i)
        if k == math.inf:
            y.append(my)
        else:
            y0 = - (i - mx) / k + my
            y.append(y0)

    return x, y
#граница между точками по правилу ближайшего соседа

def zipline (x1, y1, x2, y2):
    #м - середина отрезка
    mx = (x1+x2)/2
    my = (y1+y2)/2
    #нормаль
    if (x2 - x1) == 0:
        k = math.inf
    else:
        k = (y2 - y1) / (x2 - x1)

    return mx, my, k

def solvesystem (k1, k2, mx1, my1, mx2, my2):
    xy = numpy.array([[1., k1], [1., k2]])
    c = numpy.array([(mx1 + k1*my1), (mx2 + k2*my2)])
    numpy.linalg.solve(xy, c)
    return float(numpy.linalg.solve(xy, c)[0]), float(numpy.linalg.solve(xy, c)[1])

m14x, m14y, k14 = zipline(a1x[0], a1y[0], a2x[0], a2y[0])
m34x, m34y, k34 = zipline(a1x[2], a1y[2], a2x[0], a2y[0])
m24x, m24y, k24 = zipline(a1x[1], a1y[1], a2x[0], a2y[0])
m36x, m36y, k36 = zipline(a1x[2], a1y[2], a2x[2], a2y[2])
m35x, m35y, k35 = zipline(a1x[2], a1y[2], a2x[1], a2y[1])
m25x, m25y, k25 = zipline(a1x[1], a1y[1], a2x[1], a2y[1])
xs, ys = solvesystem(k14, k34, m14x, m14y, m34x, m34y)

start1424, stop1424 = solvesystem (k14, k24, m14x, m14y, m24x, m24y)
print(start1424, stop1424)
start1424, stop1424 = solvesystem (k14, k24, m14x, m14y, m24x, m24y)
start1424, stop1424 = solvesystem (k14, k24, m14x, m14y, m24x, m24y)
start1424, stop1424 = solvesystem (k14, k24, m14x, m14y, m24x, m24y)
start1424, stop1424 = solvesystem (k14, k24, m14x, m14y, m24x, m24y)
start1424, stop1424 = solvesystem (k14, k24, m14x, m14y, m24x, m24y)

plt.figure(1)
plt.scatter(a1x, a1y, color='pink')
plt.scatter(a2x, a2y, color='green')
x24, y24 =  normal (k24, m24x, m24y, start1424, stop1424)
    #plt.plot(x, normal (k14, m14x, m14y) )
#plt.axvline((a2x[0]-a1x[0])/2 + a1x[0])
#plt.plot(x, normal (k34, m34x, m34y) )
plt.plot(x24, y24)
    #plt.plot(x, normal (k36, m36x, m36y) )
#plt.axvline( (a2x[2]-a1x[2])/2 + a1x[2])
#plt.plot(x, normal (k35, m35x, m35y) )
#plt.plot(x, normal (k25, m25x, m25y) )
plt.xlabel('x')
plt.ylabel('y')
pylab.legend(("a1", "a2"))

plt.show()


# -*- coding: UTF-8 -*-
import sys, getopt

def main():
    segment_fname = ""
    out_fname = ""
    n = 2
    is_ner = 0

    # 处理命令行参数
    opts, args = getopt.getopt(sys.argv[1:], "hs:r:o:n:")
    for op, value in opts:
        if op == "-s":
            if is_ner == 1:
                usage()
                sys.exit()
            segment_fname = value
        elif op == "-r":
            segment_fname = value
            is_ner = True
        elif op == "-n":
            n = int(value)
        elif op == "-o":
            out_fname = value
        elif op == "-h":
            usage()
            sys.exit()
    if segment_fname == "" or n <= 0:
        usage()
        sys.exit()
    if out_fname == "":
        out_fname = str(n) + "-gram_freq"

    # 开始计数
    count(is_ner, n, segment_fname, out_fname)

def count(is_ner, n, segment_fname, out_fname):
    # 开始统计
    with open(segment_fname, 'r') as segment_file:
            with open(out_fname, 'a') as out_file:
                #创建词表
                word_list = build_word_list(is_ner, segment_file)
                # 统计n-gram频度
                n_gram_dic = freq_count(word_list, n)
                write_out(word_list, n_gram_dic, out_file, n)

def build_word_list(is_ner, segment_file):
    word_list_fin = []
    text = segment_file.read()
    lines = text.splitlines()
    for line in lines:
        line = line.strip()
        # 创建单词列表
        line = list(line)
        # 消除分词结果中的中文空格'/u3000'
        i = 0
        while i < len(line):
            if line[i] == '\u3000':
                del line[i]
            i += 1
        # 将line合并为str
        line = ''.join(line)

        # 以空格为界将分词结果存为列表
        word_list = line.split(' ')
        # 消除列表中多余的空字符和空格
        i = 0
        while i < len(word_list):
            if word_list[i] == '' or word_list[i] == ' ':
                del word_list[i]
            i += 1
        # 若是命名实体识别结果，则进行实体的替换
        if is_ner:
            i = 0
            while i < len(word_list):
                pair = word_list[i].split('/')
                if len(pair) == 2:
                    if pair[1] != 'O':
                        word_list[i] = pair[1]
                    else:
                        word_list[i] = pair[0]
                i += 1

        for word in word_list:
            word_list_fin.append(word)

    return word_list_fin

def freq_count(word_list, n):
    i = 0
    n_gram_dic = {}
    while i + n - 1 <= len(word_list)-1:
        n_gram = ' '.join(word_list[i:i+n])
        if n_gram in n_gram_dic:
            n_gram_dic[n_gram] += 1
        else:
            n_gram_dic[n_gram] = 1
        i += 1
    return n_gram_dic

def write_out(word_list, n_gram_dic, out_file, n):
    n_gram_repeat = {}
    i = 0
    while i + n - 1 <= len(word_list)-1:
        n_gram = ' '.join(word_list[i:i+n])
        if n_gram not in n_gram_repeat:
            n_gram_repeat[n_gram] = 1
            n_gram += ' '+str(n_gram_dic[n_gram])+'\n'
            out_file.write(n_gram)
        i += 1

def usage():
    print('''
        参数：
            -s:必填 -i file1      file1为分词结果文件
            -r:必填 -r file2      file2为命名实体的识别结果
                -s -r 两个参数必须填写一个
            -n:选填 -n n          n为n-gram中的n，默认为2
            -o:选填 -o file       file为输出结果文件，默认在当前目录创建一个名为"n-gram_freq"的文件
            -h:帮助信息
    ''')

if __name__ == "__main__":
    main()


import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
import numpy as np

def xavier_init(fan_in, fan_out, constant=1):
    low = -constant*np.sqrt(6.0/(fan_in + fan_out))
    high = constant*np.sqrt(6.0/(fan_in + fan_out))
    return tf.random_uniform((fan_in, fan_out), minval=low, maxval=high, dtype=tf.float32)

class VariationalAutoEncoder():
    def __init__(self, network_params, activation_fct=tf.nn.softplus, learning_rate1=0.001, learning_rate2=0.1, batch_size=100, load=False, path='./checkpoints'):
        self.mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)

        self.n_input = network_params['n_input']
        self.n_hidden_e_1 = network_params['n_hidden_e_1']
        self.n_hidden_e_2 = network_params['n_hidden_e_2']
        self.n_z = network_params['n_z']
        self.n_hidden_g_1 = network_params['n_hidden_g_1']
        self.n_hidden_g_2 = network_params['n_hidden_g_2']
        self.n_hidden_p_1 = network_params['n_hidden_p_1']
        self.n_hidden_p_2 = network_params['n_hidden_p_2']
        self.n_y = network_params['n_y']

        self.activation_fct = activation_fct
        self.learning_rate1 = learning_rate1
        self.learning_rate2 = learning_rate2

        self.x = tf.placeholder(tf.float32, [None, self.n_input])
        self.y = tf.placeholder(tf.float32, [None, self.n_y])
        self.z = tf.placeholder(tf.float32, [None, self.n_z])

        self._initialize_weights()

        self.saver = tf.train.Saver()

        self.path = path

        if load:
            print('Loading Model...')
            ckpt = tf.train.get_checkpoint_state(self.path)
            self.saver.restore(self.sess, ckpt.model_checkpoint_path)

    def save_model(self, epoch):
        self.saver.save(self.sess, self.path + '/model-' + str(epoch) + '.cptk')
        print("Saved Model")

    def _initialize_weights(self):
        # Initialize weights and biases for neural network
        self.h_e_1 = tf.Variable(xavier_init(self.n_input, self.n_hidden_e_1))
        self.h_e_2 = tf.Variable(xavier_init(self.n_hidden_e_1, self.n_hidden_e_2))
        self.h_e_mean = tf.Variable(xavier_init(self.n_hidden_e_2, self.n_z))
        self.h_e_log_sigma = tf.Variable(xavier_init(self.n_hidden_e_2, self.n_z))

        self.b_e_1 = tf.Variable(tf.zeros([self.n_hidden_e_1], dtype=tf.float32))
        self.b_e_2 = tf.Variable(tf.zeros([self.n_hidden_e_2], dtype=tf.float32))
        self.b_e_mean = tf.Variable(tf.zeros([self.n_z], dtype=tf.float32))
        self.b_e_log_sigma = tf.Variable(tf.zeros([self.n_z], dtype=tf.float32))

        self.h_g_1 = tf.Variable(xavier_init(self.n_z, self.n_hidden_g_1))
        self.h_g_2 = tf.Variable(xavier_init(self.n_hidden_g_1, self.n_hidden_g_2))
        self.h_g_mean = tf.Variable(xavier_init(self.n_hidden_g_2, self.n_input))
        self.h_g_log_sigma = tf.Variable(xavier_init(self.n_hidden_g_2, self.n_input))

        self.b_g_1 = tf.Variable(tf.zeros([self.n_hidden_g_1], dtype=tf.float32))
        self.b_g_2 = tf.Variable(tf.zeros([self.n_hidden_g_2], dtype=tf.float32))
        self.b_g_mean = tf.Variable(tf.zeros([self.n_input], dtype=tf.float32))
        self.b_g_log_sigma = tf.Variable(tf.zeros([self.n_input], dtype=tf.float32))

        self.h_p_1 = tf.Variable(xavier_init(self.n_input, self.n_hidden_e_1))
        self.h_p_2 = tf.Variable(xavier_init(self.n_hidden_p_1, self.n_hidden_p_2))
        self.h_p_mean = tf.Variable(xavier_init(self.n_hidden_p_2, self.n_y))

        self.b_p_1 = tf.Variable(tf.zeros([self.n_hidden_p_1], dtype=tf.float32))
        self.b_p_2 = tf.Variable(tf.zeros([self.n_hidden_p_2], dtype=tf.float32))
        self.b_p_mean = tf.Variable(tf.zeros([self.n_y], dtype=tf.float32))

        self._encode()
        self._generate()
        self._reconstruct()
        self._predict()
        self._create_loss()

        init = tf.global_variables_initializer()
        self.sess = tf.InteractiveSession()
        self.sess.run(init)

    def _encode(self):
        layer_e_1 = self.activation_fct(tf.add(tf.matmul(self.x, self.h_e_1), self.b_e_1))
        layer_e_2 = self.activation_fct(tf.add(tf.matmul(layer_e_1, self.h_e_2), self.b_e_2))
        self.z_e_mean = tf.add(tf.matmul(layer_e_2, self.h_e_mean), self.b_e_mean)

    def _generate(self):
        layer_g_1 = self.activation_fct(tf.add(tf.matmul(self.z, self.h_g_1), self.b_g_1))
        layer_g_2 = self.activation_fct(tf.add(tf.matmul(layer_g_1, self.h_g_2), self.b_g_2))
        self.x_g_mean = tf.nn.sigmoid(tf.add(tf.matmul(layer_g_2, self.h_g_mean), self.b_g_mean))

    def _reconstruct(self):
        layer_e_1 = self.activation_fct(tf.add(tf.matmul(self.x, self.h_e_1), self.b_e_1))
        layer_e_2 = self.activation_fct(tf.add(tf.matmul(layer_e_1, self.h_e_2), self.b_e_2))
        self.z_mean = tf.add(tf.matmul(layer_e_2, self.h_e_mean), self.b_e_mean)
        self.z_log_sigma_sq = tf.add(tf.matmul(layer_e_2, self.h_e_log_sigma), self.b_e_log_sigma)
        layer_g_1 = self.activation_fct(tf.add(tf.matmul(self.z_mean, self.h_g_1), self.b_g_1))
        layer_g_2 = self.activation_fct(tf.add(tf.matmul(layer_g_1, self.h_g_2), self.b_g_2))
        self.x_mean = tf.nn.sigmoid(tf.add(tf.matmul(layer_g_2, self.h_g_mean), self.b_g_mean))

    def _predict(self):
        layer_p_1 = self.activation_fct(tf.add(tf.matmul(self.x, self.h_p_1), self.b_p_1))
        layer_p_2 = self.activation_fct(tf.add(tf.matmul(layer_p_1, self.h_p_2), self.b_p_2))
        self.y_mean = tf.nn.sigmoid(tf.add(tf.matmul(layer_p_2, self.h_p_mean), self.b_p_mean))

    def _create_loss(self):
        gen_loss = -tf.reduce_sum(self.x * tf.log(1e-10 + self.x_mean) + (1 - self.x) * tf.log(1e-10 + 1 - self.x_mean), 1)
        latent_loss = -0.5 * tf.reduce_mean(1 + self.z_log_sigma_sq - tf.square(self.z_mean) - tf.exp(self.z_log_sigma_sq), 1)
        predict_loss = tf.reduce_mean(tf.reduce_sum(tf.squared_difference(self.y, self.y_mean), reduction_indices=[1]))

        self.cost = tf.reduce_mean(gen_loss + latent_loss)
        self.loss = tf.reduce_mean(predict_loss)
        self.optimizer1 = tf.train.AdamOptimizer(learning_rate = self.learning_rate1).minimize(self.cost)
        self.optimizer2 = tf.train.AdamOptimizer(learning_rate = self.learning_rate2).minimize(self.loss)

    def partial_fit(self, x, y):
        opt1, opt2, cost, loss = self.sess.run((self.optimizer1, self.optimizer2, self.cost, self.loss), feed_dict={self.x: x, self.y: y})

        return cost, loss

    def test(self, x, y):
        cost, loss = self.sess.run((self.cost, self.loss), feed_dict={self.x: x, self.y: y})

        return cost, loss

    def encode(self, x):
        return self.sess.run(self.z_e_mean, feed_dict={self.x: x})

    def generate(self, z):
        return self.sess.run(self.x_g_mean, feed_dict={self.z: z})

    def reconstruct(self, x):
        return self.sess.run(self.x_mean, feed_dict={self.x: x})

    def predict(self, x):
        return self.sess.run(self.y_mean, feed_dict={self.x: x})


from recommenders.similarity.s_plus import dot_product, tversky_similarity
from utils.post_processing import *
from utils.pre_processing import *
from scipy import sparse
from recommenders.nlp_strict import NLPStrict
from recommenders.nlp import NLP
from utils.sparse import *
import sys

arg = sys.argv[1:]
mode = arg[0]
if len(arg) > 1:
    topk = int(arg[1])
else:
    topk = 750

# INITIALIZATION
dr = Datareader(mode=mode, verbose=False, only_load=True)
test_pids = dr.get_test_pids()
urm = dr.get_urm()
urm.data = np.ones(len(urm.data))

# PARAMS
norm = True
work = True
split = True
skip_words = True
date = False
porter = False
porter2 = True
lanca = False
lanca2 = True
data1 = False

# NLP STRICT
nlp_strict = NLPStrict(dr)
ucm_strict = nlp_strict.get_UCM().astype(np.float64)
top_pop = dr.get_eurm_top_pop()

# Do not train on challenge set
ucm_strict_T = ucm_strict.copy()
inplace_set_rows_zero(ucm_strict_T, test_pids)
ucm_strict_T = ucm_strict_T.T

sim = tversky_similarity(ucm_strict, ucm_strict_T, k=450, alpha=0.2, beta=0.5,
                         shrink=150, target_items=test_pids)

# Compute eurm
eurm = dot_product(sim, urm, k=topk)
eurm = eurm.tocsr()
eurm = eurm[test_pids, :]

# NLP TOKENS
nlp = NLP(dr)

ucm = nlp.get_UCM(data1=data1).astype(np.float64)

# Do not train on challenge set
ucm_T = ucm.copy()
inplace_set_rows_zero(ucm_T, test_pids)
ucm_T = ucm_T.T

sim_lele = tversky_similarity(ucm, ucm_T, k=200, alpha=0.9, beta=1.0,
                              shrink=0, target_items=test_pids)

# Compute eurm
eurm_lele = dot_product(sim_lele, urm, k=topk)
eurm_lele = eurm_lele.tocsr()
eurm_lele = eurm_lele[test_pids, :]

# NLP FUSION
a = 0.2
eurm_l1 = norm_l1_row(eurm)
eurm_lele_l1 = norm_l1_row(eurm_lele)
nlp_fusion = a * eurm_l1 + (1.0 - a) * eurm_lele_l1

if dr.online():
    sparse.save_npz(ROOT_DIR + '/recommenders/script/creative/online_npz/nlp_fusion_online.npz', nlp_fusion)
else:
    sparse.save_npz(ROOT_DIR + '/recommenders/script/creative/offline_npz/nlp_fusion_offline.npz', nlp_fusion)


# Creating a script that inputs the bilateral csv into my django psqldb

import os
import django
import pandas as pd

os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'hayleecodes.settings')
django.setup()

from mysite.models import Bilateral

bilateral_data = pd.read_csv('django_app/bilateral_flow.csv', low_memory=False)

for index, row in bilateral_data.iterrows():
    region_orig = row[0]
    region_orig_id = row[1]
    region_dest = row[2]
    region_dest_id = row[3]
    country_orig = row[4]
    country_orig_id = row[5]
    country_dest = row[6]
    country_dest_id = row[7]
    regionflow_1990 = row[8]
    regionflow_1995 = row[9]
    regionflow_2000 = row[10]
    regionflow_2005 = row[11]
    countryflow_1990 = row[12]
    countryflow_1995 = row[13]
    countryflow_2000 = row[14]
    countryflow_2005 = row[15]
    metadata = row[16]

    q = Bilateral(region_orig = region_orig, region_orig_id = region_orig_id, region_dest = region_dest, \
    region_dest_id = region_orig_id, country_orig = country_orig, country_orig_id = country_orig_id, \
    country_dest = country_dest, country_dest_id = country_dest_id, regionflow_1990 = regionflow_1990, \
    regionflow_1995 = regionflow_1995, regionflow_2000 = regionflow_2000, regionflow_2005 = regionflow_2005, \
    countryflow_1990 = countryflow_1990, countryflow_1995 = countryflow_1995, countryflow_2000 = countryflow_2000, \
    countryflow_2005 = countryflow_2005, metadata = metadata)

    q.save()


import random

x = False
while not x:
    random_integer = random.randint(0, 1)
    prompt = input("Guess 'Heads' or 'Tails'\n")

    if random_integer == 0:
        result = 'Heads'
    else:
        result = 'Tails'

    if prompt == result:
        print(f" You won. It really was {result}")
        x = True
    else:
        print(f"Oh no.. it was {result} and you told {prompt}")

100 Days/Beginner/Day 4_Heads or Tails.py


import json
import logging
import traceback
from datetime import date, datetime

IGNORED_FIELDS = (
    'args',
    'asctime',
    'created',
    'exc_info',
    'filename',
    'funcName',
    'levelname',
    'levelno',
    'lineno',
    'module',
    'msecs',
    'message',
    'msg',
    'name',
    'pathname',
    'process',
    'processName',
    'relativeCreated',
    'thread',
    'threadName',
)

class CEEFormatter(logging.Formatter):
    def __init__(self, *args, **kwargs):
        """The following keyword arguments are specifi to CEEFormatter:
            ignored_fields: list of strings. This fields will not be written to the
                            json record.
            terminate: If True, the output string is newline-terminated. This is necessary
                       when using the syslog protocol over TCP, or else the messages will
                       not be received correctly. Default: False.
            colon_start: If True, the the string ": " is prepended to the @cee cookie. this
                       is neccesary for the correct parsing of the message by rsyslog/mmjsonparse.
                       if it is missing, you will get a message about a missing JSON cookie.
                       (default: True)
        The rest of the arguments are passed to logging.Formatter.

        The formatTime method is overriden to report time in UTC and
        in RFC3339 format.
        """
        _kwargs = dict(kwargs)

        self.ignored_fields = _kwargs.pop('ignored_fields', IGNORED_FIELDS)

        terminate = _kwargs.pop('terminate', False)
        colon_start = _kwargs.pop('colon_start', True)

        self._template = ((': ' if colon_start else '')
                           + '@cee: %s'
                           + ('\n' if terminate else '')
                         )

        super(CEEFormatter, self).__init__(*args, **_kwargs)

    def jsonhandler(self, obj):
        try:
            return str(obj)
        except Exception:
            return '<object of type \'{}\' cannot be converted to str>'.format(
                type(obj).__name__
            )

    def formatTime(self, log_record):
        return datetime.utcfromtimestamp(log_record.created).isoformat()

    def format(self, log_record):
        # see https://fedorahosted.org/lumberjack/wiki/FieldList
        # extra fields are dumped into the "app" dictionary.
        app_data = {k: v for k, v in log_record.__dict__.items()
                         if k not in IGNORED_FIELDS}

        app_data.update({
            'module': log_record.module,
            'function': log_record.funcName,
            'logger': log_record.name,
            })

        if log_record.exc_info:
            app_data['exception'] = '\n'.join(
                traceback.format_exception(*log_record.exc_info)
                )

        record = {
            'time': self.formatTime(log_record),
            'msg': log_record.getMessage(),
            'pri': log_record.levelname,
            'sev': log_record.levelno,
            'proc': {
                'pid': log_record.process,
                'tid': log_record.thread,
                'name': log_record.processName,
                'tname': log_record.threadName
            },
            'file': {
                'line': log_record.lineno,
                'name': log_record.filename,
                'path': log_record.pathname
            },
            'app': app_data
        }

        return self._template % (
            json.dumps(record, default=self.jsonhandler)
        )


from app.placemats.stores.task_queue import BaseTaskQueue, TASK_INIT, TASK_PENDING, TASK_FAILED, TASK_SUCCEEDED
from app.placemats.stores.mongo_cache import MongoCache
import pymongo.collection
from pymongo import ReturnDocument
import time
import uuid
from typing import Tuple

class MongoTaskQueue(BaseTaskQueue):
    def __init__(self, collection: pymongo.collection.Collection) -> None:
        super().__init__()
        self.coll = collection
        self.cache = MongoCache(self.coll, ttl=3_600)

    def enqueue(self, idempotency_key: str, task_info: dict):
        self.cache.set(key=idempotency_key, data={
            'task_info': task_info,
            'state': TASK_INIT,
        })

    def dequeue(self) -> Tuple[str, str, dict]:
        while True:
            token = str(uuid.uuid4())
            d = self.coll.find_one_and_update(
                {'data.state': TASK_INIT}, {'$set': {'data.state': TASK_PENDING, 'data.token': token}},
                return_document=ReturnDocument.AFTER)
            if d is None:
                time.sleep(0.15)
                continue
            return d['_id'], token, d['data']['task_info']

    def done(self, idempotency_key: str, token: str, exception: Exception = None):
        state = TASK_SUCCEEDED if exception is None else TASK_FAILED
        self.coll.find_one_and_update(
            {'_id': idempotency_key, 'data.state': TASK_PENDING, 'data.token': token}, {'$set': {'data.state': state}})