#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

import json
import re
import sys
import toml
import voluptuous
import voluptuous.humanize
from voluptuous import Schema, Optional, Any, All, Required, Length, Range, Msg, Match

Text = Any(str, bytes)

id_regex = re.compile(r"^[a-z0-9-]+$")
feature_schema = Schema(
    {
        Match(id_regex): {
            Required("title"): All(Text, Length(min=1)),
            Required("description"): All(Text, Length(min=1)),
            Required("bug-numbers"): All(Length(min=1), [All(int, Range(min=1))]),
            Required("restart-required"): bool,
            Required("type"): "boolean",  # In the future this may include other types
            Optional("preference"): Text,
            Optional("default-value"): Any(
                bool, dict
            ),  # the types of the keys here should match the value of `type`
            Optional("is-public"): Any(bool, dict),
            Optional("description-links"): dict,
        },
    }
)

EXIT_OK = 0
EXIT_ERROR = 1

def main(output, *filenames):
    features = {}
    errors = False
    try:
        features = process_files(filenames)
        json.dump(features, output, sort_keys=True)
    except ExceptionGroup as error_group:
        print(str(error_group))
        return EXIT_ERROR
    return EXIT_OK

class ExceptionGroup(Exception):
    def __init__(self, errors):
        self.errors = errors

    def __str__(self):
        rv = ["There were errors while processing feature definitions:"]
        for error in self.errors:
            # indent the message
            s = "\n".join("    " + line for line in str(error).split("\n"))
            # add a * at the beginning of the first line
            s = "  * " + s[4:]
            rv.append(s)
        return "\n".join(rv)

class FeatureGateException(Exception):
    def __init__(self, message, filename=None):
        super(FeatureGateException, self).__init__(message)
        self.filename = filename

    def __str__(self):
        message = super(FeatureGateException, self).__str__()
        rv = ["In"]
        if self.filename is None:
            rv.append("unknown file:")
        else:
            rv.append('file "{}":\n'.format(self.filename))
        rv.append(message)
        return " ".join(rv)

    def __repr__(self):
        # Turn "FeatureGateExcept(<message>,)" into "FeatureGateException(<message>, filename=<filename>)"
        original = super(FeatureGateException, self).__repr__()
        with_comma = original[:-1]
        # python 2 adds a trailing comma and python 3 does not, so we need to conditionally reinclude it
        if len(with_comma) > 0 and with_comma[-1] != ",":
            with_comma = with_comma + ","
        return with_comma + " filename={!r})".format(self.filename)

def process_files(filenames):
    features = {}
    errors = []

    for filename in filenames:
        try:
            with open(filename, "r") as f:
                feature_data = toml.load(f)

            voluptuous.humanize.validate_with_humanized_errors(
                feature_data, feature_schema
            )

            for feature_id, feature in feature_data.items():
                feature["id"] = feature_id
                features[feature_id] = expand_feature(feature)
        except (
            voluptuous.error.Error,
            IOError,
            FeatureGateException,
            toml.TomlDecodeError,
        ) as e:
            # Wrap errors in enough information to know which file they came from
            errors.append(FeatureGateException(e, filename))

    if errors:
        raise ExceptionGroup(errors)

    return features

def hyphens_to_camel_case(s):
    """Convert names-with-hyphens to namesInCamelCase"""
    rv = ""
    for part in s.split("-"):
        if rv == "":
            rv = part.lower()
        else:
            rv += part[0].upper() + part[1:].lower()
    return rv

def expand_feature(feature):
    """Fill in default values for optional fields"""

    # convert all names-with-hyphens to namesInCamelCase
    key_changes = []
    for key in feature.keys():
        if "-" in key:
            new_key = hyphens_to_camel_case(key)
            key_changes.append((key, new_key))

    for (old_key, new_key) in key_changes:
        feature[new_key] = feature[old_key]
        del feature[old_key]

    if feature["type"] == "boolean":
        feature.setdefault("preference", "features.{}.enabled".format(feature["id"]))
        # set default value to None so that we can test for perferences where we forgot to set the default value
        feature.setdefault("defaultValue", None)
    elif "preference" not in feature:
        raise FeatureGateException(
            "Features of type {} must specify an explicit preference name".format(
                feature["type"]
            )
        )

    feature.setdefault("isPublic", False)

    try:
        for key in ["defaultValue", "isPublic"]:
            feature[key] = process_configured_value(key, feature[key])
    except FeatureGateException as e:
        raise FeatureGateException(
            "Error when processing feature {}: {}".format(feature["id"], e)
        )

    return feature

def process_configured_value(name, value):
    if not isinstance(value, dict):
        return {"default": value}

    if "default" not in value:
        raise FeatureGateException(
            "Config for {} has no default: {}".format(name, value)
        )

    expected_keys = set(
        {
            "default",
            "win",
            "mac",
            "linux",
            "android",
            "nightly",
            "early_beta_or_earlier",
            "beta",
            "release",
            "dev-edition",
            "esr",
            "thunderbird",
        }
    )

    for key in value.keys():
        parts = [p.strip() for p in key.split(",")]
        for part in parts:
            if part not in expected_keys:
                raise FeatureGateException(
                    "Unexpected target {}, expected any of {}".format(
                        part, expected_keys
                    )
                )

    # TODO Compute values at build time, so that it always returns only a single value.

    return value

if __name__ == "__main__":
    sys.exit(main(sys.stdout, *sys.argv[1:]))


#!/usr/bin/python

import rosbag
import rospy
from sensor_msgs.msg import Image
from ouster_ros.msg import PacketMsg
import os
import sys
import numpy as np
import cv2
from cv_bridge import CvBridge

bridge = CvBridge()
amb_c = '/lidar_center/ambient_image'
int_c = '/lidar_center/intensity_image'
rng_c = '/lidar_center/range_image'
steer_topic = '/pacmod/parsed_tx/steer_rpt'
autonomy_topic = '/pacmod/as_tx/enabled'
lidar_packets_topic = '/lidar_center/lidar_packets'

steer_ratio = 14.7
channels = [amb_c, int_c, rng_c]
bag_files = sys.argv[1:]

class OusterImage(object):
    def __init__(self, ts):
        self.ts = ts
        self.amb = None
        self.rng = None
        self.inten = None
        self.steer = None

    def set_amb(self, amb):
        self.amb = amb

    def set_inten(self, inten):
        self.inten = inten

    def set_rng(self, rng):
        self.rng = rng

    def set_steer(self, steer):
        self.steer = steer

    def image(self):
        if type(self.rng) != type(None) and type(self.amb) != type(None) and type(self.inten) != type(None) and type(self.steer) != type(None):
            img = np.dstack((self.amb, self.inten, self.rng))
            return img, self.steer
        else:
            print("failed")
            return None, None

future_rng = None

def rng_img_callback(msg):
    global future_rng
    future_rng = msg

rospy.init_node('fix_ouster_images', anonymous=True)
pub = rospy.Publisher(lidar_packets_topic, PacketMsg)
sub = rospy.Subscriber(rng_c, Image, rng_img_callback, queue_size=1)

imgs = []
steers = []
for bag_file in bag_files:
    bag = rosbag.Bag(bag_file)
    first = True
    oi = OusterImage(0)
    wheel = 0.0
    autonomous = True
    autonomy_changed = False

    for topic, msg, ts in bag.read_messages():
        if topic == lidar_packets_topic:
            rospy.Rate(1400).sleep()
            pub.publish(msg)
        if topic == autonomy_topic:
            autonomy_changes = autonomous != msg.data
            autonomous = msg.data
            if autonomy_changed and autonomous:
                oi = OusterImage(0)
        if not autonomous:
            if topic == steer_topic:
                oi.set_steer(msg.manual_input / steer_ratio)
            if topic in channels:
                if msg.header.stamp.to_nsec() != oi.ts:
                    if not first:
                        img, steer = oi.image()
                        if type(img) != type(None):
                            imgs.append(img)
                            steers.append(steer)
                    oi = OusterImage(msg.header.stamp.to_nsec())
                    first = False
            if topic == amb_c:
                cv_img = bridge.imgmsg_to_cv2(msg)
                oi.set_amb(cv_img)
            elif topic == int_c:
                cv_img = bridge.imgmsg_to_cv2(msg)
                oi.set_inten(cv_img)
            elif topic == rng_c:
                for t in range(3000):
                    if future_rng is None:
                        rospy.Rate(1000).sleep()
                        continue
                assert future_rng is not None
                msg.data = future_rng.data
                future_rng = None
                cv_img = bridge.imgmsg_to_cv2(msg)
                oi.set_rng(cv_img)

    os.mkdir(bag_file[:-4])
    steers_file = os.path.join(bag_file[:-4], 'steers.csv')
    img_extension = '.png'
    with open(steers_file, 'w') as f:
        f.write('file_name,steer_angle\n')
        for i, img in enumerate(imgs):
            cv2.imwrite(os.path.join(bag_file[:-4], str(i)+img_extension), img)
            f.write(str(i)+img_extension + ',' + str(steers[i]) + '\n')
    print("Saved new folder: " + bag_file[:-4])


#!/home/saeed/Programs/miniconda3/bin/python

from datetime import datetime as dt
from datetime import timedelta as td
from random import gammavariate, choice, seed
from copy import deepcopy as dp
from LatLon import lat_lon as ll
import os, sys

#_________GET INPUT

inp = input("\n+++ Nordic file:\n")

if not os.path.exists("STATION0.HYP"): print("\n+++ No STATION0.HYP found!\n"); sys.exit(0)

#_________CONVERT PHASE

def get_mag():

    seed(1)
    mag = [gammavariate(2,2) for _ in range(500)]
    mag = [_/max(mag) for _ in mag]
    mag = [_*5.5 for _ in mag]

    return mag

random_mag = get_mag()

def get_ot(l):

    if l[16]=="6": l = l[:16]+"59.00"+l[20:]

    for i in [6,8,11,13,16]:

        if not l[i].strip():

            l = l[:i]+"0"+l[i+1:]

    ot = dt.strptime(l[:20], " %Y %m%d %H%M %S.%f")
    lat = float(l[23:30])
    lon = float(l[31:38])
    dep = float(l[39:43])
    erh = 0
    erz = 0
    rms = float(l[52:55])

    if l[56:59].strip() and float(l[56:59])!=0: mag = float(l[56:59])
    else: mag = choice(random_mag)

    return ot, lat, lon, dep, mag, erh, erz, rms

def get_ttim(ot, l, wet, log):

    for i in [18, 19, 20, 22, 23]:

        if not l[i].strip():

            l = l[:i]+"0"+l[i+1:]

    tmp = dp(ot)

    sec = float(l[22:28])
    if sec>=60.00: sec=59.999
    l = l[:22]+'%006.3f'%(sec)+l[28:]
    delayDay = False
    try:
        at = dt.strptime(l[18:28], "%H%M%S.%f")
    except ValueError:
        if float(l[18:20]) == 24:
            l = l[:18]+"00"+l[20:]
            at = dt.strptime(l[18:28], "%H%M%S.%f")
            delayDay = True
    at = at.replace(year=tmp.year, month=tmp.month, day=tmp.day, hour=at.hour, minute=at.minute, second=at.second, microsecond=at.microsecond)
    if delayDay: at = at.replace(day=at.day-1)
    if ot.hour == 23 and at.hour==0: at = at + td(days=1)
    ttim = at-ot
    ttim = ttim.total_seconds()

    if wet>1e-5 and (ttim>100 or ttim<0):
        print("Warning! Travel time exceeded 100s or is negative! Check log.dat")
        log.write(ot.isoformat()+" > "+l)

    return ttim

def get_wet(l):

    if l[14] == " ": return 1
    elif l[14] == "4": return 1e-6
    else: return (0.5)**float(l[14])

def get_pha(l, log):

    if l[10].strip() and l[10].upper()=="P":

        sta = l[:5].strip().upper()
        wet = get_wet(l)
        pha = "P"
        tti = get_ttim(ot, l, wet, log)

        line = "%4s %7.3f %.1e %1s\n"%(sta, tti, wet, pha)
        g.write(line)

    elif l[10].strip() and l[10].upper()=="S":

        sta = l[:5].strip().upper()
        wet = get_wet(l)
        pha = "S"
        tti = get_ttim(ot, l, wet, log)

        line = "%4s %7.3f %.1e %1s\n"%(sta, tti, wet, pha)
        g.write(line)

#________MAIN

with open(inp) as f, open("phase.dat", "w") as g, open("log.dat", "w") as log:

    evn = 0
    flag = False

    for l in f:

        if l.strip() and l[24:43].strip() and l[79]=="1":

            ot, lat, lon, dep, mag, erh, erz, rms = get_ot(l)
            if not ((lat>23)&(lat<42))&((lon>42)&(lon<63)): continue
            evn+=1

            header = "#, YR, MO, DY, HR, MN, SC, LAT, LON, DEP, MAG, EH, EZ, RMS, ID"
            line = "# %22s %7.3f %7.3f %5.1f %4.1f %6.1f %6.1f %5.2f %6d\n"%(ot.strftime("%Y %m %d %H %M %S.%f")[:22],
                                                                             lat, lon, dep, mag, erh, erz, rms, evn)
            g.write(line)
            flag = True

        if flag and not l.strip(): flag = False

        if flag and l.strip() and l[79] in ["4", " "]:

            pha = get_pha(l, log)

#_________CONVERT STATION

with open("STATION0.HYP") as f, open("station.dat", "w") as g:

    sta_list = []

    for l in f:

        if len(l.strip())>20 and [l[13],l[22]] == ["N", "E"]:

            sta = l[:6].strip().upper()
            if sta not in sta_list: sta_list.append(sta)
            else: continue
            lat = ll.Latitude(degree=float(l[6:8]), minute=float(l[8:13])).decimal_degree
            lon = ll.Longitude(degree=float(l[15:17]), minute=float(l[17:21])).decimal_degree
            try:
                elv = float(l[23:27])
            except ValueError:
                elv = 0
            g.write("%4s %7.3f %7.3f %5d\n"%(sta, lat, lon, elv))


from random import *
import pygame
pygame.init()

width, height = 900, 600
fps = 70
clock = pygame.time.Clock()
screen = pygame.display.set_mode((width, height))
bird_x = width // 6
bird_y = height // 2
bird = pygame.Rect(bird_x, bird_y, 50, 50)
tube1 = pygame.Rect(width, -300, 100, 550)
tube2 = pygame.Rect(width, 400, 100, 650)
tube_x = width
tubes = [tube1, tube2]
playing = True
play = False
cause = random
score = 0
font_name = pygame.font.match_font('arial')
bg = pygame.image.load('bg.jpg')

def tubes_position():
    tube1.x = tube_x
    tube2.x = tube_x
    tube1.y = -300
    tube2.y = 400

while playing:
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            playing = False
    good_actions = pygame.mouse.get_pressed()[0] or pygame.key.get_pressed()[pygame.K_SPACE]

    if not play:
        bird_y = height // 2
    if good_actions:
        play = True
        bird_y -= 5
    else:
        bird_y += 5

    if bird.top < 0 or bird.bottom > height:
        bird_y = height // 2
        play = False

    bird.y = bird_y

    screen.blit(bg, (0, 0))

    if play:
        for tube in tubes:
            pygame.draw.rect(screen, 'green', tube)
            tube.x -= 5
            if cause:
                tube.y += 1
            else:
                tube.y -= 1
            if tube2.x == -100:
                cause = not cause
                tubes_position()
    else:
        cause = not cause
        tubes_position()

    if bird.colliderect(tube1) or bird.colliderect(tube2):
        play = False

    pygame.draw.rect(screen, 'yellow', bird)
    clock.tick(fps)
    pygame.display.update()


from allauth.account.views import confirm_email as allauthemailconfirmation

from . import apiviews
from django.conf.urls import include
from django.conf.urls import url
from django.views.decorators.csrf import csrf_exempt

urlpatterns = [
    # dont remove
    # otherwise -->Reverse for 'account_email_verification_sent' not found.
    # 'account_email_verification_sent' is not a valid view function or ...
    url(r'rest_auth/login/$', apiviews.login_view),
    url(r'rest_auth/login/google/$',
        csrf_exempt(apiviews.login_google_view), name='google_login'),
    url(r'rest_auth/login/linkedIn/$',
        csrf_exempt(apiviews.login_linkedin_view), name='linkedin_login'),

    url(r'rest_auth/accounts/', include('allauth.urls')),
    url(r'rest_auth/', include('rest_auth.urls')),
    url(r'rest_auth/registration/account-confirm-email/(?P<key>[\s\d\w().+-_,:&]+)/',
        allauthemailconfirmation, name="account_confirm_email"),
    url(r'rest_auth/registration/', include('rest_auth.registration.urls')),

]


import math
import functools
import re
import random
import collections

# Finished in 477th place

if __name__ == "__main__":
    with open('./input/12.txt') as f:
        data = f.read().splitlines()

        paths = collections.defaultdict(list)

        for line in data:
            start, end = line.split('-')
            paths[start].append(end)
            paths[end].append(start)
        num_paths = 0
        stack = [('start', set(['start']))]

        while stack:
            node, seen = stack.pop()
            for end in paths[node]:
                if end == 'end':
                    num_paths += 1
                    continue
                if end not in seen:
                    new_set = seen.copy()
                    if end.lower() == end:
                        new_set.add(end)
                    stack.append((end, new_set))
        print(num_paths)

        num_paths = 0
        stack = [('start', set(['start']), False)]

        while stack:
            node, seen, used = stack.pop()
            for end in paths[node]:
                if end == 'end':
                    num_paths += 1
                    continue
                if end not in seen:
                    new_set = seen.copy()
                    if end.lower() == end:
                        new_set.add(end)
                    stack.append((end, new_set, used))
                elif not used and end != 'start' and end != 'end':
                    new_set = seen.copy()
                    if end.lower() == end:
                        new_set.add(end)
                    stack.append((end, new_set, True))
        print(num_paths)


from html.parser import HTMLParser
import codecs
import os
from shutil import copyfile
from pathlib import Path
import uuid

START_TITLE_ID = 1708

class StoneInfo():

	def __init__(self):
		self.name = ""
		self.image = ""

class MyHTMLParser(HTMLParser):

	def __init__(self):
		super().__init__()
		self.row = 0
		self.col = -1
		self.parseName = False
		self.stoneInfos = []

	def handle_starttag(self, tag, attrs):
		if tag == "tr":
			self.stoneInfos.append(StoneInfo())
			self.col = -1
			return
		if tag == "td":
			self.col += 1
			return
		if tag == "b" and self.col == 3:
			self.parseName = True
		if tag == "img" and self.col == 4:
			for attr in attrs:
				if attr[0] == "src":
					self.stoneInfos[self.row].image = attr[1].split("/")[-1:][0]

	def handle_endtag(self, tag):
		if tag == "tr":
			self.row += 1
			self.stoneName = ""
			return
		if tag == "b":
			self.parseName = False
			return

	def handle_data(self, data):
		if self.parseName:
			if self.stoneInfos[self.row].name != "":
				self.stoneInfos[self.row].name += " "
			self.stoneInfos[self.row].name += data

def prepareImageFilename(stoneInfoImage):
	pos = stoneInfoImage.find("%")
	while pos != -1:
		stoneInfoImage = stoneInfoImage[:pos] + stoneInfoImage[pos + 3:]
		pos = stoneInfoImage.find("%", pos)
	return stoneInfoImage

if not (Path("images/300pxUUID")).is_dir():
	os.makedirs("images/300pxUUID")

parser = MyHTMLParser()

with codecs.open('stone-list.html', "r", "utf-8") as htmlFile:
	with codecs.open('TourWizArt_insert_datasetObjects.sql', "w", "utf-8") as sqlfileObjects:
		with codecs.open('TourWizArt_insert_titles.sql', "w", "utf-8") as sqlfileTitles:
			parser.feed(htmlFile.read())
			stoneFiles = os.listdir("Liste der Stolpersteine in Mainz – Wikipedia-Dateien")

			index = 0
			for stoneInfo in parser.stoneInfos:
				index += 1
				found = False
				stoneInfoImageFilename = prepareImageFilename(stoneInfo.image)
				for stoneFile in stoneFiles:
					nextUuid = str(uuid.uuid4())
					if stoneInfoImageFilename == stoneFile:
						found = True
						break

				if found:
					sqlfileObjects.write("INSERT INTO dataset_object (tourDatasetUuid, datasetObjectUuid, titleTextId, location, status, artistUuid, styleTextId, year) VALUES ('693fda6d-d998-4807-8347-4f083a10bad7', '" + nextUuid + "', " + str(START_TITLE_ID + index - 1) + ", NULL, 'ACTIVE', 'c47f79dc-ad57-4d97-85ca-65b1c9b57058', 0, 0);\n")
					sqlfileTitles.write("INSERT INTO text (languageCode, text) VALUES ('en', '" + stoneInfo.name + "');\n")
					sourceFilename = "images/300px/" + stoneInfoImageFilename
					destFilename = "images/300pxUUID/" + nextUuid + ".jpg"
					if Path(destFilename).is_file():
						continue
					copyfile(sourceFilename, destFilename)

				else:
					print("----------not found " + stoneInfoImageFilename)


##
# @file xls2json.py
# @brief Convert Recipe Excel file to JSON file
# @author wangtaisheng
# @version v0.1
# @date 2022-10-11

import json
import pandas as pd
import sys
import re

def func_xls2json(xls_file):
    json_dict = { "CfgFile" : "", "RecipeFile" : "", "BaseWaveFile" : ""}
    CfgFileString = ""
    RecipeFileString = ""
    # BaseWaveFileString = ""

    # Read the excel file
    ModeTable = pd.read_excel(xls_file,sheet_name='s_ModeTable',header=1,keep_default_na=False,usecols="A:M")
    ChlGroupTable = pd.read_excel(xls_file,sheet_name='s_ChlGroupTable',keep_default_na=False)
    PulseWidthTable = pd.read_excel(xls_file,sheet_name='s_PulseWidthTable',keep_default_na=False,usecols="A:F")
    DZCarrierTable = pd.read_excel(xls_file,sheet_name='s_DZCarrierTable',keep_default_na=False,usecols="A:C")
    OutputDataTable = pd.read_excel(xls_file,sheet_name='s_OutputDataTable',keep_default_na=False,usecols="A:C")
    options = pd.read_excel(xls_file,sheet_name='options',keep_default_na=False)

    #Calculate the length of each row in ChlGroupTable
    #And load the data into _s_ChlGroupTable
    ChlGroupLength = []
    _s_ChlGroupTable = []
    for row in ChlGroupTable.index:
        tmp = ChlGroupTable.iloc[row, :]
        tmp = list(filter(None, tmp))
        _s_ChlGroupTable.extend(tmp)
        ChlGroupLength.append(len(tmp))
    print(sum(ChlGroupLength))
    print(ChlGroupLength)
    print(_s_ChlGroupTable)

    # Write RecipeFile
    RecipeFile = []
    ParseType = '{:>02}'.format(options.loc[0, 'ParseType'])
    RecipeFile.append(ParseType)

    ModeCode = re.findall(".{2}", '{:>08}'.format(options.loc[0, 'ModeCode']))
    ModeCode.reverse()
    ModeCode = ' '.join(ModeCode)
    RecipeFile.append(ModeCode)

    ModeVersion = re.findall(".{2}", '{:>08}'.format(options.loc[0, 'ModeVersion']))
    ModeVersion.reverse()
    ModeVersion = ' '.join(ModeVersion)
    RecipeFile.append(ModeVersion)

    ChlGroupNumber = '{:>02}'.format(str(format(ChlGroupTable.shape[0], 'x')))
    RecipeFile.append(ChlGroupNumber)

    StdWidthRuleNumber = '{:>02}'.format(str(format(PulseWidthTable.shape[0], 'x')))
    RecipeFile.append(StdWidthRuleNumber)

    StdExtraParamNumber = '{:>02}'.format(str(format(DZCarrierTable.shape[0], 'x')))
    RecipeFile.append(StdExtraParamNumber)

    DmaOutputDataNumber = '{:>02}'.format(str(format(OutputDataTable.shape[0], 'x')))
    RecipeFile.append(DmaOutputDataNumber)

    _ChlGroupOffset = 0x17
    ChlGroupOffset = re.findall(".{2}", '{:>04}'.format(str(format(_ChlGroupOffset, 'x'))))
    ChlGroupOffset.reverse()
    ChlGroupOffset = ' '.join(ChlGroupOffset)
    RecipeFile.append(ChlGroupOffset)

    _StdWidthRuleOffset = _ChlGroupOffset + int(ChlGroupNumber, 16) + sum(ChlGroupLength)
    StdWidthRuleOffset = re.findall(".{2}", '{:>04}'.format(str(format(_StdWidthRuleOffset, 'x'))))
    StdWidthRuleOffset.reverse()
    StdWidthRuleOffset = ' '.join(StdWidthRuleOffset)
    RecipeFile.append(StdWidthRuleOffset)

    _StdExtraParamOffset = _StdWidthRuleOffset + int(StdWidthRuleNumber, 16)*0x05
    StdExtraParamOffset = re.findall(".{2}", '{:>04}'.format(str(format(_StdExtraParamOffset, 'x'))))
    StdExtraParamOffset.reverse()
    StdExtraParamOffset = ' '.join(StdExtraParamOffset)
    RecipeFile.append(StdExtraParamOffset)

    _DmaOutputDataOffset = _StdExtraParamOffset + int(StdExtraParamNumber, 16)*0x02
    DmaOutputDataOffset = re.findall(".{2}", '{:>04}'.format(str(format(_DmaOutputDataOffset, 'x'))))
    DmaOutputDataOffset.reverse()
    DmaOutputDataOffset = ' '.join(DmaOutputDataOffset)
    RecipeFile.append(DmaOutputDataOffset)
    print(DmaOutputDataOffset)

    _ModeDataOffset = _DmaOutputDataOffset + int(DmaOutputDataNumber, 16)*0x08
    ModeDataOffset = re.findall(".{2}", '{:>04}'.format(str(format(_ModeDataOffset, 'x'))))
    ModeDataOffset.reverse()
    ModeDataOffset = ' '.join(ModeDataOffset)
    RecipeFile.append(ModeDataOffset)
    # RecipeFile.append('\r\n')
    print(ModeDataOffset)

    # Write ChlGroupTable
    ChlGroupTableSize = []
    for i in ChlGroupLength:
        ChlGroupTableSize.append('{:>02}'.format(str(format(i, 'x'))))
    ChlGroupTableSize = ' '.join(ChlGroupTableSize)
    RecipeFile.append(ChlGroupTableSize)
    # RecipeFile.append('\r\n')

    s_ChlGroupTable = []
    ModeOutputData_t = list(OutputDataTable.loc[:, 'ModeOutputData_t'])
    for i in _s_ChlGroupTable:
        if i in ModeOutputData_t:
            # print(ModeOutputData_t.index(i))
            s_ChlGroupTable.append('{:>02}'.format(str(format(ModeOutputData_t.index(i), 'x'))))
        elif i == 'CHL_NULL':
            # print('ff')
            s_ChlGroupTable.append('ff')
            # s_ChlGroupTable.append('\r\n')
    s_ChlGroupTable = ' '.join(s_ChlGroupTable)
    RecipeFile.append(s_ChlGroupTable)
    print(s_ChlGroupTable)

    # Write PulseWidthTable
    s_PulseWidthTable = []
    # print(PulseWidthTable)
    for row in PulseWidthTable.index:
        s_PulseWidthTable.append('{:>02}'.format(str(format(PulseWidthTable.loc[row, 'WidthMin'], 'x'))))
        s_PulseWidthTable.append('{:>02}'.format(str(format(PulseWidthTable.loc[row, 'WidthMax'], 'x'))))
        s_PulseWidthTable.append('{:>02}'.format(str(format(PulseWidthTable.loc[row, 'Constant'], 'x'))))
        s_PulseWidthTable.append('{:>02}'.format(str(format(PulseWidthTable.loc[row, 'Interval'], 'x'))))
        s_PulseWidthTable.append('{:>02}'.format(str(format(PulseWidthTable.loc[row,'StepValUs'], 'x'))))
        # s_PulseWidthTable.append('\r\n')
    s_PulseWidthTable = ' '.join(s_PulseWidthTable)
    RecipeFile.append(s_PulseWidthTable)
    print('s_PulseWidthTable')
    print(s_PulseWidthTable)

    # Write DZCarrierTable
    s_DZCarrierTable = []
    for row in DZCarrierTable.index:
        s_DZCarrierTable.append('{:>02}'.format(str(format(DZCarrierTable.loc[row, 'DeadZone'], 'x'))))
        s_DZCarrierTable.append('{:>02}'.format(str(format(DZCarrierTable.loc[row,  'Carrier'], 'x'))))
        # s_DZCarrierTable.append('\r\n')
    s_DZCarrierTable = ' '.join(s_DZCarrierTable)
    RecipeFile.append(s_DZCarrierTable)
    print(s_DZCarrierTable)

    # Write s_OutputDataTable
    s_OutputDataTable = []
    for row in OutputDataTable.index:
        tmp = re.findall(".{2}", '{:>08}'.format(OutputDataTable.loc[row, 'Pos']))
        tmp.reverse()
        tmp = ' '.join(tmp)
        s_OutputDataTable.append(tmp)
        tmp = re.findall(".{2}", '{:>08}'.format(OutputDataTable.loc[row, 'Neg']))
        tmp.reverse()
        tmp = ' '.join(tmp)
        s_OutputDataTable.append(tmp)
        # s_OutputDataTable.append('\r\n')
    s_OutputDataTable = ' '.join(s_OutputDataTable)
    RecipeFile.append(s_OutputDataTable)
    print(s_OutputDataTable)

    #Write s_ModeTable
    s_ModeTable = []
    CHL_TABLE = []
    LastAddress = _ChlGroupOffset + int(ChlGroupNumber, 16)
    for x in ChlGroupLength:
        CHL_TABLE.append(LastAddress)
        LastAddress += x
    print(CHL_TABLE)
    PulseWidth_t = list(PulseWidthTable.loc[:, 'PulseWidth_t'])
    DZCarrier_t = list(DZCarrierTable.loc[:, 'DZCarrier_t'])
    PulseType_t = {'PULSE_SINGLE':'00', 'PULSE_ALTERNATE':'01', 'PULSE_IN_TURN':'02', 'PULSE_ALTERNATE_REPEAT':'03'}
    BoostType_t = {'BOOST_CLOSE':'00', 'BOOST_AUTO':'01', 'BOOST_TRIA':'02', 'BOOST_SAWUP':'03',
                   'BOOST_SAWDOWN':'04', 'BOOST_UPHOLD':'05', 'BOOST_DOWNHOLD':'06'}
    AutoAdjust_t = {'AUTO_ADJUST_FRQ':'01', 'AUTO_ADJUST_VCYCLE':'02',
                    'AUTO_ADJUST_RUNTIME':'03', 'AUTO_ADJUST_IDLETIME':'04', 'AUTO_ADJUST_NONE':'00'}
    # print(ModeTable)
    for row in ModeTable.index:
        CHL_TABLEX = re.findall(".{2}", '{:>08}'.format(str(format(CHL_TABLE[int(ModeTable.loc[row, 'CHL_TABLEX'])], 'x'))))
        CHL_TABLEX.reverse()
        CHL_TABLEX = ' '.join(CHL_TABLEX)
        s_ModeTable.append(CHL_TABLEX)
        print(CHL_TABLEX)

        _Pulse = ''
        _Pulse += PulseType_t[ModeTable.iloc[row, 1]] + ' '
        tmp = re.findall(".{2}", '{:>04}'.format(str(format(ModeTable.loc[row, 'FrqHZx10'], 'x'))))
        tmp.reverse()
        _Pulse += ' '.join(tmp) + ' '
        if ModeTable.loc[row, 'WidthIndex'] in PulseWidth_t:
            _Pulse += '{:>02}'.format(str(format(PulseWidth_t.index(ModeTable.loc[row, 'WidthIndex']), 'x'))) + ' '
            _Pulse += '{:>02}'.format(str(format(PulseWidth_t.index(ModeTable.loc[row, 'WidthIndex']), 'x'))) + ' '
        else:
            print(ModeTable.loc[row, 'WidthIndex'] + ' not in PulseWidthTable')

        if ModeTable.loc[row, 'ExtraParam'] in DZCarrier_t:
            _Pulse += '{:>02}'.format(str(format(DZCarrier_t.index(ModeTable.loc[row, 'ExtraParam']), 'x')))
        else:
            print(ModeTable.loc[row, 'ExtraParam'] + ' not in DZCarrierTable')
        s_ModeTable.append(_Pulse)

        _Boost = ''
        _Boost += BoostType_t[ModeTable.iloc[row, 5]] + ' '
        tmp = re.findall(".{2}", '{:>04}'.format(str(format(ModeTable.loc[row, 'Cycle'], 'x'))))
        tmp.reverse()
        _Boost += ' '.join(tmp)
        s_ModeTable.append(_Boost)

        tmp = re.findall(".{2}", '{:>04}'.format(str(format(ModeTable.loc[row, 'RunTimeMs'], 'x'))))
        tmp.reverse()
        _RunTimeMs = ' '.join(tmp)
        s_ModeTable.append(_RunTimeMs)

        tmp = re.findall(".{2}", '{:>04}'.format(str(format(ModeTable.loc[row, 'IdleMs'], 'x'))))
        tmp.reverse()
        _IdleMs = ' '.join(tmp)
        s_ModeTable.append(_IdleMs)

        _AutoAdjust = ''
        _AutoAdjust += AutoAdjust_t[ModeTable.iloc[row, 9]] + ' '
        if ModeTable.iloc[row, 9] == 'AUTO_ADJUST_NONE':
            _AutoAdjust += '00 00 00'
        else :
            _AutoAdjust += '{:>02}'.format(str(format(ModeTable.loc[row, 'Time'], 'x'))) + ' '
            tmp = re.findall(".{2}", '{:>04}'.format(str(format(ModeTable.loc[row, 'Value'], 'x'))))
            tmp.reverse()
            _AutoAdjust += ' '.join(tmp)
        s_ModeTable.append(_AutoAdjust)

    #if has MODE_DELAY_XMS
        if ModeTable.loc[row, 'MODE_DELAY_XMS'] != '':
            DelayXms = []
            CHL_TABLEX = re.findall(".{2}", '{:>08}'.format(str(format(CHL_TABLE[0], 'x'))))
            CHL_TABLEX.reverse()
            CHL_TABLEX = ' '.join(CHL_TABLEX)
            DelayXms.append(CHL_TABLEX)
            DelayXms.append(PulseType_t['PULSE_SINGLE'])
            DelayXms.append('88 04 00 00 00 00 00 00 00 00')
            DelayValue = re.findall(".{2}", '{:>04}'.format(str(format(ModeTable.loc[row, 'MODE_DELAY_XMS'], 'x'))))
            DelayValue.reverse()
            DelayValue = ' '.join(DelayValue)
            DelayXms.append(DelayValue)
            DelayXms.append('00 00 00 00')
            # print(DelayXms)
            DelayXms = ' '.join(DelayXms)
            # s_ModeTable.append('\r\n')
            s_ModeTable.append(DelayXms)

    s_ModeTable = ' '.join(s_ModeTable)
    RecipeFile.append(s_ModeTable)

    RecipeFile = ' '.join(RecipeFile)
    RecipeFileString = RecipeFile
    print('recipe file string: ')
    print(RecipeFile)

    #Write CfgFile
    CfgFile = []
    CfgFileLength = '00 18'
    CfgFile.append(CfgFileLength)
    RecipeFileLength = re.findall(".{2}", '{:>08}'.format(str(format(len(RecipeFileString.split(' ')), 'x'))))
    RecipeFileLength = ' '.join(RecipeFileLength)
    CfgFile.append(RecipeFileLength)
    BaseWaveFileLength = '00 00 00 00'
    CfgFile.append(BaseWaveFileLength)

    CfgFileChecksum = ''
    RecipeFileChecksum = ''
    BaseWaveFileChecksum = '00 00 00 00'

    CfgChecksummedData = []

    RecipeFileComboID = '00 00 00 00'
    CfgChecksummedData.append(RecipeFileComboID)
    RecipeFileComboVersion = '00 01'
    CfgChecksummedData.append(RecipeFileComboVersion)
    RecipeFileNum = '01'
    CfgChecksummedData.append(RecipeFileNum)
    BaseWaveFileNum = '00'
    CfgChecksummedData.append(BaseWaveFileNum)

    RecipeID = re.findall(".{2}", '{:>08}'.format(str(format(options.loc[0, 'RecipeID'], 'x'))))
    RecipeID = ' '.join(RecipeID)
    CfgChecksummedData.append(RecipeID)
    RecipeVersion = re.findall(".{2}", '{:>016}'.format(str(format(options.loc[0, 'RecipeVersion'], 'x'))))
    RecipeVersion = ' '.join(RecipeVersion)
    CfgChecksummedData.append(RecipeVersion)

    RecipeLength  = re.findall(".{2}", '{:>04}'.format(str(format(len(RecipeFileString.split(' ')), 'x'))))
    RecipeLength   = ' '.join(RecipeLength)
    CfgChecksummedData.append(RecipeLength)
    BaseWaveLength = '00 00'
    CfgChecksummedData.append(BaseWaveLength)

    CfgChecksummedData = ' '.join(CfgChecksummedData)
    # print(CfgChecksummedData)
    CheckList = CfgChecksummedData.split(' ')
    CheckList = [int(i, 16) for i in CheckList]
    CfgFileChecksum = re.findall(".{2}", '{:>08}'.format(str(format(sum(CheckList), 'x'))))
    CfgFileChecksum = ' '.join(CfgFileChecksum)
    CfgFile.append(CfgFileChecksum)
    # print(CfgFileChecksum)

    CheckList = RecipeFileString.split(' ')
    CheckList = [int(i, 16) for i in CheckList]
    RecipeFileChecksum = re.findall(".{2}", '{:>08}'.format(str(format(sum(CheckList), 'x'))))
    RecipeFileChecksum = ' '.join(RecipeFileChecksum)
    CfgFile.append(RecipeFileChecksum)
    CfgFile.append(BaseWaveFileChecksum )
    CfgFile.append(CfgChecksummedData)

    CfgFile = ' '.join(CfgFile)
    CfgFileString = CfgFile

    #putting data into the dictionary and converting it to the json file
    json_dict["CfgFile"] = CfgFileString
    json_dict["RecipeFile"] = RecipeFileString
    return json.dumps(json_dict, indent=4)

    with open("./OutputRecipe.json",'w') as f:
        json.dump(json_dict, f, indent=4)
        f.close()

class Xls2Json:
    def __init__(self):
        self.xls_file = ''
        self.json_file = ''
        self.result = None

    def set_file(self, xls_file, json_file='default.json'):
        self.xls_file = xls_file
        self.json_file = json_file

    def convert(self):
        self.result = func_xls2json(self.xls_file)
        with open(self.json_file, 'w') as f:
            f.write(self.result)
            f.close()
            print('Convert Done!')

    def get_result(self):
        return self.result

if __name__ == '__main__':
    xls_file = sys.argv[1]
    json_file = sys.argv[2]
    xls2json = Xls2Json()
    xls2json.set_file(xls_file, json_file)
    xls2json.convert()


# -*- coding: utf-8 -*-
# @Date: 2020/8/5 13:58
# @Author: Du Jing
# @FileName: plot
# ---- Description ----
#

import sys

import numpy as np
import tensorflow as tf
from sklearn.metrics import confusion_matrix, classification_report
import matplotlib.pyplot as plt

import util
from dataset import Dataset
import args
from graph import model

# 加载数据集
test_data_path = Dataset(None).generate_test_data(args.test_path, args.nfeature, args.seq_length, args.test_size)
test_data = np.load(test_data_path)
testData, testLabel, testx, testy = test_data['inputs'], test_data['labels'], test_data['seqlen'], test_data['dim'],
feed_dict_test = {model.x_input: testData,
                  model.y_true: testLabel,
                  model.seqLen: testx,
                  model.dropout: 0,
                  model.training: False}

saver = tf.train.Saver()
with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    if tf.train.checkpoint_exists(args.model_save_dir):
        model_file = tf.train.latest_checkpoint(args.model_save_dir)
        saver.restore(sess, model_file)
        y_pred_test, test_acc, test_loss = sess.run([model.y_pred, model.accuracy, model.loss],
                                                    feed_dict=feed_dict_test)
        mhas = sess.run([model.mhas], feed_dict_test)[0]
        plt.figure(dpi=300, figsize=(7.6, 12.8))
        for i, att in enumerate(mhas):
            print(mhas[i][0].shape)
            plt.subplot(6, 1, i+1)
            plt.imshow(np.transpose(att[0]))
        plt.show()

        print(classification_report(y_true=testLabel, y_pred=y_pred_test))
        matrix = confusion_matrix(y_true=testLabel, y_pred=y_pred_test)
        print(matrix)
        if args.plot_matrix:
            util.plot_confusion_matrix(matrix, args.classes, args.figure_title, args.figure_save_path)
    else:
        sys.stderr.write('Checkpoint Not Found!')
        sys.exit(1)


'''
xplsek03 | 2021 | FIT VUT Brno, bakalarska prace Razeni manipulaci pro morici linky
'''

from task.sbh.graph import Graph
from task.bab.edd import Edd
import logging
#from task.output.timer import timer

'''
Funkce pro provadeni Shifting bottleneck heuristiky.
'''
class Sbh:

    '''
    Inicializace harmonogramu nad jeraby a vanami.
    @param vany - seznam van
    @param jeraby - seznam jerabu
    @param clen - jedinec populace
    @return tuple se slovniky harmonogramu
    '''
    def init_harmonogramy(vany, jeraby, clen):

        # harmogram zpracovani na vanach: {id_vany: [operace zpracovavane na teto vane], .. }
        harmonogram_vany = {}

        for vana in vany:
            # filtruj podle konkretniho id vany: seznam operaci, ktere se provadi na jedne vane
            filtered = list(filter(lambda x: (x.id_stroj == vana['_id']), clen['namaceni']))

            # pridej do seznamu harmonogramu, pokud se ve vane bude neco namacet. DEBUG: promenna, self vany je overkill?
            if len(filtered):
                harmonogram_vany[vana['_id']] = filtered

        # harmogram zpracovani na jerabech: {id_stroj: [operace zpracovavane na tomto jerabu], .. }
        harmonogram_jeraby = {}
        # sestav plan na jednotlivych jerabech
        for jerab in jeraby:
            filtered = list(filter(lambda x: (x.id_stroj == jerab), clen['transporty']))
            # pridej do seznamu harmonogramu pokud na jerabu bude neco prepravovaneho. DEBUG: mozna pouzit jinou promennou, overkill
            if len(filtered):
                harmonogram_jeraby[jerab] = filtered

        return (harmonogram_jeraby, harmonogram_vany)

    '''
    SBH - shifting bottleneck heuristika.
    @param graf - graf nad kterym se ma provadet, jednoduchy G nebo uplny G'
    @param slovnik_van
    @param slovnik_produktu
    @param harmonogram - slovnik, klic: stroj hodnota: seznam operaci na stroji
    @param clen - jedinec populace
    @param simple - G = True, G' = False
    @param slovnik_jerabu
    @return cmax nebo None
    '''
    def sbh(graf, slovnik_van, slovnik_produktu, harmonogram, clen, simple=True, slovnik_jerabu=None):

        # pro simple poprve uz mas pridanou jednu vanu v m0, tedy prvni prvek harmonogramu zahod pro simple
        first = True

        # postupne planuj cleny harmonogramu, v jednom harmonogramu je seznam namaceni ruznych jobu v jedne vane nebo seynam transportu
        for m_id, m in harmonogram.items():

            # pouze simple, skipni EDD u 1. vany - pridas jak to je
            if first and simple:
                # proved vypocet fake EDD
                useky = Sbh.fake_edd(m, m_id, slovnik_van, graf)
                # nahrej do grafu disjunktivni spojnice prvni vany, id vem z harmonogram (harmonogram_vany)
                Graph.add_disjunctive(graf, useky, slovnik_van, slovnik_jerabu, simple)
                first = False
                continue

            # v kazde iteraci pocitas vlastni cmax
            cmax, pozitivni_cyklus = Graph.bellman(graf, node_from="start", node_to="end")
            if pozitivni_cyklus:
                return None

            # proved branch and bound pro konkretni stroj. best = Node() | None pokud to nenajde reseni
            node = Edd.branch_and_bound(slovnik_van, slovnik_jerabu, graf, m, cmax, is_namaceni=simple)

            # BaB nenasel vysledek - chyba aproximace
            if not node:
                logging.debug('CHYBA BAB')
                return None

            # vloz do grafu ziskane spojnice - o hodnote exp_max te vany nebo LT + ET u uplneho grafu
            Graph.add_disjunctive(graf, node.casove_useky, slovnik_van, slovnik_jerabu, simple)

            # over na pozitivni cyklus
            cmax, pozitivni_cyklus = Graph.bellman(graf, node_from="start", node_to="end")
            if pozitivni_cyklus:
                # odted vime ze obsahuje pozitivni cyklus
                # odeber pridane disjunktivni hrany
                Graph.remove_disjunctive(graf, node.casove_useky)
                # vytvor LB a UB pro kazdy prvek, rozmezi v jakem muze swapovat s dalsim prvkem
                Sbh.init_lb_ub(node.casove_useky)
                # postupne prehazuj dva prvky dokud nenajdes odpovidajici harmonogram na stroji, co netvori pozitivni cyklus
                cmax = Graph.sbh_reparation(graf, node.casove_useky, slovnik_van, slovnik_jerabu, simple)
                if not cmax:
                    return None

            # updatuj - serad podle noveho - zaznam harmonogramu, kvuli velkemu grafu - aby vedel ktere disjunktivni aktualni spojnice ma vytvaret
            mapa = [i['stroj'].graphid for i in node.casove_useky]
            m.sort(key=lambda x: mapa.index(x.graphid))

        # vrat vypocitany cmax
        return cmax

    '''
    Pomocna funkce pro generovani poradi upper a lower bounds, ktere budou respektovany
    pri prehazovani kombinace prvku v pripade opravy sekvence v SBH v kroku 4.
    @param useky - casove useky vracene z Node()
    '''
    def init_lb_ub(useky):

       for j in range(len(useky)):
            # jdi doleva, defaultne nastav 0
            useky[j]['lb'] = 0
            it = 0
            for l in reversed(useky[:j]):
                if l['stroj'].id_produkt == useky[j]['stroj'].id_produkt:
                    useky[j]['lb'] = j - it
                    break
                it += 1
            # jdi doprava
            useky[j]['ub'] = len(useky) - 1
            for i, r in enumerate(useky[j:]):
                if r['stroj'].id_produkt == useky[j]['stroj'].id_produkt:
                    useky[j]['ub'] = j + i

    '''
    Pridani operaci prvniho stroje bez provadeni EDD. Pouze u jednoducheho grafu.
    @param m - seznam operaci provadenych na stroji
    @param id - id stroje
    @param slovnik_van
    @param graf - jednoduchy graf G'
    @return vygenerovany seznam casovych useku
    '''
    def fake_edd(m, id, slovnik_van, graf):

        # finalni useky
        useky = []
        # projdi seznam polozek
        for item in m:
            rj, pozitivni_cyklus = Graph.bellman(graf, node_from="start", node_to=item.graphid)
            if pozitivni_cyklus:
                return None
            pj = slovnik_van[id]['exp_max']

            useky.append({'start': rj, 'end': rj + pj, 'stroj': item})

            # je potreba useky projit a nastavit precedence v ramci produktu, aby nektere - pokud jsou - nebezely zaraz na stejne vane
            m_index = 0  # index v ramci m, pokud te zajima prava polozka tak je to m_index+1
            for left,right in zip(useky[:-1], useky[1:]):

                # pokud ten co ma nasledovat za nim prekryva cas, posun ho
                if right['start'] < left['end']:
                    delta = left['end'] - right['start']
                    right['start'] += delta
                    right['end'] += delta

                try:
                    # predchazejici uzel, muze byt v jine vane
                    prev = m[m_index+1].prev.prev
                    # pokud by se ta operace nevesla do mezery mezi temi co jsi posunul, musis zvetsit mezeru
                    if slovnik_van[prev.id_stroj]['exp_max'] > right['start'] - left['end']:
                        delta = left['end'] + slovnik_van[prev.id_stroj]['exp_max'] - right['start']
                        right['start'] += delta
                        right['end'] += delta

                except AttributeError:
                    pass

                m_index += 1

        return useky


from flask import Flask, request, jsonify, abort, redirect

from .exceptions import MissingParameters
from .info import package_info
from .parse import parse_environment

from conda.exceptions import ResolvePackageNotFound

def create_app():
    app = Flask(__name__)

    @app.route("/")
    def index():
        return "OK"

    @app.route("/package")
    def package():
        name = request.args.get("name")  # Support package, or name being key
        if not name:
            raise MissingParameters

        channel = request.args.get("channel", "pkgs/main")
        version = request.args.get("version", "")  # Optional
        _pkg = package_info(channel, name, version)
        if request.args.get("download"):
            return redirect(_pkg["url"])
        else:
            return jsonify(_pkg), 200

    @app.route("/parse", methods=["POST"])
    def parse():
        """
            Page for posting a file to to get the Conda dependencies back.
            Solves a conda environment

            POST Parameters two options:
                multipart/form-data:
                    file: an environment.y(a)ml file
                application/x-www-form-urlencoded:
                    file: the text of the environment file
                    filename: the filename (needs to be .yml or .yaml)
            Query Parameters:
                [force_solve=1]
                    existance of any value will cause this flag to be set and solve
                    whether there is a lock file or not.
            Returns:
                json with "error" or with "dependencies"/"channels"
        """
        force_solve = bool(request.args.get("force_solve", False))

        # get the file from either files or form
        if request.content_type.startswith("application/x-www-form-urlencoded"):
            body = request.form.get("file")
            filename = request.form.get("filename")
        elif request.content_type.startswith("multipart/form-data"):
            f = request.files.get("file")
            filename = f.filename if hasattr(f, "filename") else f.name
            body = f.read()

        return jsonify(parse_environment(filename, body, force_solve)), 200

    @app.errorhandler(ResolvePackageNotFound)
    def not_found(e):
        message = f"Error: Package(s) not found: {e}"
        return jsonify(error=404, text=message), 404

    @app.errorhandler(MissingParameters)
    def missing_params(e):
        message = f"Error: Please provide a `name=` query parameter"
        return jsonify(error=404, text=message), 404

    return app

def main(debug=False):
    app = create_app()
    app.run(host="0.0.0.0", debug=debug)


from Yukki import app, SUDOERS
from pyrogram import filters, Client
from pyrogram.types import Message
from Yukki.YukkiUtilities.database.onoff import (is_on_off, add_on, add_off)
from ..YukkiUtilities.helpers.filters import command

@Client.on_message(command("maintenance") & filters.user(SUDOERS))
async def smex(_, message):
    usage = "**usage:**\n/maintenance [enable|disable]"
    if len(message.command) != 2:
        return await message.reply_text(usage)
    chat_id = message.chat.id
    state = message.text.split(None, 1)[1].strip()
    state = state.lower()
    if state == "enable":
        user_id = 1
        await add_on(user_id)
        await message.reply_text("✅ maintenance mode enabled\n\n• from now on, user can't play music after the maintenance mode is disabled.")
    elif state == "disable":
        user_id = 1
        await add_off(user_id)
        await message.reply_text("❌ maintenance mode disabled\n\n• from now on, user can play music again.")
    else:
        await message.reply_text(usage)

@Client.on_message(command("sptest") & filters.user(SUDOERS))
async def sls_skfs(_, message):
    usage = "**usage:**\n/speedtest [enable|disable]"
    if len(message.command) != 2:
        return await message.reply_text(usage)
    chat_id = message.chat.id
    state = message.text.split(None, 1)[1].strip()
    state = state.lower()
    if state == "enable":
        user_id = 2
        await add_on(user_id)
        await message.reply_text("✅ **speedtest enabled**")
    elif state == "disable":
        user_id = 2
        await add_off(user_id)
        await message.reply_text("❌ **speedtest disabled**")
    else:
        await message.reply_text(usage)


import dateutil.parser
from django.contrib.auth import get_user_model
from django.db.models import Q
from django.http import Http404
from django.shortcuts import get_object_or_404
from django.utils.translation import ugettext as _
from rest_framework import status, viewsets
from rest_framework.decorators import detail_route
from rest_framework.permissions import AllowAny
from rest_framework.response import Response
from rest_framework.views import APIView

from services.rest.permissions import (SafeMethodsOnlyPermission,
                                       UserCanAddCompanyPermission,
                                       UserCanEditCompanyPermission,
                                       UserIsOperatorPermission)
from services.rest.serializers import (AddCompanySerializer, CompanySerializer,
                                       CountrySerializer, OperatorSerializer,
                                       OptionHolderSerializer,
                                       OptionPlanSerializer,
                                       OptionTransactionSerializer,
                                       PositionSerializer, SecuritySerializer,
                                       ShareholderSerializer, UserSerializer,
                                       UserWithEmailOnlySerializer)
from shareholder.models import (Company, Country, Operator, OptionPlan,
                                OptionTransaction, Position, Security,
                                Shareholder)

User = get_user_model()

# --- VIEWSETS

class ShareholderViewSet(viewsets.ModelViewSet):
    """
    API endpoint that allows users to be viewed or edited.
    """
    # FIXME filter by user perms
    serializer_class = ShareholderSerializer
    permission_classes = [
        UserIsOperatorPermission,
    ]

    def get_object(self):
        try:
            return Shareholder.objects.get(pk=self.kwargs.get('pk'))
        except Shareholder.DoesNotExist:
            raise Http404

    def get_queryset(self):
        user = self.request.user
        return Shareholder.objects.filter(company__operator__user=user)\
            .distinct()

    @detail_route(methods=['get'])
    def number_segments(self, request, pk=None):
        shareholder = self.get_object()
        kwargs = {}

        if request.GET.get('date'):
            kwargs.update({'date': request.GET.get('date')[:10]})

        data = {}
        for security in shareholder.company.security_set.all():
            if security.track_numbers:
                kwargs.update({'security': security})
                data.update({
                    security.pk: shareholder.current_segments(**kwargs)
                })
        return Response(data, status=status.HTTP_200_OK)

class OperatorViewSet(viewsets.ModelViewSet):
    """
    API endpoint that allows users to be viewed or edited.
    """
    # FIXME filter by user perms
    serializer_class = OperatorSerializer
    permission_classes = [
        UserIsOperatorPermission,
    ]

    def get_object(self, pk):
        try:
            return Operator.objects.get(pk=pk)
        except Operator.DoesNotExist:
            raise Http404

    def get_queryset(self):
        user = self.request.user
        return Operator.objects.filter(company__operator__user=user)\
            .distinct()

    def destroy(self, request, pk=None):
        operator = self.get_object(pk)
        company_ids = request.user.operator_set.all().values_list(
            'company__id', flat=True).distinct()
        # cannot remove himself
        if operator not in request.user.operator_set.all():
            # user can only edit corps he manages
            if operator.company.id in company_ids:
                operator.delete()
                return Response(status=status.HTTP_204_NO_CONTENT)
        # else:
        return Response(status=status.HTTP_403_FORBIDDEN)

class CompanyViewSet(viewsets.ModelViewSet):
    """
    API endpoint that allows users to be viewed or edited.
    """
    # FIXME filter by user perms
    queryset = Company.objects.all()
    serializer_class = CompanySerializer
    permission_classes = [
        UserCanEditCompanyPermission,
    ]

    def get_queryset(self):
        user = self.request.user
        return Company.objects.filter(operator__user=user)

    # FIXME add perms like that to decor. permission_classes=[IsAdminOrIsSelf]
    @detail_route(methods=['post'])
    def upload(self, request, pk=None):
        obj = self.get_object()
        obj.logo = request.FILES['logo']
        obj.save()
        return Response(CompanySerializer(
            obj,
            context={'request': request}).data,
            status=status.HTTP_201_CREATED)

    @detail_route(methods=['get'])
    def option_holder(self, request, pk=None):
        """ returns the captable part for all option holders """
        obj = self.get_object()
        ohs = obj.get_active_option_holders()
        page = self.paginate_queryset(ohs)
        if page is not None:
            serializer = OptionHolderSerializer(
                page, many=True, context={'request': request})
            return self.get_paginated_response(serializer.data)
        serializer = OptionHolderSerializer(
            ohs, many=True, context={'request': request})
        return Response(serializer.data)

# --- VIEWS

class AddCompanyView(APIView):
    """ view to initially setup a company """

    queryset = Company.objects.none()
    permission_classes = [
        UserCanAddCompanyPermission,
    ]

    def post(self, request, format=None):
        serializer = AddCompanySerializer(data=request.data)
        if serializer.is_valid() and request.user.is_authenticated():
            serializer.save(user=request.user)
            return Response(serializer.data, status=status.HTTP_201_CREATED)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

class AddShareSplit(APIView):
    """ creates a share split. danger: can be set to be in the past and ALL
    following transactions must be adjusted
    """
    permission_classes = [
        UserCanAddCompanyPermission,
    ]

    def _validate_data(self, data):
        errors = {}
        if not data.get('execute_at'):
            errors.update({'execute_at': [_('Field may not be empty.')]})
        if not data.get('dividend'):
            errors.update({'dividend': [_('Field may not be empty.')]})
        if not data.get('divisor'):
            errors.update({'divisor': [_('Field may not be empty.')]})
        if not data.get('security'):
            errors.update({'security': [_('Field may not be empty.')]})

        if not errors:
            return True, {}
        else:
            return False, errors

    def post(self, request, fomat=None):
        data = request.data
        is_valid, errors = self._validate_data(data)
        if is_valid:
            # get company and run company.split_shares(data)
            company = request.user.operator_set.earliest('id').company
            data.update({
                'execute_at': dateutil.parser.parse(data['execute_at']),
                'security': Security.objects.get(id=data['security']['pk'])
            })
            company.split_shares(data)

            positions = Position.objects.filter(
                buyer__company__operator__user=request.user).order_by(
                    '-bought_at')

            serializer = PositionSerializer(
                positions, many=True, context={'request': request})
            return Response(
                {'success': True, 'data': serializer.data},
                status=status.HTTP_201_CREATED)

        return Response(errors, status=status.HTTP_400_BAD_REQUEST)

class AvailableOptionSegmentsView(APIView):

    permission_classes = [UserIsOperatorPermission]

    def get(self, request, optionsplan_id, shareholder_id):
        """
        returns available option segments from shareholder X and
        optionplan Y
        """
        optionplan = get_object_or_404(OptionPlan, pk=optionsplan_id)
        shareholder = get_object_or_404(Shareholder, pk=shareholder_id)

        kwargs = {
            'security': optionplan.security,
            'optionplan': optionplan,
        }

        if request.GET.get('date'):
            kwargs.update({'date': request.GET.get('date')[:10]})

        return Response(shareholder.current_options_segments(**kwargs))

class LanguageView(APIView):
    """
    Endpint delivering language options
    """
    permission_classes = (AllowAny,)

    def get(self, *args, **kwargs):
        from django_languages.languages import LANGUAGES
        languages = []
        for language in LANGUAGES:
            languages.append({
                'iso': language[0],
                'name': language[1],
            })
        return Response(languages)

class UserViewSet(viewsets.ModelViewSet):
    """ API endpoint to get user base info """
    serializer_class = UserSerializer
    permission_classes = [
        SafeMethodsOnlyPermission,
    ]

    def get_queryset(self):
        user = self.request.user
        return User.objects.filter(id=user.id)

class CountryViewSet(viewsets.ModelViewSet):
    """ API endpoint to get user base info """
    serializer_class = CountrySerializer
    permission_classes = [
        SafeMethodsOnlyPermission,
    ]

    def get_queryset(self):
        countries = Country.objects.all()
        return countries

class InviteeUpdateView(APIView):
    """ API endpoint to get user base info """
    permission_classes = (AllowAny,)

    def post(self, request, format=None):

        serializer = UserWithEmailOnlySerializer(data=request.data)
        if serializer.is_valid():
            serializer.save(username=serializer.validated_data['email'])
            return Response(serializer.data, status=status.HTTP_201_CREATED)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

class PositionViewSet(viewsets.ModelViewSet):
    """ API endpoint to get positions """
    serializer_class = PositionSerializer
    permission_classes = [
        UserIsOperatorPermission,
    ]

    def get_queryset(self):
        user = self.request.user
        return Position.objects.filter(
            Q(buyer__company__operator__user=user) |
            Q(seller__company__operator__user=user)
        ).distinct().order_by('-bought_at', '-pk')

    @detail_route(
        methods=['post'], permission_classes=[UserIsOperatorPermission])
    def confirm(self, request, pk=None):
        """ confirm position and make it unchangable """
        position = self.get_object()
        position.is_draft = False
        position.save()
        return Response({"success": True}, status=status.HTTP_200_OK)

    def destroy(self, request, pk=None):
        """ delete position. but not if is_draft-False"""

        position = self.get_object()
        if position.is_draft is True:
            position.delete()
            return Response(
                {"success": True}, status=status.HTTP_204_NO_CONTENT)
        else:
            return Response(
                {
                    'success': False,
                    'errors': [_('Confirmed position cannot be deleted.')]
                },
                status=status.HTTP_400_BAD_REQUEST)

class SecurityViewSet(viewsets.ModelViewSet):
    """ API endpoint to get options """
    serializer_class = SecuritySerializer
    permission_classes = [
        UserIsOperatorPermission,
    ]

    def get_queryset(self):
        user = self.request.user
        return Security.objects.filter(company__operator__user=user)

class OptionPlanViewSet(viewsets.ModelViewSet):
    """ API endpoint to get options """
    serializer_class = OptionPlanSerializer
    permission_classes = [
        # UserCanAddOptionPlanPermission,
    ]

    def get_queryset(self):
        user = self.request.user
        return OptionPlan.objects.filter(company__operator__user=user)

    # FIXME add perms like that to decor. permission_classes=[IsAdminOrIsSelf]
    @detail_route(methods=['post'])
    def upload(self, request, pk=None):
        op = self.get_object()
        # modify data
        serializer = OptionPlanSerializer(data=request.data)
        # add file to serializer
        if serializer.is_valid():
            op.pdf_file = request.FILES['pdf_file']
            op.save()
            op.generate_pdf_file_preview()
            return Response(OptionPlanSerializer(
                op,
                context={'request': request}).data,
                status=status.HTTP_201_CREATED)
        else:
            return Response(
                serializer.errors,
                status=status.HTTP_400_BAD_REQUEST)

class OptionTransactionViewSet(viewsets.ModelViewSet):
    """ API endpoint to get options """
    serializer_class = OptionTransactionSerializer
    permission_classes = [
        UserIsOperatorPermission
    ]

    def get_queryset(self):
        user = self.request.user
        return OptionTransaction.objects.filter(
            option_plan__company__operator__user=user)

    @detail_route(
        methods=['post'], permission_classes=[UserIsOperatorPermission])
    def confirm(self, request, pk=None):
        """ confirm position and make it unchangable """
        position = self.get_object()
        position.is_draft = False
        position.save()
        return Response({"success": True}, status=status.HTTP_200_OK)

    def destroy(self, request, pk=None):
        """ delete position. but not if is_draft-False"""

        position = self.get_object()
        if position.is_draft is True:
            position.delete()
            return Response(
                {"success": True}, status=status.HTTP_204_NO_CONTENT)
        else:
            return Response(
                {
                    'success': False,
                    'errors': [_('Confirmed position cannot be deleted.')]
                },
                status=status.HTTP_400_BAD_REQUEST)


import tensorflow as tf
from tensorflow import keras
import tensorflow_datasets as tfds
from numpy import load, unique
from numpy import argmax
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from plot_acc_loss import plot_acc_loss
from tensorflow.keras.applications import VGG16
import yaml
from confusion_matrix_metrics import convert_prob , conf_mat, metrics
from sklearn.metrics import roc_curve,roc_auc_score,confusion_matrix, accuracy_score,recall_score,f1_score,precision_score,plot_confusion_matrix,ConfusionMatrixDisplay,precision_recall_curve
import matplotlib.pyplot as plt
import pandas as pd
from sklearn.model_selection import train_test_split

config = None
with open('config.yml') as f: # reads .yml/.yaml files
    config = yaml.load(f)

dataset_dir  = config['net']['dir']
BATCH_SIZE = config['train']['bs']

validation_datagen = ImageDataGenerator(rescale=1./255)
new_model = keras.models.load_model(r"D:\ai intro\Munca_v2\Experimente Confusion Matrix\big dataset\19\damn2226_01082022.h5")
new_model.summary()

test_generator = validation_datagen.flow_from_directory(dataset_dir + '/test',
                                                        target_size=config['net']['img'],
                                                        batch_size=config['train']['bs'],
                                                        class_mode='binary',
                                                        shuffle=False)
print (test_generator)
probs=new_model.predict_generator(test_generator)

#print(y_test.shape,probs.shape)

confusion_matrix_manual=conf_mat(probs,test_generator.classes)
preds=convert_prob(probs)
cm=confusion_matrix(preds,test_generator.classes)
print("Matrice de confuzie calculata manual : ", confusion_matrix_manual,"si caculata cu functia sklearn:",cm)

#accuracy,senzitivity,specifity,precision,FPR,f1=metrics(confusion_matrix_manual,test_generator.classes)
#print("ACC:",accuracy,"TPR:",senzitivity,"TNR:",specifity,"PPV:" ,precision,"FPR:",FPR,"f1:",f1)

acc=accuracy_score(preds,test_generator.classes)
preci=precision_score(preds,test_generator.classes)
reca=recall_score(preds,test_generator.classes)
F1=f1_score(preds,test_generator.classes)
print("acc",acc,"PPV",preci,"FPR",reca,"f1:",F1)

ConfusionMatrixDisplay.from_predictions(test_generator.classes, preds)
plt.show()

print(test_generator.classes[:5], type(test_generator.classes))
print(probs[:5])

fpr, tpr, thresholds = roc_curve(test_generator.classes, probs, pos_label=1)
plt.plot(fpr,tpr, marker='.', label='ROC_CURVE')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.legend()
plt.show()

auc = roc_auc_score(test_generator.classes,  probs)
print('AUC: %.3f' % auc)

precision, recall, thresholds = precision_recall_curve(test_generator.classes, probs)
plt.plot(precision,recall, marker='.', label='Precision_recall')
plt.xlabel('Precision')
plt.ylabel('Recall')
plt.legend()
plt.show()

test_loss, test_acc = new_model.evaluate_generator(test_generator, steps=len(test_generator))
print('test acc:', test_acc)


import argparse

parser = argparse.ArgumentParser(description='Counting cookies.')
parser.add_argument('filename', type=str)
parser.add_argument('-d', type = str)
args = parser.parse_args()

try:
  file = open(args.filename, "r")
except:
  print("Invalid file")

lineList = [] #List of strings of every line in the file.
for line in file:
  lineList.append(line)

cookieList = [] #Filtered list of cookies that have correct timestamp
for line in lineList: #Processing lines
  tempArr = line.split(',')
  cookie = tempArr[0]
  timeStamp = tempArr[1]
  timeStamp_filt = timeStamp[:10]
  if timeStamp_filt == args.d: #Timestamp matches argument given
    cookieList.append(cookie)

highcount = 0 #Highest number of cookies
cookieCounts = {} #Dictionary with key = cookie, value = count of number of times that cookie appears.
for cookie in cookieList:
  if cookie in cookieCounts:
    cookieCounts[cookie] += 1
  else:
    cookieCounts[cookie] = 1
  highcount = max(highcount, cookieCounts[cookie])
for cookie in cookieCounts:
  if cookieCounts[cookie] == highcount: #Printing all cookies with highest count
    print(cookie)


from . import views
from django.conf.urls import url

urlpatterns = [
    url(r'^profile/', views.ProfileView.as_view(),
        name="ProfilePage"),
    url(r'^feeds/', views.FeedsView.as_view(),
        name="FeedsPage"),
    url(r'^location/', views.LocationView.as_view(),
        name="LocationPage")
]


from django.conf.urls import url

from . import views

urlpatterns = [
    url(r'^$', views.index, name='index'),
    url(r'^api/secrets$', views.getSecrets, name='secrets'),
    url(r'^api/synthesize$', views.synthesize, name='synthesize'),
    url(r'^program/(\w*)$', views.index, name='program'),
    url(r'^about/$', views.about, name='about'),
    url(r'^faq/$', views.faq, name='faq'),
    url(r'^feed/$', views.feed, name='feed'),
    url(r'^discussion/(\w*)$', views.discussion, name='discussion'),
    url(r'^profile/$', views.profile, name='profile'),
    url(r'^profile/edit$', views.profile_edit, name='profile_edit'),
    url(r'^profile/(\w*)$', views.profile, name='profile'),
]


from model import FullyConnectedNetwork
from utils import download_data
import matplotlib.pyplot as plt

# add foundations.set_tensorboard_logdir() code here

# download data
train_df, test_df = download_data()

# prepare data
input_size = len(train_df.columns) - 1 # don't include the target when counting inputs
numeric_columns = ['machine_hours_current_meter', 'age_in_years', 'target']
categorical_sizes = {col: train_df[col].nunique() for col in train_df.columns if col not in numeric_columns}

# define hyperparameters
# replace following with foundations.load_parameters()
hyperparameters = {'n_epochs': 5,
                   'batch_size': 128,
                   'validation_percentage': 0.1,
                   'dense_blocks': [{'size': 256, 'dropout_rate': 0}],
                   'embedding_factor': 0.5,
                   'learning_rate':0.0001,
                   'lr_plateau_factor':0.1,
                   'lr_plateau_patience':3,
                   'early_stopping_min_delta':0.001,
                   'early_stopping_patience':5}

# train
model = FullyConnectedNetwork(input_size, hyperparameters, categorical_sizes)
hist = model.train(train_df)

val_mse_history = hist.history['val_mean_squared_error']
plt.plot(list(range(1, len(val_mse_history)+1)), val_mse_history)
plt.savefig('plots/validation_mse.png')

# add foundations.save_artifact() code here

# evaluate model performance on test set
mse = model.evaluate(test_df)
print("test mean squared error: ", mse)

# add foundations.log_metric() code here


import tweepy
import numpy as np
from textblobing import TextBlob
from flask import Flask, render_template, request,redirect,session,jsonify
from cleantext import clean_tweet

app=Flask(__name__,static_folder="static")

consumer_key = 'your consumer key'
consumer_secret='your consumer secret'

access_token='your access token'
access_token_secret='your access token secret'

auth=tweepy.OAuthHandler(consumer_key,consumer_secret)
auth.set_access_token(access_token,access_token_secret)

api=tweepy.API(auth)

# def home():
#     return render_template('index.html')
@app.route("/analysis",methods=['POST'])
def tweet_analysis():
    if request.method== "POST":
        ques=request.form['ques']
    query=ques + "-filter:retweets"
    tweets=api.search_tweets(q=query,lang='en',count=5,result_type='popular')

    subjectivities= []
    polarities=[]
    vals=[]

    print_result="none"
    for tweet in tweets:
        tweet.text=clean(tweet.text, no_emoji=True)
        phrase=TextBlob(tweet.text)
        if phrase.sentiment.polarity != 0.0 and phraase.sentiment.subjectivity !=0.0:
            polarities.append(phrase.sentiment.polarity)
            subjectivities.append(phrase.sentiment.subjectivity)
            vals.append(tweet.text)

        print('Tweet: '+tweet.text)
        print('Polarity: '+ str(phrase.sentiment.polarity) + '\Subjectivity: '+ str(phrase.sentiment.subjectivity))
        print('..........................')

    result= {'polarity':polarities,'subjectivity':subjectivities}

    get_weighted_polarity_mean= np.average(resukt['polarity'], weights=result['subjectivity'])

    get_polarity_mean=np.mean(result['polarity'])

    if get_polarity_mean >0.0:
        print_result="POSITIVE"
    elif get_polarity_mean ==0.0:
        print_result="NEUTRAL"
    else:
        print_result="NEGATIVE"

    RES= {'polarity':polarities,
          'subjectivity':subjectivities,
          'values':vals,
          'get_weighted_polarity_mean':get_weighted_polarity_mean,
          'get_polarity_mean':get_polarity_mean,
          'RESULT': print_result
          }

    return jsonify(RES)

if __name__ == "__main__":
    app.run(debug=True)


import configparser
import time
from MainClass import EsModules
from datetime import datetime

runtime = datetime.now().strftime('%Y-%m-%d %H-%M-%S')
config = configparser.ConfigParser()
config.read('es_test_config.ini')
ES_environment = config['DEFAULT']['environment']
location_file= config['DEFAULT']['testlib']
# print(ES_environment)

#parsed_info = json.dumps((connection_es.info()), indent=4)
#print(parsed_info)

# search term entered in autosuggest
#search_term = input("Enter Search Term: ")
f=open(location_file,'r',encoding='utf-8')
search_term = f.readlines()
f.close()

# language code used in location alias
alias_lan_code = config['DEFAULT']['languagecode']

# country context value for query
country_ctx = config['DEFAULT']['country_context']

# designates if the index is new
is_New = config['DEFAULT'].getboolean('is_New')

if is_New:
    test_index = "-new"
else:
    test_index = ""
x=0
for line in search_term:
    ssearch_term = search_term[x].strip('\n')
    res = EsModules.es_location_suggest(ES_environment, alias_lan_code, test_index, ssearch_term, country_ctx)
    print(ssearch_term)
    EsModules.query_output(res, ssearch_term,alias_lan_code,ES_environment, runtime)
    x = x + 1
    time.sleep(0.01)


# -----------------------------------------------------------------------------
# download_AMBA.py
# Authors: Antoine Beauchamp, Yohan Yee

"""
Download in-situ hybridization data sets from the Allen Mouse Brain Atlas
"""

# Packages --------------------------------------------------------------------

import os
import argparse
import requests
import numpy                as np
import pandas               as pd
import multiprocessing      as mp
from pyminc.volumes.factory import volumeFromFile, volumeFromDescription
from zipfile                import ZipFile
from functools              import partial
from tqdm                   import tqdm

# Functions -------------------------------------------------------------------

def parse_args():

    """Parse command line arguments"""

    parser = argparse.ArgumentParser(
                 formatter_class = argparse.ArgumentDefaultsHelpFormatter
             )

    parser.add_argument(
        '--dataset',
        type = str,
        default = 'coronal',
        choices = ['coronal', 'sagittal'],
        help = 'AMBA dataset to import.'
    )

    parser.add_argument(
        '--outdir',
        type = str,
        default = 'expression/',
        help = 'Path to directory in which to download the data.'
    )

    parser.add_argument(
        '--metadata',
        type = str,
        default = 'metadata.csv',
        help = 'File in --outdir containing AMBA metadata.'
    )

    parser.add_argument(
        '--parallel',
        type = str,
        default = 'false',
        choices = ['true', 'false'],
        help = 'Option to run in parallel.'
    )

    parser.add_argument(
        '--nproc',
        type = int,
        default = mp.cpu_count(),
        help = 'Number of CPUs to use in parallel.'
    )

    parser.add_argument(
        '--verbose',
        type = str,
        default = 'true',
        choices = ['true', 'false'],
        help = 'Verbosity.'
    )

    args = vars(parser.parse_args())

    return args

def fetch_metadata(dataset = 'coronal', outdir='./',
                   outfile = 'metadata.csv', verbose = True):

    """
    Download metadata for in-situ hybridization data sets

    Arguments
    ---------
    dataset: str, optional
        String indicating whether to download the coronal or sagittal
        (default 'coronal')
    outdir: str, optional
        Directory in which to download the metadata.
        (default './')
    outfile: str, optional
        Name of csv file in which to save the metadata.
        (default 'metadata.csv')

    Returns
    -------
    None
    """

    abi_query_metadata = ("http://api.brain-map.org/api/v2/data/SectionDataSet/"
                          "query.csv?criteria="
                          "[failed$eqfalse],"
                          "plane_of_section[name$eq{}],"
                          "products[abbreviation$eqMouse],"
                          "treatments[name$eqISH],"
                          "genes"
                          "&tabular="
                          "data_sets.id+as+experiment_id,"
                          "data_sets.section_thickness,"
                          "data_sets.specimen_id,"
                          "plane_of_sections.name+as+plane,"
                          "genes.acronym+as+gene,"
                          "genes.name+as+gene_name,"
                          "genes.chromosome_id,"
                          "genes.entrez_id,"
                          "genes.genomic_reference_update_id,"
                          "genes.homologene_id,"
                          "genes.organism_id"
                          "&start_row=0"
                          "&num_rows=all".format(dataset))

    (pd.read_csv(abi_query_metadata)
        .drop_duplicates()
        .to_csv(outdir+outfile, index=False))

    if verbose:
        print('Metadata downloaded at: {}'.format(outdir+outfile))

    return

def fetch_expression(experiment_id, outdir = './tmp/'):

    """
    Download the expression energy for an ISH experiment

    Description
    -----------
    This function downloads the gridded expression energy for a single
    in-situ hybridization experiment in the Allen Mouse Brain Atlas.
    The downloaded data is in RAW format and is stored in a file named
    according to the experiment ID.

    Arguments
    ---------
    experiment_id: int
        The ID of the in-situ hybridization experiment to download.
    outdir: str, optional
        Directory in which to download the expression energy file.
        (default './tmp/')

    Returns
    -------
    outfile: str
        The name of the file containing the expression energy.
    success: int
        Integer indicating whether the download was successful.
    """

    if not os.path.exists(outdir):
        os.mkdir(outdir)

    tmpdir = outdir+str(experiment_id)+'/'
    os.mkdir(tmpdir)

    abi_query_expr = ('http://api.brain-map.org/grid_data/download/{}'
                      .format(experiment_id))
    amba_request = requests.get(abi_query_expr)

    tmpfile = tmpdir+str(experiment_id)+'.zip'
    with open(tmpfile, 'wb') as file:
        file.write(amba_request.content)

    outfile = outdir+str(experiment_id)+'.raw'
    with ZipFile(tmpfile, 'r') as file:
        try:
            file.extract('energy.raw', path = tmpdir)
            os.rename(tmpdir+'energy.raw', outfile)
            success = 1
        except KeyError as err:
            print('Error for experiment {}: {}. Ignoring.'
                  .format(experiment_id, err))
            success = 0

    os.remove(tmpfile)
    os.rmdir(tmpdir)

    return outfile, success

def rawtominc_wrapper(infile, outfile = None, keep_raw = False):

    """
    Convert a RAW file to MINC format

    Arguments
    ---------
    infile: str
    outfile: str, optional
        Name of output MINC file. If None, will use the same name as
        the RAW file.
        (default None)
    keep_raw: bool, optional
        Option to keep RAW file. If False, the input file will be
        deleted.
        (default False)

    Returns
    -------
    outfile: str
        Name of output MINC file.
    success: int
        Integer indicating whether conversion was a success.
    """

    if outfile is None:
        outfile = infile.replace('.raw', '.mnc')

    try:
        rawtominc = ('cat {} '
                     '| rawtominc {} '
                     '-signed -float '
                     '-ounsigned '
                     '-oshort '
                     '-xstep 0.2 '
                     '-ystep 0.2 '
                     '-zstep 0.2 '
                     '-clobber 58 41 67'.format(infile, outfile))
        success = 1
    except:
        success = 0

    os.system(rawtominc)

    if keep_raw is not True:
        os.remove(infile)

    return outfile, success

def transform_space(infile, outfile = None, voxel_orientation = 'RAS',
                    world_space = 'MICe', expansion_factor = 1.0,
                    volume_type = None, data_type = None, labels = False):

    """
    Transform the coordinate space of a MINC file

    Author: Yohan Yee

    Arguments
    ---------
    infile: str
        Name of the MINC file to transform.
    outfile: str, optional
        Name of the output MINC file. If None, the input file will be
        overwritten.
        (default None)
    voxel_orientation: str, optional
        (default 'RAS')
    world_space: str, optional
        (default 'MICe')
    expansion_factor: float, optional
        (default 1.0)
    volume_type:
        (default None)
    data_type:
        (default None)
    labels: bool, optional
        (default None)

    Returns
    -------
    outfile: str
        Name of the transformed file.
    """

    def reorient_to_standard(dat):
        dat = np.rot90(dat, k=1, axes=(0, 2))
        dat = np.rot90(dat, k=1, axes=(0, 1))

        shape = dat.shape
        dat = np.ravel(dat)
        dat = np.reshape(dat, shape)

        return(dat)

    def do_nothing(dat):
        return(dat)

    # %% Coordinate definitions

    # Centers are listed as x,y,z; reverse these when writing out
    # Centers listed in um in CCFv3 coordinates
    centers_RAS = {"MICe"   :   [5700, 7900, 2700],
                   "CCFv3"  :   [0, 13200, 8000]}
    centers_PIR = {"MICe"   :   [5300, 5300, 5700],
                   "CCFv3"  :   [0, 0, 0]}

    # Direction cosines
    direction_cosines_RAS = {"MICe" :[[1, 0, 0], [0, 1, 0], [0, 0, 1]],
                             "CCFv3":[[0, 0, 1], [-1, 0, 0], [0, -1, 0]]}

    direction_cosines_PIR = {"MICe" :[[0, -1, 0], [0, 0, -1], [1, 0, 0]],
                             "CCFv3":[[1, 0, 0], [0, 1, 0], [0, 0, 1]]}

    # Map arguments to functions/dicts/values
    map_voxel_orientations = {"RAS":reorient_to_standard,
                              "PIR":do_nothing}

    map_centers = {"RAS":centers_RAS,
                   "PIR":centers_PIR}

    map_dir_cosines = {"RAS":direction_cosines_RAS,
                       "PIR":direction_cosines_PIR}

    size_10 = 1320*800*1140
    size_25 = 528*320*456
    size_50 = 264*160*228
    size_100 = 132*80*114
    size_200 = 58*41*67

    map_resolutions = {size_10: 10,
                       size_25: 25,
                       size_50: 50,
                       size_100: 100,
                       size_200: 200}

    vol = volumeFromFile(infile)

    res = map_resolutions[vol.data.size]

    # Voxel orientation
    if voxel_orientation in map_voxel_orientations:
        new_data = map_voxel_orientations[voxel_orientation](vol.data)
    else:
        print("Invalid voxel orientation")
        sys.exit(1)

    # World coordinate system
    centers = [expansion_factor*c/(1000)
               for c in map_centers[voxel_orientation][world_space]]
    steps = [expansion_factor*res/1000] * 3
    xdc = map_dir_cosines[voxel_orientation][world_space][0]
    ydc = map_dir_cosines[voxel_orientation][world_space][1]
    zdc = map_dir_cosines[voxel_orientation][world_space][2]

    # Types
    vtype = vol.volumeType if volume_type is None else volume_type
    dtype = vol.dtype if data_type is None else data_type
    labels = vol.labels if labels is None else labels

    if outfile is None:
        outfile = infile
        tmpfile = infile.replace('.mnc', '')+'_tmp.mnc'
    else:
        tmpfile = outfile

    outvol = volumeFromDescription(outputFilename=tmpfile,
                                   dimnames=["zspace", "yspace", "xspace"],
                                   sizes=new_data.shape,
                                   starts=[-c for c in reversed(centers)],
                                   steps=[s for s in reversed(steps)],
                                   x_dir_cosines=xdc,
                                   y_dir_cosines=ydc,
                                   z_dir_cosines=zdc,
                                   volumeType=vtype,
                                   dtype=dtype,
                                   labels=labels)

    outvol.data = new_data
    outvol.writeFile()
    outvol.closeVolume()

    if outfile == infile:
        os.rename(tmpfile, infile)

    return outfile

def download_data(experiment, outdir):

    """
    Download and transform the data from an ISH experiment

    Arguments
    ---------
    experiment: list
        List containing ISH experiment information. Element 0 must
        contain the experiment ID as `int`. Element 1 must contain
        the gene acronym as `str`.
    outdir: str
        Directory in which to download the ISH image.

    Returns
    -------
    None
    """

    experiment_id = experiment[0]
    gene = experiment[1]

    rawfile, success = fetch_expression(experiment_id, outdir = outdir)

    if success == 1:
        mincfile, success = rawtominc_wrapper(infile = rawfile)

        outfile = transform_space(infile = mincfile, voxel_orientation = 'RAS',
                                  world_space = 'MICe', expansion_factor = 1.0)

        os.rename(outfile, outdir+'{}_{}.mnc'.format(gene, experiment_id))

    return

# Main -----------------------------------------------------------------------

def main():

    #Get command line arguments
    args = parse_args()
    dataset = args['dataset']
    outdir = args['outdir']
    metadata = args['metadata']
    parallel = True if args['parallel'] == 'true' else False
    verbose = True if args['verbose'] == 'true' else False

    #Format the path properly
    outdir = os.path.join(outdir, '')

    #If outdir does not exist, create it
    if os.path.exists(outdir) == False:
        if verbose:
            print('Output directory {} not found. Creating it...'
                  .format(outdir))
        os.mkdir(outdir)

    #If AMBA metadata file not found, download it from the web
    if os.path.isfile(outdir+metadata) == False:
        if verbose:
            print('Metadata file {} not found in {}. Fetching from API...'
                  .format(metadata, outdir))
        fetch_metadata(dataset = dataset,
                       outdir = outdir,
                       outfile = metadata,
                       verbose = verbose)

    #Import AMBA metadata
    dfMetadata = pd.read_csv(outdir+metadata, index_col = None)

    #Extract experiment IDs and gene names from metadata
    nrows = dfMetadata.shape[0]
    experiments = [(dfMetadata.loc[i, 'experiment_id'], dfMetadata.loc[i, 'gene'])
                   for i in range(0, nrows)]

    #Create output sub-directory based on data set specified
    outdir = os.path.join(outdir, dataset, '')
    if os.path.exists(outdir) == False:
        os.mkdir(outdir)

    #Partial version of function for iteration
    download_data_partial = partial(download_data, outdir = outdir)

    if verbose:
        print('Downloading {} AMBA dataset to: {}'.format(dataset, outdir))

    if parallel:

        nproc = args['nproc']
        pool = mp.Pool(nproc)

        if verbose:
            print('Running in parallel on {} CPUs...'.format(nproc))

        #Download data in parallel. Show progress bar.
        results_tqdm = []
        for result in tqdm(pool.imap(download_data_partial, experiments),
                           total = len(experiments)):
            results_tqdm.append(result)

        pool.close()
        pool.join()

    else:

        experiments_tqdm = tqdm(experiments)
        results = list(map(download_data_partial, experiments_tqdm))

    return

if __name__ == '__main__':
    main()


from sqlalchemy import MetaData, String, Table
from sqlalchemy.dialects import postgresql

CIDR_TABLE_COLUMNS = [
    # table name, column name
    ('security_group_rules', 'cidr'),
    ('provider_fw_rules', 'cidr'),
    ('networks', 'cidr'),
    ('networks', 'cidr_v6')]

def upgrade(migrate_engine):
    """Convert String columns holding IP addresses to INET for postgresql."""
    meta = MetaData()
    meta.bind = migrate_engine
    dialect = migrate_engine.url.get_dialect()

    if dialect is postgresql.dialect:
        for table, column in CIDR_TABLE_COLUMNS:
            # can't use migrate's alter() because it does not support
            # explicit casting
            migrate_engine.execute(
                "ALTER TABLE %(table)s "
                "ALTER COLUMN %(column)s TYPE INET USING %(column)s::INET"
                % locals())
    else:
        for table, column in CIDR_TABLE_COLUMNS:
            t = Table(table, meta, autoload=True)
            getattr(t.c, column).alter(type=String(43))

def downgrade(migrate_engine):
    """Convert columns back to the larger String(255)."""
    meta = MetaData()
    meta.bind = migrate_engine
    for table, column in CIDR_TABLE_COLUMNS:
        t = Table(table, meta, autoload=True)
        getattr(t.c, column).alter(type=String(39))


from collections import deque
dx = [0, 1, 0, -1]
dy = [1, 0, -1, 0]

def burn():
    for _ in range(len(fire)):
        x, y = fire.popleft()
        for k in range(4):
            nx = x + dx[k]
            ny = y + dy[k]
            if 0 <= nx < N and 0 <= ny < M:
                if arr[nx][ny] != '#' and arr[nx][ny] != '*':
                    arr[nx][ny] = '*'
                    fire.append((nx, ny))

def move():
    isgo = False
    for _ in range(len(start)):
        x, y = start.popleft()
        for k in range(4):
            nx = x + dx[k]
            ny = y + dy[k]
            if 0 <= nx < N and 0 <= ny < M:
                if visited[nx][ny] == 0 and arr[nx][ny] != '#' and arr[nx][ny] != '*':
                    isgo = True
                    visited[nx][ny] = visited[x][y] + 1
                    start.append((nx, ny))
            else:
                return visited[x][y]
    if not isgo:
        return 'IMPOSSIBLE'

T = int(input())
for _ in range(T):
    M, N = map(int, input().split())
    arr = []
    fire = deque()
    start = deque()
    for i in range(N):
        arr.append(list(input().strip()))
        for j in range(M):
            if arr[i][j] == '*':
                fire.append((i, j))
            if arr[i][j] == '@':
                start.append((i, j))
    visited = [[0] * M for _ in range(N)]
    visited[start[0][0]][start[0][1]] = 1

    result = 0
    while True:
        burn()
        result = move()
        if result:
            break

    print(result)


from ariadne import ObjectType
from djangocities.pages.models import Page

site = ObjectType("Site")

@site.field("pages")
def resolve_pages(obj, info):
    return obj.page_set.all()

@site.field("default_page")
def resolve_default_page(obj, info):
    DEFAULT_FILE_NAME = "index.html"
    pages = obj.page_set.filter(site=obj).values_list("file_name", flat=True)

    if len(pages) == 0:
        return ""
    if DEFAULT_FILE_NAME in pages:
        return DEFAULT_FILE_NAME

    return pages[0]


import torch
from torch import Tensor
import torch.nn as nn

class BasicConv(nn.Module) :
    def __init__(self, in_channels, out_channels, kernel_size, stride = 1, padding = 0) :
        super(BasicConv, self).__init__()
        self.bn = nn.BatchNorm2d(in_channels)
        self.relu = nn.ReLU()
        self.conv = nn.Conv2d(in_channels, out_channels, kernel_size=kernel_size, stride=stride, padding=padding)

    def forward(self, x) :
        out = self.bn(x)
        out = self.relu(out)
        out = self.conv(out)

        return out

class Transition(nn.Module) :
    def __init__(self, in_channels) :
        super(Transition, self).__init__()

        self.conv = BasicConv(in_channels, in_channels, 1)
        self.avgpool = nn.AvgPool2d(kernel_size=2, stride= 2)

    def forward(self, x) :

        out = self.conv(x)
        out = self.avgpool(out)

        return out

class Bottleneck(nn.Module) :
    def __init__(self, in_channels, out_channels) :
        super(Bottleneck, self).__init__()

        self.conv1 = BasicConv(in_channels, 4*out_channels, kernel_size = 1)
        self.conv2 = BasicConv(4*out_channels, out_channels, kernel_size=3, stride=1, padding= 1)

    def forward(self, x) :

        out = self.conv1(x)
        out = self.conv2(out)

        return torch.cat((out, x), dim=1)

class DenseBlock(nn.Module) :
    def __init__(self, in_channels, num_layer, growth_rate) :
        super(DenseBlock, self).__init__()

        self.denseblock = self._make_layer(num_layer, in_channels, growth_rate)

    def _make_layer(self, num_layer, init_in_channel, growth_rate) :

        in_channels = init_in_channel
        layers = []
        for i in range(num_layer) :
            layers.append(Bottleneck(in_channels, growth_rate))
            in_channels = in_channels + growth_rate

        return nn.Sequential(*layers)

    def forward(self, x) :

        out = self.denseblock(x)

        return out

class DenseNet(nn.Module) :
    def __init__(self, growth_rate, num_classes=10) :
        super(DenseNet, self).__init__()

        num_channel = 3
        self.conv = BasicConv(num_channel, 16, kernel_size = 7, stride = 2)
        #num_channel = 16

        self.maxpool = nn.MaxPool2d(kernel_size = 3, stride = 2)

        self.denseblock1 = DenseBlock(num_channel, 6, growth_rate)
        num_channel = num_channel + 6 * growth_rate
        self.transition1 = Transition(num_channel)

        self.denseblock2 = DenseBlock(num_channel, 12, growth_rate)
        num_channel = num_channel + 12 * growth_rate
        self.transition2 = Transition(num_channel)

        self.denseblock3 = DenseBlock(num_channel, 24, growth_rate)
        num_channel = num_channel + 24 * growth_rate
        self.transition3 = Transition(num_channel)

        self.denseblock4 = DenseBlock(num_channel, 16, growth_rate)
        num_channel = num_channel + 16 * growth_rate

        self.avgpool = nn.AdaptiveAvgPool2d(1)
        self.fc = nn.Linear(num_channel, num_classes)

        self.feature = nn.Sequential(
            #self.conv,
            #self.maxpool,
            self.denseblock1,
            self.transition1,
            self.denseblock2,
            self.transition2,
            self.denseblock3,
            self.transition3,
            self.denseblock4
        )

    def forward(self, x) :

        out = self.feature(x)
        out = self.avgpool(out)
        out = torch.flatten(out, 1)
        out = self.fc(out)

        return out


#from html.parser import HTMLParser
from HTMLParser import HTMLParser
import os.path
import matplotlib.pyplot as plt; plt.rcdefaults()
import numpy as np
import matplotlib.pyplot as plt
import datetime

class LinksParser(HTMLParser):
    def __init__(self):
        HTMLParser.__init__(self)
        self.list = dict()
        self.list['external link'] = 0
        self.list['internal link'] = 0
        self.list['external js'] = 0
        self.list['internal js'] = 0
        self.list['inline js'] = 0
        self.list['external css'] = 0
        self.list['internal css'] = 0
        self.list['inline css'] = 0

    def handle_starttag(self, tag, attrs):
        if tag == 'a' and "href" in dict(attrs):
            link = dict(attrs)["href"]
            if link.startswith(('http', 'ftp', 'www')):
                self.list['external link'] += 1
            else:
                self.list['internal link'] += 1
        elif tag == 'script':
            if 'src' in dict(attrs):
                link = dict(attrs)["src"]
                if link.startswith(('http', 'ftp', 'www', '//')):
                    self.list['external js'] += 1
                else:
                    self.list['internal js'] += 1
            else:
                self.list['inline js'] += 1
        elif tag == 'link':
            if dict(attrs)['rel'] == 'stylesheet':
                if 'href' in dict(attrs):
                    link = dict(attrs)["href"]
                    if link.startswith(('http', 'ftp', 'www', '//')):
                        self.list['external css'] += 1
                    else:
                        self.list['internal css'] += 1
                else:
                    self.list['inline css'] += 1

def analyze(data):
    h=LinksParser()
    # page='<!DOCTYPE html>
Spectacular Mountain
'
    h.feed(data)
    print(h.list)

    objects = list()
    values = list()
    for key, value in h.list.iteritems():
        objects.append(key)
        values.append(value)
    y_pos = np.arange(len(objects))

    plt.bar(y_pos, values, align='center', alpha=0.5)
    plt.xticks(y_pos, objects, rotation='vertical')
    plt.ylabel('Usage')
    plt.title('Links usage')

    directory = 'links/'
    if not os.path.isdir(directory):
        os.makedirs(directory)
    plt.savefig(directory + datetime.datetime.now().strftime("%d%m%Y%H%M%S%f") + '.png', bbox_inches='tight')
    plt.clf()