import argparse
import os
import pugsql
from dateutil.parser import parse
from datetime import datetime, timedelta
import pytz

def date_to_unix(date_str, timezone="Asia/Taipei"):
    date = parse(date_str)
    tw = pytz.timezone(timezone)
    loc_dt = tw.localize(date)
    start_unix = loc_dt.timestamp()
    end_unix = start_unix + 86400
    return start_unix, end_unix

def main(args):
    queries = pugsql.module("queries")
    queries.connect(os.getenv("DB_URL"))

    date_start_unix, date_end_unix = date_to_unix(args.date)

    for stats in queries.count_site_stats(
        time_start=date_start_unix, time_end=date_end_unix
    ):
        queries.upsert_stats({**stats, "date": args.date})

if __name__ == "__main__":
    utc_now = datetime.utcnow()
    tw_now = utc_now + timedelta(hours=8)
    tw_yesterday = tw_now.date() - timedelta(days=1)
    str_tw_yesterday = tw_yesterday.strftime("%Y-%m-%d")

    parser = argparse.ArgumentParser()
    parser.add_argument(
        "-d",
        "--date",
        help="date to calculate stats on",
        type=str,
        default=str_tw_yesterday,
    )

    args = parser.parse_args()
    main(args)


import logging
import re
from superdesk.errors import SuperdeskApiError
from superdesk import get_resource_service
from superdesk.services import CacheableService
from flask_babel import _

logger = logging.getLogger(__name__)

class FilterConditionService(CacheableService):
    def on_create(self, docs):
        self._check_equals(docs)
        self._check_parameters(docs)

    def on_update(self, updates, original):
        doc = dict(original)
        doc.update(updates)
        self._check_equals([doc])
        self._check_parameters([doc])

    def delete(self, lookup):
        referenced_filters = self._get_referenced_filter_conditions(lookup.get("_id"))
        if referenced_filters.count() > 0:
            references = ",".join([pf["name"] for pf in referenced_filters])
            raise SuperdeskApiError.badRequestError(
                _("Filter condition has been referenced in content filter: {references}").format(references=references)
            )
        return super().delete(lookup)

    def _get_referenced_filter_conditions(self, id):
        lookup = {"content_filter.expression.fc": {"$in": [id]}}
        content_filters = get_resource_service("content_filters").get_from_mongo(req=None, lookup=lookup)
        return content_filters

    def _check_parameters(self, docs):
        parameters = get_resource_service("filter_condition_parameters").get_from_mongo(req=None, lookup=None)
        for doc in docs:
            parameter = [p for p in parameters if p["field"] == doc["field"]]
            if not parameter or len(parameter) == 0:
                raise SuperdeskApiError.badRequestError(
                    _("Filter condition:{condition} has unidentified field: {field}").format(
                        condition=doc["name"], field=doc["field"]
                    )
                )
            if doc["operator"] not in parameter[0]["operators"]:
                raise SuperdeskApiError.badRequestError(
                    _("Filter condition:{condition} has unidentified operator: {operator}").format(
                        condition=doc["name"], operator=doc["operator"]
                    )
                )

    def _check_equals(self, docs):
        """Checks if any of the filter conditions in the docs already exists

        :param docs: List of filter conditions to be tested
        :raises SuperdeskApiError: if any of the filter conditions in the docs
        already exists
        """
        for doc in docs:
            existing_docs = self.get_from_mongo(None, {"field": doc["field"], "operator": doc["operator"]})
            for existing_doc in existing_docs:
                if "_id" in doc and doc["_id"] == existing_doc["_id"]:
                    continue
                if self._are_equal(doc, existing_doc):
                    raise SuperdeskApiError.badRequestError(
                        _("Filter condition:{condition} has identical settings").format(condition=existing_doc["name"])
                    )

    def check_similar(self, filter_condition):
        """Checks for similar items

        Checks if the given filter condition already exists (for text fields like headline) or
        if there's any other filter condition that contains the given filter
        condition (for controlled vocabulary fields like urgency).
        For example: if filter_condition ['urgency' in 3,4] exists and if
        filter condition ['urgency' in 3] is searched we'll have a match

        :param filter_condition: Filter conditions to be tested
        :return: Returns the list of matching filter conditions
        """
        parameters = list(
            get_resource_service("filter_condition_parameters").get_from_mongo(
                req=None, lookup={"field": filter_condition["field"]}
            )
        )
        parameter = [p for p in parameters if p["field"] == filter_condition["field"]]
        if "in" in parameter[0]["operators"] or "nin" in parameter[0]["operators"]:
            # this is a controlled vocabulary field so find the overlapping values
            existing_docs = list(
                self.get_from_mongo(
                    None,
                    {
                        "field": filter_condition["field"],
                        "operator": filter_condition["operator"],
                        "value": {"$regex": re.compile(".*{}.*".format(filter_condition["value"]), re.IGNORECASE)},
                    },
                )
            )
            parameter[0]["operators"].remove(filter_condition["operator"])
            existing_docs.extend(
                list(
                    self.get_from_mongo(
                        None,
                        {
                            "field": filter_condition["field"],
                            "operator": parameter[0]["operators"][0],
                            "value": {"$not": re.compile(".*{}.*".format(filter_condition["value"]), re.IGNORECASE)},
                        },
                    )
                )
            )
        else:
            # find the exact matches
            existing_docs = list(
                self.get_from_mongo(
                    None,
                    {
                        "field": filter_condition["field"],
                        "operator": filter_condition["operator"],
                        "value": {"$regex": re.compile("{}".format(filter_condition["value"]), re.IGNORECASE)},
                    },
                )
            )
        return existing_docs

    def _are_equal(self, fc1, fc2):
        def get_comparer(fc):
            return "".join(sorted(fc["value"].upper())) if "," in fc["value"] else fc["value"].upper()

        return all(
            [fc1["field"] == fc2["field"], fc1["operator"] == fc2["operator"], get_comparer(fc1) == get_comparer(fc2)]
        )


import random
from PIL import Image, ImageDraw, ImageFont

import requests
import json
import csv
import io

from dotenv import dotenv_values
config = dotenv_values(".env")

def wrap_text(text, width):
    words = text.split()
    lines = []
    line = ""
    for word in words:
        if len(line) + len(word) <= width:
            line += word + " "
        else:
            lines.append(line.strip())
            line = word + " "
    lines.append(line.strip())
    return lines

def read_phrases_from_file(filename):
    phrases = []
    with open(filename, "r") as file:
        reader = csv.reader(file)
        for row in reader:
            phrases.append(row[0])
    return phrases

def get_theme_for_phrase(phrase, filename):
    with open(filename, "r") as file:
        reader = csv.reader(file)
        for row in reader:
            if row[0] == phrase:
                return row[1]
    return ""

def get_image_color(image_path):
    # Ouvrir l'image en utilisant Pillow
    image = Image.open(image_path)

    # Convertir l'image en tableau de pixels
    pixels = image.convert("RGB").getdata()

    # Initialiser des compteurs pour les couleurs R, G et B
    r = 0
    g = 0
    b = 0

    # Pour chaque pixel dans l'image, additionner les valeurs R, G et B
    for pixel in pixels:
        r += pixel[0]
        g += pixel[1]
        b += pixel[2]

    # Diviser les totaux pour les couleurs par le nombre de pixels pour obtenir la couleur moyenne
    average_color = (r / len(pixels), g / len(pixels), b / len(pixels))

    # transforme la couleur moyenne compatible avec la fonction fill de Pillow
    average_color = tuple(int(x) for x in average_color)

    return average_color

# retourne une police aléatoire du dossier fonts
def get_random_font():
    fonts = os.listdir("fonts")
    random_font = random.choice(fonts)
    return random_font

def get_complement_color(color):
    # obtenir la couleur complémentaire
    complement_color = (255 - color[0], 255 - color[1], 255 - color[2])

    return complement_color

def generate_text_image(text, themebg):
    # définir la taille de l'image
    width, height = 512, 512

    # créer une image blanche
    image = Image.new("RGB", (width, height), (255, 255, 255))

    # définir la police et la taille du texte
    font = ImageFont.truetype("fonts/{}".format(get_random_font()), 36)

    # créer un objet dessin pour écrire sur l'image
    draw = ImageDraw.Draw(image)

    # obtenir les coordonnées pour centrer le texte sur l'image
    text_width, text_height = draw.textsize(text, font)
    text_x = (width - text_width) / 2
    text_y = (height - text_height) / 2

    # Charger l'image de fond
    theme_image = Image.open("themes/{}".format(themebg))

    # obtenir la couleur moyenne de l'image de fond
    average_color = get_image_color(theme_image.filename)

    # obtenir la couleur complémentaire de la couleur moyenne
    text_color = get_complement_color(average_color)

    # Redimensionner l'image pour remplir complètement le cadre
    theme_ratio = theme_image.width / theme_image.height
    image_ratio = width / height

    if theme_ratio > image_ratio:
        new_width = int(height * theme_ratio)
        theme_image = theme_image.resize((new_width, height), Image.ANTIALIAS)
        left = (new_width - width) / 2
        right = new_width - left
        top = 0
        bottom = height
        theme_image = theme_image.crop((left, top, right, bottom))
    else:
        new_height = int(width / theme_ratio)
        theme_image = theme_image.resize((width, new_height), Image.ANTIALIAS)
        top = (new_height - height) / 2
        bottom = new_height - top
        left = 0
        right = width
        theme_image = theme_image.crop((left, top, right, bottom))

    # Coller l'image de fond sur l'image principale
    image.paste(theme_image, (0, 0))

    position = (text_x, text_y)

    # dessiner un rectangle autour du texte avec une marge de 10 pixels et la couleur moyenne opacifiée
    draw.rectangle((text_x - 10, text_y - 10, text_x + text_width + 10, text_y + text_height + 10), fill=average_color)
    # écrire le texte sur l'image
    draw.text((text_x, text_y), text, font=font, fill=text_color)

    return image

def search_image(query):
    # définir l'URL de l'API Unsplash avec le dotenv
    url = "https://api.unsplash.com/search/photos?query={}&client_id={}".format(query, config["CLIENT_ID"])

    # faire une requête à l'API
    response = requests.get(url)

    # vérifier que la requête a réussi
    if response.status_code == 200:
        data = json.loads(response.content.decode('utf-8'))

        # obtenir le lien de l'image trouvée
        image_url = data["results"][0]["urls"]["regular"]

        # télécharger l'image
        image = requests.get(image_url).content

        # mettre l'image dans un objet Image
        image = Image.open(io.BytesIO(image))

        return image
    else:
        print("Erreur lors de la requête à l'API Unsplash")
        return None

def save_image(image, filename):
    image.save(filename, "JPEG")

# for dev purposes
# del all images in folder img/ and themes/ except .gitkeep
import os
for filename in os.listdir("img"):
    if filename != ".gitkeep":
        os.remove(os.path.join("img", filename))
for filename in os.listdir("themes"):
    if filename != ".gitkeep":
        os.remove(os.path.join("themes", filename))

file = "phrases_test.csv"
phrases = read_phrases_from_file(file)
for i, phrase in enumerate(phrases):
    # générer une image avec le texte

    # trouver une image correspondant au thème de la phrase
    theme = get_theme_for_phrase(phrase, file)
    theme_image = search_image(theme)
    save_image(theme_image, "themes/theme_image_{}.jpeg".format(i))

    phrase = wrap_text(phrase, 20) # ceci retourne une liste de lignes
    phrase = "\n".join(phrase) # ceci transforme la liste en une chaîne de caractères
    text_image = generate_text_image(phrase, "theme_image_{}.jpeg".format(i))

    # enregistrer l'image sur le disque dur
    save_image(text_image, "img/text_image_{}.jpeg".format(i))


from code.main import suma, resta, multiplicacion, division, modulo
import pytest

@pytest.mark.parametrize(
        (
            "a",
            "b",
            "valor_esperado"
        ),
        [
            (
                3,
                4,
                7
            ),
            (
                10,
                2,
                12
            )
        ],
        ids=[
            "Test1",
            "Test2"
        ]
)
def test_suma(a, b, valor_esperado):
    assert suma(a, b) == valor_esperado

#@pytest.mark.skip(reason = "Funcion a depricar")
@pytest.mark.xfail()
def test_division():
    assert division(1, 0) == 1

def test_division():
    assert modulo(3, 1) == 0


import os
from configparser import ConfigParser

PATH = os.path.abspath(os.path.expanduser("~/.esgp.cfg"))

class Configuration(object):

    def __init__(self):
        self.algorithm = 'MD5'
        self.length = 10
        self.domain_settings = []

    def read(self):

        parser = ConfigParser()
        parser.read(PATH)

        if parser.has_section('defaults'):
            self.algorithm = parser.get('defaults', 'algorithm', fallback=self.algorithm)
            self.length = int(parser.get('defaults', 'length', fallback=self.length))

        for section in parser.sections():
            if section == "defaults":
                continue

            self.domain_settings.append({
                'domain': section,
                'algorithm': parser.get(section, 'algorithm', fallback=self.algorithm),
                'length': parser.get(section, 'length', fallback=self.length)
            })

    def write(self):

        parser = ConfigParser()

        parser.add_section('defaults')
        parser.set('defaults', 'algorithm', self.algorithm)
        parser.set('defaults', 'length', str(self.length))

        for settings in self.domain_settings:
            domain = settings['domain']
            if not domain:
                continue
            parser.add_section(domain)
            parser.set(domain, 'algorithm', settings['algorithm'])
            parser.set(domain, 'length', str(settings['length']))

        with open(PATH, 'w') as o:
            parser.write(o)

    def get_domain_settings(self, domain):
        for d in self.domain_settings:
            if d.get('domain', '') == domain:
                return d


import ccxt
import json

exchange = ccxt.bybit()
with open('symbols.json', 'r') as f:
    symbols = json.load(f)

# symbols = ['BTC/USDT', 'ETH/USDT', 'BNB/USDT', 'DOGE/USDT', 'XRP/USDT', 'MATIC/USDT', 'ADA/USDT', 'SOL/USDT', 'SHIB/USDT', 'LTC/USDT']
output_data = []

for symbol in symbols:
    ticker = exchange.fetch_ticker(symbol)
    data = {
        'symbol': symbol,
        'last_price': ticker['last'],
        '24h_change': ticker['percentage'],
        '24h_high': ticker['high'],
        '24h_low': ticker['low'],
        '24h_turnover': ticker['quoteVolume'],
    }
    output_data.append(data)

with open('data/market_data.json', 'w') as f:
    json.dump(output_data, f, indent=2)


"""
Preprocess mcd (modis land cover) dataset.

MODIS land cover fractions
https://lpdaac.usgs.gov/product_search/?collections=Combined+MODIS&collections=Terra+MODIS&collections=Aqua+MODIS&view=list

In:
Spatial:  0.0083 deg

Out:
Spatial:  0.033 deg

Steps:
1) Harmonize
2) Regrid

"""

import os
import xarray as xr
import logging

import numpy as np
from utils.pyutils import exit_if_exists, rm_existing
from utils.cdo_wrappers import cdo_remap
from dataprocessing.plotting import plot_var
from dataprocessing.datasets.config import \
    dir_source, \
    dir_target, \
    overwrite

logging.info('Processing dataset: mcd')

file_in = os.path.join(
    dir_source, '0d0083_static/MCD12Q1/V005/Data/v005_2/MCD12Q1plusC4_fraction.GLOBAL01KM.2001001.LC.01KM.nc'
)
file_out = os.path.join(
    dir_target, 'processed/0d033/static/mcd.nc'
)
file_tmp = file_out.replace('.nc', '_tmp.nc')

exit_if_exists(file_out, overwrite)
os.makedirs(os.path.dirname(file_out), exist_ok=True)

ds = xr.open_dataset(file_in)
ds = ds.rename({
    'MCD12Q1plusC4_fraction': 'data',
    'longitude': 'lon',
    'latitude': 'lat'})

lat_attrs = dict(
    long_name='Latitude',
    standard_name='latitude',
    units='degrees_north',
    axis='Y',
    valid_min=-90.0,
    valid_max=90.0
)
lon_attrs = dict(
    long_name='Longitude',
    standard_name='longitude',
    units='degrees_east',
    axis='X',
    modulo=360.0,
    topology='circular',
    valid_min=-180.0,
    valid_max=180.0,
)
ds.lat.attrs.update(lat_attrs)
ds.lon.attrs.update(lon_attrs)

ds.attrs['classes'] = np.array([
    l[1][1:].decode('utf-8').replace(' ', '_').lower()
    for l in ds.Legend.values
    ])[:-2]

ds = ds.drop('Legend')
ds['data'] = ds.data.expand_dims('var', 0)

ds = ds.where(~ds.data.isnull(), 0)

ds.to_netcdf(file_tmp)
ds.close()

cdo_remap(
    in_files=file_tmp,
    out_files=file_out,
    nlat_target=180*30,
    nlon_target=360*30,
    remap_alg='laf')

rm_existing(file_tmp)

plot_path = __file__.replace('.py', '.jpg')
plot_var(path=file_out, plot_path=plot_path)

logging.info('Done processing dataset: mcd')


import pygame,sys
import pygame.transform
from pygame.locals import *
import numpy
import random

pygame.init()

clock = pygame.time.Clock()

tileWidth = 16
tileHeight = 16

rowSize = 24
columnSize = 10

WINDOWHEIGHT=tileHeight*rowSize
WINDOWWIDTH=tileWidth*columnSize

SURFACE = pygame.display.set_mode((WINDOWWIDTH,WINDOWHEIGHT))

pygame.display.set_caption('Tetris New Version')

AQUA      = (0, 255, 255)
BLUE      = (0, 0, 255)
BLACK     = (0, 0, 0)
# Fuchsia   = (255, 0,255)
# Gray      = (128, 128, 128)
GREEN     = (0, 205, 0)
# Maroon    = (128, 0, 0)
# Navy Blue = (0, 0, 128)
# Olive     = (128, 128, 0)
PURPLE    = (153, 50, 204)
#RED       = (255, 0, 0)
# Silver    = (192, 192, 192)
# Teal      = (0, 128, 128)
WHITE     = (255, 255, 255)
RED       = (255, 0, 0)
YELLOW    = (255, 255, 0)
ORANGE    = (255, 140, 0)

field = numpy.zeros((rowSize, columnSize),dtype=numpy.int)

def drawLevel():
    SURFACE.fill(BLACK)
    #Draw the level
    for row in range(rowSize):
        for column in range(columnSize):
            if field[row,column] == 1:
                pygame.draw.rect(SURFACE,YELLOW,(tileWidth*column,row*tileHeight,tileWidth,tileHeight))
            elif field[row,column] == 2:
                pygame.draw.rect(SURFACE,PURPLE,(tileWidth*column,row*tileHeight,tileWidth,tileHeight))
            elif field[row,column] == 3:
                pygame.draw.rect(SURFACE,AQUA,(tileWidth*column,row*tileHeight,tileWidth,tileHeight))
            elif field[row,column] == 4:
                pygame.draw.rect(SURFACE,BLUE,(tileWidth*column,row*tileHeight,tileWidth,tileHeight))
            elif field[row,column] == 5:
                pygame.draw.rect(SURFACE,ORANGE,(tileWidth*column,row*tileHeight,tileWidth,tileHeight))
            elif field[row,column] == 6:
                pygame.draw.rect(SURFACE,GREEN,(tileWidth*column,row*tileHeight,tileWidth,tileHeight))
            elif field[row,column] == 7:
                pygame.draw.rect(SURFACE,RED,(tileWidth*column,row*tileHeight,tileWidth,tileHeight))

figure=random.randint(1,7)
def putFigure(figure,figureRow,figureColumn,check=True):

    if figure == 1:
        if figureRow == rowSize:
            return False
        if numpy.sum(field[figureRow-1:figureRow+1,figureColumn-1:figureColumn+1]) > 0:
            return False
        if figureColumn-1 < 0:
            return False
        if figureColumn > columnSize - 1:
            return False
        if check == False:
            field[figureRow-1:figureRow+1,figureColumn-1:figureColumn+1]=1
    elif figure == 2:
        if figureRow+1 == rowSize:
            return False
        if figureColumn-1 < 0:
            return False
        if figureColumn+1 > columnSize - 1:
            return False
        if numpy.sum(field[figureRow,figureColumn-1:figureColumn+2]) > 0:
            return False
        if numpy.sum(field[figureRow:figureRow+2,figureColumn]) > 0:
            return False
        if check == False:
            field[figureRow,figureColumn-1:figureColumn+2]=2
            field[figureRow:figureRow+2,figureColumn]=2
    elif figure == 3:
        if figureRow == rowSize:
            return False
        if figureColumn-1 < 0:
            return False
        if figureColumn+2 > columnSize - 1:
            return False
        if numpy.sum(field[figureRow,figureColumn-1:figureColumn+3]) > 0:
            return False
        if check == False:
            field[figureRow,figureColumn-1:figureColumn+3]=3
    elif figure == 4:
        if figureRow+1 == rowSize:
            return False
        if figureColumn < 0:
            return False
        if figureColumn+1 > columnSize-1:
            return False
        if numpy.sum(field[figureRow-1:figureRow+2,figureColumn]) > 0:
            return False
        if numpy.sum(field[figureRow+1,figureColumn:figureColumn+2]) > 0:
            return False
        if check == False:
            field[figureRow-1:figureRow+2,figureColumn]=4
            field[figureRow+1,figureColumn:figureColumn+2]=4
    elif figure == 5:
        if figureRow+1 == rowSize:
            return False
        if figureColumn-1 < 0:
            return False
        if figureColumn > columnSize - 1:
            return False
        if numpy.sum(field[figureRow-1:figureRow+2,figureColumn]) > 0:
            return False
        if numpy.sum(field[figureRow+1,figureColumn-1:figureColumn+1]) > 0:
            return False
        if check == False:
            field[figureRow-1:figureRow+2,figureColumn]=5
            field[figureRow+1,figureColumn-1:figureColumn+1]=5
    elif figure == 6:
        if figureRow == rowSize:
            return False
        if figureColumn-1 < 0:
            return False
        if figureColumn+1 > columnSize -1:
            return False
        if numpy.sum(field[figureRow-1,figureColumn:figureColumn+2]) > 0:
            return False
        if numpy.sum(field[figureRow,figureColumn-1:figureColumn+1]) > 0:
            return False
        if check == False:
            field[figureRow-1,figureColumn:figureColumn+2]=6
            field[figureRow,figureColumn-1:figureColumn+1]=6
    elif figure == 7:
        if figureRow == rowSize:
            return False
        if figureColumn-1 < 0:
            return False
        if figureColumn+1 > columnSize - 1:
            return False
        if numpy.sum(field[figureRow-1,figureColumn-1:figureColumn+1]) > 0:
            return False
        if numpy.sum(field[figureRow,figureColumn:figureColumn+2]) > 0:
            return False
        if check == False:
            field[figureRow-1,figureColumn-1:figureColumn+1]=7
            field[figureRow,figureColumn:figureColumn+2]=7
    return True

def removeFigure(figure,figureRow,figureColumn):

    if figure == 1:
        field[figureRow-1:figureRow+1,figureColumn-1:figureColumn+1]=0
    elif figure == 2:
        field[figureRow,figureColumn-1:figureColumn+2]=0
        field[figureRow:figureRow+2,figureColumn]=0
    elif figure == 3:
        field[figureRow,figureColumn-1:figureColumn+3]=0
    elif figure == 4:
        field[figureRow-1:figureRow+2,figureColumn]=0
        field[figureRow+1,figureColumn:figureColumn+2]=0
    elif figure == 5:
        field[figureRow-1:figureRow+2,figureColumn]=0
        field[figureRow+1,figureColumn-1:figureColumn+1]=0
    elif figure == 6:
        field[figureRow-1,figureColumn:figureColumn+2]=0
        field[figureRow,figureColumn-1:figureColumn+1]=0
    elif figure == 7:
        field[figureRow-1,figureColumn-1:figureColumn+1]=0
        field[figureRow,figureColumn:figureColumn+2]=0

def EndGame():
    basicFont = pygame.font.SysFont(None, 30)
    text = basicFont.render('Game Over!', True, RED)
    textRect = text.get_rect()
    textRect.centerx = WINDOWWIDTH/2
    textRect.centery = WINDOWHEIGHT/2
    SURFACE.blit(text,textRect)
    pygame.display.update()

gameOver = False
figureRow=1
figureColumn=int(columnSize/2)-1
putFigure(figure,figureRow,figureColumn)

SPEED = 60
downSpeed = SPEED/5
counter = 0

while True:
    removeFigure(figure, figureRow, figureColumn)
    for event in pygame.event.get():
        if event.type == QUIT:
            pygame.quit()
            sys.exit()
        if gameOver == True:
            continue
        if event.type == KEYDOWN:
            figureNewRow = figureRow
            figureNewColumn = figureColumn
            if event.key == K_RIGHT:
                figureNewColumn = figureColumn+1
            if event.key == K_LEFT:
                figureNewColumn = figureColumn-1
            if event.key == K_DOWN:
                figureNewRow = figureRow+1
            if putFigure(figure,figureNewRow,figureNewColumn) == True:
                figureRow = figureNewRow
                figureColumn = figureNewColumn
    if gameOver == True:
        continue

    counter = counter + 1
    if counter % downSpeed == 0:
        if putFigure(figure,figureRow+1,figureColumn) == True:
            figureRow = figureRow+1
        else:
            putFigure(figure,figureRow,figureColumn,False)
            figure=random.randint(1,7)
            figureRow=1
            figureColumn=int(columnSize/2)-1
            if putFigure(figure,figureRow, figureColumn) == False:
                EndGame()
                gameOver = True
                continue

    putFigure(figure,figureRow,figureColumn,False)

    drawLevel()
    clock.tick(SPEED)
    pygame.display.update()


import sys
import pathlib
pth=str(pathlib.Path().absolute())
print(pth)
sys.path.append(('\\').join(pth.split('\\')[:-1])+"\\Utils")
from Utilities import *
from MF import *
from Visualization_utilities import *
from transformation import Transformation
from Signatures import Signature
import os
import open3d as o3d
import numpy as np

def getRotationMatrix(vect, ang):
    cosA = np.cos(ang)
    sinA = np.sin(ang)
    oneMinusCosA = 1-cosA
    out = np.zeros((3, 3))
    out[0, 0] = (cosA + vect[0] * vect[0] * oneMinusCosA)
    out[0, 1] = (vect[1] * vect[0] * oneMinusCosA + vect[2] * sinA)
    out[0, 2] = (vect[2] * vect[0] * oneMinusCosA - vect[1] * sinA)
    out[1, 0] = (vect[0] * vect[1] * oneMinusCosA - vect[2] * sinA)
    out[1, 1] = (cosA + vect[1] * vect[1] * oneMinusCosA)
    out[1, 2] = (vect[2] * vect[1] * oneMinusCosA + vect[0] * sinA)
    out[2, 0] = (vect[0] * vect[2] * oneMinusCosA + vect[1] * sinA)
    out[2, 1] = (vect[1] * vect[2] * oneMinusCosA - vect[0] * sinA)
    out[2, 2] = (cosA + vect[2] * vect[2] * oneMinusCosA)
    return out

def getRotationMatrixFromVector(vect1, vect2):
    vect_normal = np.cross(vect1, vect2)
    vect_normal = vect_normal/np.linalg.norm(vect_normal)
    dotProduct = np.dot(vect1/np.linalg.norm(vect1), vect2/np.linalg.norm(vect2))
    ang = np.arccos(dotProduct)
    return getRotationMatrix(vect1, ang)

#pts = 'Beam-W_W8x2_pulg_ac_210.pts'
#pts = 'Beam-W_W8x1000_pulg_mb_368.pts'
pts = 'Beam-W_W4x1301_pulg_mb_261.pts'
#pts = "Beam-W_W8x2102_pulg_ac_208.pts"
dpf=o3d.io.read_point_cloud("E:\\INNOVATE\\DB_TRAINING_1024_PTS\\"+pts.split('_')[0]+"\\"+pts)

ev1,ev2=project_pcl_pca(dpf,
             voxel_size=0.02,
             nb=5,
             std_ratio=1.5,
             p_th=0.1,
             ig="Linearity",
             upper=True
            )

rot = getRotationMatrixFromVector(ev1, np.array([0.0, 0.0, 1.0]))
cloud = dpf
mesh_frame = o3d.geometry.TriangleMesh.create_coordinate_frame(size=80.0, origin=cloud.get_center())
cloud.rotate(rot)

p = np.asarray(cloud.points)
n_p = np.copy(p)

low_bound = np.min(n_p[:, 2])
high_bound = np.max(n_p[:, 2])
l = high_bound - low_bound
N_SLICES = 10
nd = {i:0 for i in range(N_SLICES)}
pd = {i:[] for i in range(N_SLICES)}

for cont, i in enumerate(n_p):
    for j in range(N_SLICES):
        if i[2]>= low_bound+j*l/N_SLICES and i[2] < low_bound+(j+1)*l/N_SLICES:
            nd[j] += 1
            pd[j].append([n_p[cont, 0], n_p[cont, 1]])
            n_p[cont, 2] = low_bound+j*l/N_SLICES
print(nd)
test = o3d.geometry.PointCloud()
test.points = o3d.utility.Vector3dVector(n_p)
#o3d.visualization.draw_geometries([test])

ww = []
hh = []
mm = []
for i in pd.keys():
    pd[i] = np.array(pd[i])
    co = pd[i].T.dot(pd[i])
    _, v = np.linalg.eig(co)
    aux = pd[i]
    init = np.random.random((aux.shape[0], 3))
    init[:, :2] = aux
    naux = o3d.geometry.PointCloud()
    naux.points = o3d.utility.Vector3dVector(init)
    m_ = np.array([0., 0., np.arctan(v[0, 1]/v[0, 0])])
    mm.append([0., 0., np.arctan(v[0, 1]/v[0, 0])])
    naux.rotate(naux.get_rotation_matrix_from_axis_angle(m_))
    puntex = np.asarray(naux.points)
    w = np.max(puntex[:, 1]) - np.min(puntex[:, 1])
    h = np.max(puntex[:, 0]) - np.min(puntex[:, 0])
    ww.append(w)
    hh.append(h)

    #naux.translate(-naux.get_center())
    #mesh_frame = o3d.geometry.TriangleMesh.create_coordinate_frame(size=8.0, origin=np.array([0., 0., 0.]))
    #o3d.visualization.draw_geometries([naux, mesh_frame])
w_out = np.mean(np.array(ww))
h_out = np.mean(np.array(hh))

mm =np.array(mm)
N = len(dpf.points)
if N>= 5000:
    dpf, _ = dpf.remove_statistical_outlier(nb_neighbors=int(N/200), std_ratio = 0.9)
bb = dpf.get_oriented_bounding_box()
R = bb.R

x, y, z = R[0, 0], R[1, 0], R[2, 0] #Extrusion around x axis of bounding box
N = np.array([x, y, z])
P = np.array(dpf.get_center())
s = []
for i in dpf.points:
    v = i-P
    dist = np.dot(v, N)
    #pp = i-dist*N
    s.append(dist)

l = max(s) - min(s)
base = P+N*max(s)
mm_ = np.mean(mm, 0)
print(0.9*w_out, 0.9*h_out, base, l, N, naux.get_rotation_matrix_from_axis_angle(mm_))


# coding: utf-8

"""
    CyberSource Merged Spec

    All CyberSource API specs merged together. These are available at https://developer.cybersource.com/api/reference/api-reference.html

    OpenAPI spec version: 0.0.1

    Generated by: https://github.com/swagger-api/swagger-codegen.git
"""

from __future__ import absolute_import

import sys
import os
import re

# python 2 and python 3 compatibility library
from six import iteritems

from ..configuration import Configuration
from ..api_client import ApiClient

class AuthenticationExemptionsApi(object):
    """
    NOTE: This class is auto generated by the swagger code generator program.
    Do not edit the class manually.
    Ref: https://github.com/swagger-api/swagger-codegen
    """

    def __init__(self, merchant_config, api_client=None):
        config = Configuration()
        if api_client:
            self.api_client = api_client
        else:
            if not config.api_client:
                config.api_client = ApiClient()
            self.api_client = config.api_client
        self.api_client.set_configuration(merchant_config)

    def authentication_exemptions(self, authentication_exemptions_request, **kwargs):
        """
        Authentication Exemptions Service
        A new CYBS branded service to connect to VISA’s REST API to enable Visa Transaction Advisor (VTA) as a standalone service for merchants to support PSD2/SCA adoption and exemptions processing startegy in Europe.VTA Provides intelligent risk data to merchants as inputs to their in-house fraud management tools for fraud mitigation on Visa transactions.
        This method makes a synchronous HTTP request by default. To make an
        asynchronous HTTP request, please define a `callback` function
        to be invoked when receiving the response.
        >>> def callback_function(response):
        >>>     pprint(response)
        >>>
        >>> thread = api.authentication_exemptions(authentication_exemptions_request, callback=callback_function)

        :param callback function: The callback function
            for asynchronous request. (optional)
        :param AuthenticationExemptionsRequest authentication_exemptions_request: (required)
        :return: RiskV1AuthenticationExemptionsPost201Response
                 If the method is called asynchronously,
                 returns the request thread.
        """
        kwargs['_return_http_data_only'] = True
        if kwargs.get('callback'):
            return self.authentication_exemptions_with_http_info(authentication_exemptions_request, **kwargs)
        else:
            (data) = self.authentication_exemptions_with_http_info(authentication_exemptions_request, **kwargs)
            return data

    def authentication_exemptions_with_http_info(self, authentication_exemptions_request, **kwargs):
        """
        Authentication Exemptions Service
        A new CYBS branded service to connect to VISA’s REST API to enable Visa Transaction Advisor (VTA) as a standalone service for merchants to support PSD2/SCA adoption and exemptions processing startegy in Europe.VTA Provides intelligent risk data to merchants as inputs to their in-house fraud management tools for fraud mitigation on Visa transactions.
        This method makes a synchronous HTTP request by default. To make an
        asynchronous HTTP request, please define a `callback` function
        to be invoked when receiving the response.
        >>> def callback_function(response):
        >>>     pprint(response)
        >>>
        >>> thread = api.authentication_exemptions_with_http_info(authentication_exemptions_request, callback=callback_function)

        :param callback function: The callback function
            for asynchronous request. (optional)
        :param AuthenticationExemptionsRequest authentication_exemptions_request: (required)
        :return: RiskV1AuthenticationExemptionsPost201Response
                 If the method is called asynchronously,
                 returns the request thread.
        """

        all_params = ['authentication_exemptions_request']
        all_params.append('callback')
        all_params.append('_return_http_data_only')
        all_params.append('_preload_content')
        all_params.append('_request_timeout')

        params = locals()
        for key, val in iteritems(params['kwargs']):
            if key not in all_params:
                raise TypeError(
                    "Got an unexpected keyword argument '%s'"
                    " to method authentication_exemptions" % key
                )
            params[key] = val
        del params['kwargs']
        # verify the required parameter 'authentication_exemptions_request' is set
        if ('authentication_exemptions_request' not in params) or (params['authentication_exemptions_request'] is None):
            raise ValueError("Missing the required parameter `authentication_exemptions_request` when calling `authentication_exemptions`")

        collection_formats = {}

        path_params = {}

        query_params = []

        header_params = {}

        form_params = []
        local_var_files = {}

        body_params = None
        if 'authentication_exemptions_request' in params:
            body_params = params['authentication_exemptions_request']
        # HTTP header `Accept`
        header_params['Accept'] = self.api_client.\
            select_header_accept(['application/hal+json;charset=utf-8'])

        # HTTP header `Content-Type`
        header_params['Content-Type'] = self.api_client.\
            select_header_content_type(['application/json;charset=utf-8'])

        # Authentication setting
        auth_settings = []

        return self.api_client.call_api(f'/risk/v1/authentication-exemptions', 'POST',
                                        path_params,
                                        query_params,
                                        header_params,
                                        body=body_params,
                                        post_params=form_params,
                                        files=local_var_files,
                                        response_type='RiskV1AuthenticationExemptionsPost201Response',
                                        auth_settings=auth_settings,
                                        callback=params.get('callback'),
                                        _return_http_data_only=params.get('_return_http_data_only'),
                                        _preload_content=params.get('_preload_content', True),
                                        _request_timeout=params.get('_request_timeout'),
                                        collection_formats=collection_formats)


from flask import request
from flask_json import FlaskJSON, JsonError
from mongoengine import *
from __init__ import app, fasthash_dictionary
from decorators import *
from models import *
import json

@userRoute('/api/profile')
def profile(postdata):
    print("profile with", postdata)
    username = postdata.get("username", None)
    user = User.objects.get(username=username)
    try:
        profile = Profile.objects.get(user=user)
    except Profile.DoesNotExist:
        print("Realize there is no profile")
        profile = Profile(user=user)
        profile.save()
    for node in Node.objects.all():
        if node.serial not in profile.means.keys():
            profile.means[node.serial] = 0
    profile.save()
    # nodes = Node.objects()
    return {'profile': json.loads(profile.to_json())}

@freeRoute('/api/question')
def question(postdata):
    question_serial = postdata.get('serial', None)
    question_id = postdata.get('id', None)
    if question_serial:
        question = Question.objects.get(serial=question_serial)
    elif question_id:
        question = Question.objects.get(id=question_id)
    else:
        return {'status':400, 'description':'Can\'t find the question. No serial or id provided.'}
    return {'status':200, 'question':json.loads(question.to_json())}

@userRoute('/api/auto_question')
def auto_question(postdata):
    import random
    from pprint import pprint
    username = postdata.get('username', '')
    user = User.objects.get(username=username)
    try:
        profile = Profile.objects.get(user=user)
    except Profile.DoesNotExist:
        profile = Profile(user=user).save()
    for node in Node.objects.all():
        if node.serial not in profile.means.keys():
            profile.means[node.serial] = 0
    profile.save()
    means = profile.means
    is_complete = lambda k: means[k] >= 0.9
    is_incomplete = lambda k: means[k] < 0.9
    is_accesible = lambda k: all( is_complete(node.serial) for node in Node.objects.get(serial=k).antes )
    relevant_means = { key:value for (key, value) in profile.means.items() if is_incomplete(key) and is_accesible(key) }
    pprint(relevant_means)
    try:
        node_serial = sorted(relevant_means.keys())[0]
    except:
        node_serial = random.choice([ node.serial for node in Node.objects.all() ])
    node = Node.objects.get(serial=node_serial)
    question = random.choice(list(Question.objects.filter(node=node.id)))
    return {'status':200, 'question':json.loads(question.to_json())}

@userRoute('/api/update_mean')
def update_mean(postdata):
    username = postdata.get('username', '')
    user = User.objects.get(username=username)
    try:
        profile = Profile.objects.get(user=user)
    except Profile.DoesNotExist:
        profile = Profile(user=user).save()
    for node in Node.objects.all():
        if node.serial not in profile.means.keys():
            profile.means[node.serial] = 0
    profile.save()
    question_serial = postdata.get('question_serial', '')
    node = Question.objects.get(serial=question_serial).node
    mean_delta = postdata.get('mean_delta', 0)
    profile.means[node.serial] = max(0, min(1, profile.means[node.serial]+mean_delta))
    profile.save()
    return {'status':200}

@freeRoute('/api/node_questions')
def node_questions(postdata):
    node_id = postdata.get('node_id', None)
    node_serial = postdata.get('node_serial', None)
    if node_serial:
        node = Node.objects.get(serial=node_serial)
    elif node_id:
        node = Node.objects.get(id=node_id)
    else:
        return {'status':400, 'description':'Can\'t find the node. No serial or id provided.'}
    questions = Question.objects.filter(node=node.id)
    return {'status':200, 'name':node.name, 'questions':json.loads(questions.to_json())}

@freeRoute('/api/node')
def node(postdata):
    node_serial = postdata.get('node_serial', None)
    if node_serial:
        try:
            node = Node.objects.get(serial=node_serial)
        except Node.DoesNotExist:
            return {'status':400, 'description':'The serial provider is not in the database.'}
    else:
        return {'status':400, 'description':'Can\'t find the node. No serial or id provided.'}
    return {'status':200, 'node':json.loads(node.to_json())}


"""
Modified Settler Bot
"""
# Let's start by importing the Halite Starter Kit so we can interface with the Halite engine
import hlt
import time
from datetime import datetime
import logging
import pprint

from random import sample, randint

VERSION = "0.9.3"
DEBUG = False

def closest_target_from_list(gamemap, source, target_list, range=1000):
    #find closest enemy ignoring obstacles
    closest = None
    for foreign_entity in target_list:
        if source.owner == foreign_entity.owner:
            continue
        distance = source.calculate_distance_between(foreign_entity)
        if distance < range:
            range = distance
            closest = foreign_entity
    return closest

def find_nearby_target(gamemap, entity, range=40):
    logger = logging.getLogger(__name__)
    all_nearby = gamemap.nearby_entities_by_distance(entity)
    if len(all_nearby) == 0:
        return None
    logger.debug("nearby_entities_by_distance(entity={}) len: {}"
                 .format(entity.id, len(all_nearby)))
    try:
        index = 0
        for key in all_nearby.keys():
            nearby_entity = all_nearby[key][0]
            if 'Ship' == nearby_entity.__class__.__name__:
                logger.debug("nearby from {},{}: {}, {} x,y {},{}"
                             .format(entity.x, entity.y,
                                     nearby_entity.__class__.__name__,
                                     nearby_entity,
                                     nearby_entity.x, nearby_entity.y))
                return nearby_entity
    except KeyError:
        pass
    return None

def find_closest_unowned_planet(gamemap, ship, planetlist, range=1000):
    logger = logging.getLogger(__name__)
    best_planet = None
    logger.debug(" ship: {} find_closest_unowned_planet, len planetlist: {}"
                 .format(ship.id, len(planetlist)))
    for planet in planetlist:
        distance = ship.calculate_distance_between(planet)
        logger.debug(" ship: {} planet: {} distance: {:.02f}, prior: {:.02f}"
                     .format(ship.id, planet.id, distance, range))
        if distance < range:
            best_planet = planet
            range = distance
    return best_planet

def planet_dock_if_nearby(gamemap, ship):
    for planet in gamemap.all_planets():
        if ship.can_dock(planet) and not planet.is_owned():
            return planet
        if planet.owner == ship.owner and not planet.is_full() and ship.can_dock(planet):
            return planet

def avoid_collision_with_list(ship, entity_list):
    '''Given a list of ships, compute destination away from center'''
    center = hlt.entity.Position((sum(entity.x for entity in entity_list)/len(entity_list)),
                                 (sum(entity.y for entity in entity_list)/len(entity_list)))
    # now we compute the anti-position for a closest_point_to analog
    target = hlt.entity.Position((2*ship.x - center.x),
                                 (2*ship.y - center.y))
    return target

def navigate(ship, destination, game_map, speed=hlt.constants.MAX_SPEED/2, ignore_ships=True):
    return ship.navigate(destination, game_map, speed, ignore_ships=ignore_ships)

def halite2_main():
    logger = logging.getLogger(__name__)
    # GAME START
    # Here we define the bot's name as Settler and initialize the game, including communication with the Halite engine.
    game = hlt.Game("Settler %s" % VERSION)
    turn = 0

    while True:
        # TURN START
        # Update the map for the new turn and get the latest version

        turn_start_time = time.time()
        turn += 1

        logger.info("turn: {} start at: {}".format(turn, datetime.now().isoformat('T')))
        game_map = game.update_map()
        logger.info("turn: {} game.update_map took: {}"
                    .format(turn, time.time() - turn_start_time))

        #
        # variables that may factor in policy
        #
        enemy_ships = [s for s in game_map._all_ships() if s.owner != game_map.get_me()]
        enemy_docked_ships = [s for s in enemy_ships if s.docking_status != s.DockingStatus.UNDOCKED]
        enemy_undocked_ships = [s for s in enemy_ships if s.docking_status == s.DockingStatus.UNDOCKED]
        my_ships = game_map.get_me().all_ships()
        my_docked_ships = [s for s in my_ships if s.docking_status != s.DockingStatus.UNDOCKED]
        my_undocked_ships = [s for s in my_ships if s.docking_status == s.DockingStatus.UNDOCKED]
        logger.info("turn: {} enemy: {} enemy docked: {} my ships: {} my undocked: {}"
                     .format(turn, len(enemy_ships), len(enemy_docked_ships),
                             len(my_ships), len(my_undocked_ships)))

        # ship_time_limit: total seconds permitted
        # ship_speed: limit navigation speed
        # ship_action_limit: total permitted navigate calls or other expensive checks
        # action_ships: give the above limit, these ships will act this turn
        #               sample/shuffle ships to avoid deadlocks
        # action_planet_percent: percentage of above (based on id hash) that will dock
        #               or move to planets
        if turn > 200 or len(my_undocked_ships) > 100:
            ship_time_limit = 1.5
            ship_speed = hlt.constants.MAX_SPEED*0.4
            ship_action_limit = 50
            ship_dock_ratio = 5
            ship_dock_enemy_range = 6
            ship_shun_center_planets = False
            action_ships = my_undocked_ships[:min(ship_action_limit, len(my_undocked_ships))]
            action_ship_long_range = 50
            action_planet_percent = 15
            action_docked_percent = 15
            action_target_docked_range = 50
            navigate_ignore_ships_threshold = 100
        elif turn > 100 and len(my_undocked_ships) > 100:
            ship_time_limit = 1.6
            ship_speed = hlt.constants.MAX_SPEED*0.5
            ship_action_limit = 70
            ship_dock_ratio = 4
            ship_dock_enemy_range = 7
            ship_shun_center_planets = False
            action_ships = my_undocked_ships[:min(ship_action_limit, len(my_undocked_ships))]
            action_ship_long_range = 100
            action_planet_percent = 15
            action_docked_percent = 15
            action_target_docked_range = 50
            navigate_ignore_ships_threshold = 100
        elif turn > 50:
            ship_time_limit = 1.8
            ship_speed = hlt.constants.MAX_SPEED*0.95
            ship_action_limit = 100
            ship_dock_ratio = 2
            ship_dock_enemy_range = 10
            ship_shun_center_planets = False
            action_ships = my_undocked_ships[:min(ship_action_limit, len(my_undocked_ships))]
            action_ship_long_range = 100
            action_planet_percent = 70
            action_docked_percent = 70
            action_target_docked_range = 100
            navigate_ignore_ships_threshold = 100
        elif turn > 10:
            ship_time_limit = 1.9
            ship_speed = hlt.constants.MAX_SPEED
            ship_action_limit = 100
            ship_dock_ratio = 1
            ship_dock_enemy_range = 10
            ship_shun_center_planets = False
            action_ships = my_undocked_ships
            action_ship_long_range = 50
            action_planet_percent = 75
            action_docked_percent = 75
            action_target_docked_range = 100
            navigate_ignore_ships_threshold = 100
        else:
            ship_time_limit = 1.9
            ship_speed = hlt.constants.MAX_SPEED
            ship_action_limit = 100
            ship_dock_ratio = 1
            ship_dock_enemy_range = 20
            ship_shun_center_planets = True
            action_ships = sample(my_undocked_ships, min(ship_action_limit, len(my_undocked_ships)))
            action_ship_long_range = 50
            action_planet_percent = 90
            action_docked_percent = 0
            action_target_docked_range = 100
            navigate_ignore_ships_threshold = 100

        unowned_planets = [p for p in game_map.all_planets() if not p.is_owned()]
        unowned_planets.sort(key=lambda x: x.id)
        if ship_shun_center_planets:
            unowned_planets = [p for p in unowned_planets if p.id > 3]

        if len(my_undocked_ships) + len(enemy_undocked_ships) > navigate_ignore_ships_threshold:
            ignore_ships = True
        else:
            ignore_ships = False

        logger.info("turn: {} end of prep, start of ship loop after: {:.03f}"
                    .format(turn, (time.time() - turn_start_time)))

        #
        # local variables
        #
        ship_actions = 0
        #only 1 ship can dock per turn, so limit our attempts?
        planet_docking = []
        # Here we define the set of commands to be sent to the Halite engine at the end of the turn
        command_queue = []

        for ship in action_ships:
            if (time.time() - turn_start_time) > ship_time_limit:
                logger.debug("turn: {} ship: {} skipping due to time {}"
                             .format(turn, ship.id, (time.time() - turn_start_time)))
                break

            # limit the number of ships we look at per turn, just in case
            ship_actions += 1
            if ship_actions > ship_action_limit:
                logger.debug("turn: {} ship: {} skipping due to ship actions {} after {:.03f} seconds"
                             .format(turn, ship.id, ship_actions, time.time() - turn_start_time))
                continue

            if turn == 1:
                avoidance = avoid_collision_with_list(ship, my_undocked_ships)
                logger.debug("turn: {} ship: {} collision avoidance: {},{}"
                             .format(turn, ship.id, avoidance.x, avoidance.y))
                navigate_command = ship.navigate(avoidance,
                                                 game_map, speed = hlt.constants.MAX_SPEED,
                                                 ignore_ships=False)
                if navigate_command:
                    command_queue.append(navigate_command)
                continue

            # if we're next to a planet, maybe do that
            elif ship_dock_ratio == 1 or randint(0, ship_dock_ratio-1) == 0:
                # before docking check for nearby enemies
                target_ship = closest_target_from_list(game_map, ship,
                                                       enemy_ships, ship_dock_enemy_range)
                if target_ship:
                    logger.info("turn: {} ship: {} defensive targetting ship: {}"
                                .format(turn, ship.id, target_ship.id))
                    navigate_command = ship.navigate(ship.closest_point_to(target_ship, 4.5),
                                                     game_map, speed = hlt.constants.MAX_SPEED,
                                                     ignore_ships=ignore_ships)
                    if navigate_command:
                        command_queue.append(navigate_command)
                    continue
                if not target_ship:
                    docking_target = planet_dock_if_nearby(game_map, ship)
                    if docking_target:
                        if docking_target.id not in planet_docking:
                            logger.info("turn: {} ship: {} docking to planet: {}"
                                        .format(turn, ship.id, docking_target.id))
                            planet_docking.append(docking_target.id)
                            command_queue.append(ship.dock(docking_target))
                        # this will not fall through and instead wait a turn and check again
                        continue
                    # else fall through to planet/ship navigation

            if hash(ship.id) % 100 < action_planet_percent:
                # find ship's closest planet to navigate to
                logger.debug("ship id {} hash {} is planet action (percent: {})"
                             .format(ship.id, hash(ship.id) % 100, action_planet_percent))
                planet = find_closest_unowned_planet(game_map, ship, unowned_planets, 100)
                if planet:
                    logger.info("turn: {} ship: {} x,y {},{} off to planet: {} x,y {},{}"
                                .format(turn, ship.id, ship.x, ship.y,
                                        planet.id, planet.x, planet.y))
                    target_point = ship.closest_point_to(planet)
                    navigate_command = ship.navigate(target_point, game_map,
                                                     speed=ship_speed,
                                                     ignore_ships=ignore_ships)
                    if ship_shun_center_planets and planet.id < 4:
                        logger.debug("planet: {} used when shunned".format(planet.id))
                    if navigate_command:
                        command_queue.append(navigate_command)
                    unowned_planets.remove(planet)
                    continue
                # else this will fall through to aggressive ship behavior

            # anti-ship action
            logger.debug("ship id {} hash {} is ship action"
                         .format(ship.id, hash(ship.id) % 100))
            if hash(ship.id) % 100 < action_docked_percent and len(enemy_docked_ships) > 0:
                time_1 = time.time()
                target_ship = closest_target_from_list(game_map, ship,
                                                       enemy_docked_ships,
                                                       action_target_docked_range)
                if target_ship:
                    logger.debug("ship: {} targetting docked closest id,x,y {},{},{} in {:.06f}"
                                 .format(ship.id,
                                         target_ship.id, target_ship.x, target_ship.y,
                                         time.time() - time_1))
            if not target_ship and hash(ship.id) % 100 < action_ship_long_range:
                time_1 = time.time()
                target_ship_1 = closest_target_from_list(game_map, ship, enemy_ships, 200)
                if target_ship_1:
                    logger.debug("ship: {} targetting closest returned id,x,y {},{},{} in {:.06f}"
                                 .format(ship.id,
                                         target_ship_1.id, target_ship_1.x, target_ship_1.y,
                                         time.time() - time_1))

            if target_ship:
                logger.info("turn: {} ship: {} targetting ship: {}"
                            .format(turn, ship.id, target_ship.id))
                distance = 2.75 + (ship.id % 3)*0.5
                navigate_command = ship.navigate(ship.closest_point_to(target_ship, distance),
                                                 game_map, speed=ship_speed,
                                                 ignore_ships=ignore_ships)
                if navigate_command:
                    command_queue.append(navigate_command)
            else:
                logger.info("turn: {} ship: {} no work".format(turn, ship.id))

        # Send our set of commands to the Halite engine for this turn
        turn_end_presend = time.time()
        logger.info("turn: {} end ships time: {:.03f}"
                         .format(turn, turn_end_presend - turn_start_time))
        game.send_command_queue(command_queue)
        #logger.debug("turn: {} send_command_queue len: {} took: {:.03f}"
        #             .format(turn, len(command_queue), time.time() - turn_end_presend))
        logger.info("turn: {} end, took: {:.03f}"
                    .format(turn, time.time() - turn_start_time))
        # TURN END
    # GAME END

if __name__ == "__main__":
        #logging.basicConfig(level=logging.INFO)
        halite2_main()


from __future__ import unicode_literals

from smewt.base import utils, ImportTask
from smewt.ontology import Media, CollectionSettings
from smewt.base.textutils import u
import json
import os
import logging

log = logging.getLogger(__name__)

class Collection(object):
    """A Collection keeps track of the files in the folders for us. when any changes
    happen there, it creates new import tasks for new files or removes references to
    deleted files."""

    def __init__(self, name, validFiles, mediaTagger, dataGraph, taskManager, folders = {}):
        log.debug('Initializing %s Collection', name)
        # identifies the collection
        self.name = name
        # validFiles is a list of conditions a file should meet to be
        # considered to belong to this collection
        self.validFiles = validFiles
        self.mediaTagger = mediaTagger

        self.graph = dataGraph
        self.taskManager = taskManager

        # folders is a list of (folder name, bool) indicating whether
        # they should be traversed recursively
        self.folders = folders

        self.loadSettings()

    def findOrCreate(self):
        for c in utils.tolist(self.graph.config.get('collections')):
            if c.name == self.name:
                return c
        result = CollectionSettings.fromDict({ 'name': self.name, 'folders': [] }, self.graph)
        self.addToConfig(result)
        return result

    def loadSettings(self):
        c = self.findOrCreate().toDict()
        self.folders = c['folders']

    def addToConfig(self, c):
        if c not in utils.tolist(self.graph.config.get('collections')):
            self.graph.config.append('collections', c)

    def saveSettings(self):
        c = self.findOrCreate()
        c.folders = json.dumps(self.folders)

    def checkIntegrity():
        # TODO: delete files in set(files) - set(dirwalk())
        pass

    def setFolders(self, folders):
        self.folders = folders
        self.saveSettings()

    def addFolder(self):
        self.folders.append(['', True])
        self.saveSettings()

    def deleteFolder(self, index):
        del self.folders[index]
        self.saveSettings()

    def collectionFiles(self):
        for folder, recursive in self.folders:
            for f in utils.dirwalk(folder, self.validFiles, recursive):
                p = os.path.split(f)
                if '.AppleDouble' in p:
                    continue
                if p[-1].startswith('._'):
                    continue
                yield f

    def modifiedFiles(self):
        lastModified = dict((f.filename, f.get('lastModified', None)) for f in self.graph.find_all(Media))
        for f in self.collectionFiles():
            # yield a file if we haven't heard of it yet or if it has been modified recently
            if f not in lastModified or os.path.getmtime(f) > lastModified[f]:
                yield f

    def importFiles(self, files):
        for f in files:
            # new import task
            log.info('Import in %s collection: %s' % (u(self.name), u(f)))
            if self.taskManager:
                importTask = ImportTask(self.graph, self.mediaTagger, f)
                self.taskManager.add(importTask)

        # save newly imported files
        self.saveSettings()

    def update(self):
        log.info('Updating %s collection' % self.name)
        self.importFiles(self.modifiedFiles())

    def rescan(self):
        log.info('Rescanning %s collection' % self.name)
        self.importFiles(self.collectionFiles())


import sklearn.metrics
import torch
import pytorch_lightning as pl
from torchvision.models import resnet50, densenet121
import numpy as np
import cem.train.utils as utils

################################################################################
## HELPER FUNCTIONS
################################################################################

def compute_bin_accuracy(c_pred, y_pred, c_true, y_true):
    c_pred = c_pred.reshape(-1).cpu().detach() > 0.5
    y_probs = y_pred.cpu().detach()
    y_pred = y_probs > 0.5
    c_true = c_true.reshape(-1).cpu().detach()
    y_true = y_true.reshape(-1).cpu().detach()
    c_accuracy = sklearn.metrics.accuracy_score(c_true, c_pred)
    c_auc = sklearn.metrics.roc_auc_score(c_true, c_pred, multi_class='ovo')
    c_f1 = sklearn.metrics.f1_score(c_true, c_pred, average='macro')
    y_accuracy = sklearn.metrics.accuracy_score(y_true, y_pred)
    y_auc = sklearn.metrics.roc_auc_score(y_true, y_probs)
    y_f1 = sklearn.metrics.f1_score(y_true, y_pred)
    return (c_accuracy, c_auc, c_f1), (y_accuracy, y_auc, y_f1)

def compute_accuracy(
    c_pred,
    y_pred,
    c_true,
    y_true,
):
    if (len(y_pred.shape) < 2) or (y_pred.shape[-1] == 1):
        return compute_bin_accuracy(
            c_pred,
            y_pred,
            c_true,
            y_true,
        )
    c_pred = (c_pred.cpu().detach().numpy() >= 0.5).astype(np.int32)
    # Doing the following transformation for when labels are not
    # fully certain
    c_true = (c_true.cpu().detach().numpy() > 0.5).astype(np.int32)
    y_probs = torch.nn.Softmax(dim=-1)(y_pred).cpu().detach()
#     used_classes = np.unique(y_true.cpu().detach())
#     y_probs = y_probs[:, sorted(list(used_classes))]
    y_pred = y_pred.argmax(dim=-1).cpu().detach()
    y_true = y_true.cpu().detach()

    c_accuracy = c_auc = c_f1 = 0
    for i in range(c_true.shape[-1]):
        true_vars = c_true[:, i]
        pred_vars = c_pred[:, i]
        c_accuracy += sklearn.metrics.accuracy_score(
            true_vars, pred_vars
        ) / c_true.shape[-1]

        if len(np.unique(true_vars)) == 1:
            c_auc += np.mean(true_vars == pred_vars)/c_true.shape[-1]
        else:
            c_auc += sklearn.metrics.roc_auc_score(
                true_vars,
                pred_vars,
            )/c_true.shape[-1]
        c_f1 += sklearn.metrics.f1_score(
            true_vars,
            pred_vars,
            average='macro',
        )/c_true.shape[-1]
    y_accuracy = sklearn.metrics.accuracy_score(y_true, y_pred)
    try:
        y_auc = sklearn.metrics.roc_auc_score(
            y_true,
            y_probs,
            multi_class='ovo',
        )
    except Exception as e:
        y_auc = 0.0
    try:
        y_f1 = sklearn.metrics.f1_score(y_true, y_pred, average='macro')
    except:
        y_f1 = 0.0
    return (c_accuracy, c_auc, c_f1), (y_accuracy, y_auc, y_f1)

################################################################################
## BASELINE MODEL
################################################################################

class ConceptBottleneckModel(pl.LightningModule):
    def __init__(
        self,
        n_concepts,
        n_tasks,
        concept_loss_weight=0.01,
        task_loss_weight=1,

        extra_dims=0,
        bool=False,
        sigmoidal_prob=True,
        sigmoidal_extra_capacity=True,
        bottleneck_nonlinear=None,
        output_latent=False,

        x2c_model=None,
        c_extractor_arch=utils.wrap_pretrained_model(resnet50),
        c2y_model=None,
        c2y_layers=None,

        optimizer="adam",
        momentum=0.9,
        learning_rate=0.01,
        weight_decay=4e-05,
        weight_loss=None,
        task_class_weights=None,

        active_intervention_values=None,
        inactive_intervention_values=None,
        intervention_policy=None,

        top_k_accuracy=None,
        gpu=int(torch.cuda.is_available()),
    ):
        """
        Constructs a joint Concept Bottleneck Model (CBM) as defined by
        Koh et al. 2020.

        :param int n_concepts: The number of concepts given at training time.
        :param int n_tasks: The number of output classes of the CBM.
        :param float concept_loss_weight: Weight to be used for the final loss'
            component corresponding to the concept classification loss. Default
            is 0.01.
        :param float task_loss_weight: Weight to be used for the final loss'
            component corresponding to the output task classification loss.
            Default is 1.

        :param int extra_dims: The number of extra unsupervised dimensions to
            include in the bottleneck. Defaults to 0.
        :param Bool bool: Whether or not we threshold concepts in the bottleneck
            to be binary. Only relevant if the bottleneck uses a sigmoidal
            activation. Defaults to False.
        :param Bool sigmoidal_prob: Whether or not to use a sigmoidal activation
            for the bottleneck's activations that are aligned with training
            concepts. Defaults to True.
        :param Bool sigmoidal_extra_capacity:  Whether or not to use a sigmoidal
            activation for the bottleneck's unsupervised activations (when
            extra_dims > 0). Defaults to True.
        :param str bottleneck_nonlinear: A valid nonlinearity name to use for
            any unsupervised extra capacity in this model (when extra_dims > 0).
            It may overwrite `sigmoidal_extra_capacity` if
            sigmoidal_extra_capacity is True. If None, then no activation will
            be used. Will be soon deprecated. It must be one of [None,
            "sigmoid", "relu", "leakyrelu"] and defaults to None.

        :param Pytorch.Module x2c_model: A valid pytorch Module used to map the
            CBM's inputs to its bottleneck layer with `n_concepts + extra_dims`
            activations. If not given, then one may provide a generator
            function via the c_extractor_arch argument.
        :param Fun[(int), Pytorch.Module] c_extractor_arch: If x2c_model is None,
            then one may provide a generator function for the input to concept
            model that takes as an input the size of the bottleneck (using
            an argument called `output_dim`) and returns a valid Pytorch Module
            that maps this CBM's inputs to the bottleneck of the requested size.
        :param Pytorch.Module c2y_model:  A valid pytorch Module used to map the
            CBM's bottleneck (with size n_concepts + extra_dims`) to `n_tasks`
            output activations (i.e., the output of the CBM).
            If not given, then a simple leaky-ReLU MLP, whose hidden
            layers have sizes `c2y_layers`, will be used.
        :param List[int] c2y_layers: List of integers defining the size of the
            hidden layers to be used in the MLP to predict classes from the
            bottleneck if c2y_model was NOT provided. If not given, then we will
            use a simple linear layer to map the bottleneck to the output
            classes.

        :param str optimizer:  The name of the optimizer to use. Must be one of
            `adam` or `sgd`. Default is `adam`.
        :param float momentum: Momentum used for optimization. Default is 0.9.
        :param float learning_rate:  Learning rate used for optimization.
            Default is 0.01.
        :param float weight_decay: The weight decay factor used during
            optimization. Default is 4e-05.
        :param List[float] weight_loss: Either None or a list with n_concepts
            elements indicating the weights assigned to each predicted concept
            during the loss computation. Could be used to improve
            performance/fairness in imbalanced datasets.
        :param List[float] task_class_weights: Either None or a list with
            n_tasks elements indicating the weights assigned to each output
            class during the loss computation. Could be used to improve
            performance/fairness in imbalanced datasets.

        :param List[float] active_intervention_values: A list of n_concepts
            values to use when positively intervening in a given concept (i.e.,
            setting concept c_i to 1 would imply setting its corresponding
            predicted concept to active_intervention_values[i]). If not given,
            then we will assume that we use `1` for all concepts. This
            parameter is important when intervening in CBMs that do not have
            sigmoidal concepts, as the intervention thresholds must then be
            inferred from their empirical training distribution.
        :param List[float] inactive_intervention_values: A list of n_concepts
            values to use when negatively intervening in a given concept (i.e.,
            setting concept c_i to 0 would imply setting its corresponding
            predicted concept to inactive_intervention_values[i]). If not given,
            then we will assume that we use `0` for all concepts. This
            parameter is important when intervening in CBMs that do not have
            sigmoidal concepts, as the intervention thresholds must then be
            inferred from their empirical training distribution.
        :param Callable[(np.ndarray, np.ndarray, np.ndarray), np.ndarray] intervention_policy:
            An optional intervention policy to be used when intervening on a
            test batch sample x (first argument), with corresponding true
            concepts c (second argument), and true labels y (third argument).
            The policy must produce as an output a list of concept indices to
            intervene (in batch form) or a batch of binary masks indicating
            which concepts we will intervene on.

        :param List[int] top_k_accuracy: List of top k values to report accuracy
            for during training/testing when the number of tasks is high.
        :param Bool gpu: whether or not to use a GPU device or not.
        """
        gpu = int(gpu)
        super().__init__()
        self.n_concepts = n_concepts
        self.intervention_policy = intervention_policy
        self.output_latent = output_latent
        if x2c_model is not None:
            # Then this is assumed to be a module already provided as
            # the input to concepts method
            self.x2c_model = x2c_model
        else:
            self.x2c_model = c_extractor_arch(
                output_dim=(n_concepts + extra_dims)
            )

        # Now construct the label prediction model
        if c2y_model is not None:
            # Then this method has been provided to us already
            self.c2y_model = c2y_model
        else:
            # Else we construct it here directly
            units = [n_concepts + extra_dims] + (c2y_layers or []) + [n_tasks]
            layers = []
            for i in range(1, len(units)):
                layers.append(torch.nn.Linear(units[i-1], units[i]))
                if i != len(units) - 1:
                    layers.append(torch.nn.LeakyReLU())
            self.c2y_model = torch.nn.Sequential(*layers)

        # Intervention-specific fields/handlers:
        init_fun = torch.cuda.FloatTensor if gpu else torch.FloatTensor
        if active_intervention_values is not None:
            self.active_intervention_values = init_fun(
                active_intervention_values
            )
        else:
            # Setting to 5 for prob = 1 (as that would result in its sigmoid
            # value being very close to 1) and -5 if prob=0 (as that will
            # go to zero when applied a sigmoid)
            self.active_intervention_values = init_fun(
                [1 for _ in range(n_concepts)]
            ) * (
                5.0 if not sigmoidal_prob else 1.0
            )
        if inactive_intervention_values is not None:
            self.inactive_intervention_values = init_fun(
                inactive_intervention_values
            )
        else:
            # Setting to 5 for prob = 1 (as that would result in its sigmoid
            # value being very close to 1) and -5 if prob=0 (as that will
            # go to zero when applied a sigmoid)
            self.inactive_intervention_values = init_fun(
                [1 for _ in range(n_concepts)]
            ) * (
                -5.0 if not sigmoidal_prob else 0.0
            )

        # For legacy purposes, we wrap the model around a torch.nn.Sequential
        # module
        self.sig = torch.nn.Sigmoid()
        if sigmoidal_extra_capacity:
            # Keeping this for backwards compatability
            bottleneck_nonlinear = "sigmoid"
        if bottleneck_nonlinear == "sigmoid":
            self.bottleneck_nonlin = torch.nn.Sigmoid()
        elif bottleneck_nonlinear == "leakyrelu":
            self.bottleneck_nonlin = torch.nn.LeakyReLU()
        elif bottleneck_nonlinear == "relu":
            self.bottleneck_nonlin = torch.nn.ReLU()
        elif (bottleneck_nonlinear is None) or (
            bottleneck_nonlinear == "identity"
        ):
            self.bottleneck_nonlin = lambda x: x
        else:
            raise ValueError(
                f"Unsupported nonlinearity '{bottleneck_nonlinear}'"
            )

        self.loss_concept = torch.nn.BCELoss(weight=weight_loss)
        self.loss_task = (
            torch.nn.CrossEntropyLoss(weight=task_class_weights)
            if n_tasks > 1 else torch.nn.BCEWithLogitsLoss(
                weight=task_class_weights
            )
        )
        self.bool = bool
        self.concept_loss_weight = concept_loss_weight
        self.task_loss_weight = task_loss_weight
        self.momentum = momentum
        self.learning_rate = learning_rate
        self.weight_decay = weight_decay
        self.optimizer_name = optimizer
        self.extra_dims = extra_dims
        self.top_k_accuracy = top_k_accuracy
        self.n_tasks = n_tasks
        self.sigmoidal_prob = sigmoidal_prob
        self.sigmoidal_extra_capacity = sigmoidal_extra_capacity

    def _unpack_batch(self, batch):
        x = batch[0]
        if isinstance(batch[1], list):
            y, c = batch[1]
        else:
            y, c = batch[1], batch[2]
        return x, y, c

    def _standardize_indices(self, intervention_idxs, batch_size):
        if isinstance(intervention_idxs, list):
            intervention_idxs = np.array(intervention_idxs)
        if isinstance(intervention_idxs, np.ndarray):
            intervention_idxs = torch.IntTensor(intervention_idxs)

        if intervention_idxs is None or (
            isinstance(intervention_idxs, torch.Tensor) and
            ((len(intervention_idxs) == 0) or intervention_idxs.shape[-1] == 0)
        ):
            return None
        if not isinstance(intervention_idxs, torch.Tensor):
            raise ValueError(
                f'Unsupported intervention indices {intervention_idxs}'
            )
        if len(intervention_idxs.shape) == 1:
            # Then we will assume that we will do use the same
            # intervention indices for the entire batch!
            intervention_idxs = torch.tile(
                torch.unsqueeze(intervention_idxs, 0),
                (batch_size, 1),
            )
        elif len(intervention_idxs.shape) == 2:
            assert intervention_idxs.shape[0] == batch_size, (
                f'Expected intervention indices to have batch size {batch_size} '
                f'but got intervention indices with shape {intervention_idxs.shape}.'
            )
        else:
            raise ValueError(
                f'Intervention indices should have 1 or 2 dimensions. Instead we got '
                f'indices with shape {intervention_idxs.shape}.'
            )
        if intervention_idxs.shape[-1] == self.n_concepts:
            # We still need to check the corner case here where all indices are
            # given...
            elems = torch.unique(intervention_idxs)
            if len(elems) == 1:
                is_binary = (0 in elems) or (1 in elems)
            elif len(elems) == 2:
                is_binary = (0 in elems) and (1 in elems)
            else:
                is_binary = False
        else:
            is_binary = False
        if not is_binary:
            # Then this is an array of indices rather than a binary array!
            intervention_idxs = intervention_idxs.to(dtype=torch.long)
            result = torch.zeros(
                (batch_size, self.n_concepts),
                dtype=torch.bool,
                device=intervention_idxs.device,
            )
            result[:, intervention_idxs] = 1
            intervention_idxs = result
        assert intervention_idxs.shape[-1] == self.n_concepts, (
                f'Unsupported intervention indices with shape {intervention_idxs.shape}.'
            )
        if isinstance(intervention_idxs, np.ndarray):
            # Time to make it into a torch Tensor!
            intervention_idxs = torch.BoolTensor(intervention_idxs)
        intervention_idxs = intervention_idxs.to(dtype=torch.bool)
        return intervention_idxs

    def _concept_intervention(
        self,
        c_pred,
        intervention_idxs=None,
        c_true=None,
    ):
        if (c_true is None) or (intervention_idxs is None):
            return c_pred
        c_pred_copy = c_pred.clone()
        intervention_idxs = self._standardize_indices(
            intervention_idxs=intervention_idxs,
            batch_size=c_pred.shape[0],
        )
        intervention_idxs = intervention_idxs.to(c_pred.device)
        if self.sigmoidal_prob:
            c_pred_copy[intervention_idxs] = c_true[intervention_idxs]
        else:
            batched_active_intervention_values =  torch.tile(
                torch.unsqueeze(self.active_intervention_values, 0),
                (c_pred.shape[0], 1),
            )

            batched_inactive_intervention_values =  torch.tile(
                torch.unsqueeze(self.inactive_intervention_values, 0),
                (c_pred.shape[0], 1),
            )

            c_pred_copy[intervention_idxs] = (
                (
                    c_true[intervention_idxs] *
                    batched_active_intervention_values[intervention_idxs]
                ) +
                (
                    (c_true[intervention_idxs] - 1) *
                    -batched_inactive_intervention_values[intervention_idxs]
                )
            )

        return c_pred_copy

    def _forward(
        self,
        x,
        intervention_idxs=None,
        c=None,
        y=None,
        train=False,
        latent=None,
    ):
        if latent is None:
            latent = self.x2c_model(x)
        if self.sigmoidal_prob or self.bool:
            if self.extra_dims:
                # Then we only sigmoid on the probability bits but
                # let the other entries up for grabs
                c_pred_probs = self.sig(latent[:, :-self.extra_dims])
                c_others = self.bottleneck_nonlin(latent[:,-self.extra_dims:])
                c_pred =  torch.cat([c_pred_probs, c_others], axis=-1)
                c_sem = c_pred_probs
            else:
                c_pred = self.sig(latent)
                c_sem = c_pred
        else:
            # Otherwise, the concept vector itself is not sigmoided
            # but the semantics
            c_pred = latent
            if self.extra_dims:
                c_sem = self.sig(latent[:, :-self.extra_dims])
            else:
                c_sem = self.sig(latent)
        # Now include any interventions that we may want to include
        if (intervention_idxs is None) and (c is not None) and (
            self.intervention_policy is not None
        ):
            intervention_idxs, c_int = self.intervention_policy(
                x=x,
                c=c,
                pred_c=c_sem,
                y=y,
            )
        else:
            c_int = c
        c_pred = self._concept_intervention(
            c_pred=c_pred,
            intervention_idxs=intervention_idxs,
            c_true=c_int,
        )
        if self.bool:
            y = self.c2y_model((c_pred > 0.5).float())
        else:
            y = self.c2y_model(c_pred)
        if self.output_latent:
            return c_sem, c_pred, y, latent
        return c_sem, c_pred, y

    def forward(self, x, c=None, y=None, latent=None, intervention_idxs=None):
        return self._forward(
            x,
            train=False,
            c=c,
            y=y,
            intervention_idxs=intervention_idxs,
            latent=latent,
        )

    def predict_step(
        self,
        batch,
        batch_idx,
        intervention_idxs=None,
        dataloader_idx=0,
    ):
        x, y, c = self._unpack_batch(batch)
        return self._forward(
            x,
            intervention_idxs=intervention_idxs,
            c=c,
            y=y,
            train=False,
        )

    def _run_step(
        self,
        batch,
        batch_idx,
        train=False,
        intervention_idxs=None,
    ):
        x, y, c = self._unpack_batch(batch)
        if self.output_latent:
            c_sem, c_logits, y_logits, _ = self._forward(
                x,
                intervention_idxs=intervention_idxs,
                c=c,
                y=y,
                train=train,
            )
        else:
            c_sem, c_logits, y_logits = self._forward(
                x,
                intervention_idxs=intervention_idxs,
                c=c,
                y=y,
                train=train,
            )
        if self.task_loss_weight != 0:
            task_loss = self.loss_task(
                y_logits if y_logits.shape[-1] > 1 else y_logits.reshape(-1),
                y,
            )
            task_loss_scalar = task_loss.detach()
        else:
            task_loss = 0
            task_loss_scalar = 0
        if self.concept_loss_weight != 0:
            # We separate this so that we are allowed to
            # use arbitrary activations (i.e., not necessarily in [0, 1])
            # whenever no concept supervision is provided
            concept_loss = self.loss_concept(c_sem, c)
            loss = self.concept_loss_weight * concept_loss + task_loss
            concept_loss_scalar = concept_loss.detach()
        else:
            loss = task_loss
            concept_loss_scalar = 0.0
        # compute accuracy
        (c_accuracy, c_auc, c_f1), (y_accuracy, y_auc, y_f1) = compute_accuracy(
            c_sem,
            y_logits,
            c,
            y,
        )
        result = {
            "c_accuracy": c_accuracy,
            "c_auc": c_auc,
            "c_f1": c_f1,
            "y_accuracy": y_accuracy,
            "y_auc": y_auc,
            "y_f1": y_f1,
            "concept_loss": concept_loss_scalar,
            "task_loss": task_loss_scalar,
            "loss": loss.detach(),
            "avg_c_y_acc": (c_accuracy + y_accuracy) / 2,
        }
        if self.top_k_accuracy is not None:
            y_true = y.reshape(-1).cpu().detach()
            y_pred = y_logits.cpu().detach()
            labels = list(range(self.n_tasks))
            for top_k_val in self.top_k_accuracy:
                y_top_k_accuracy = sklearn.metrics.top_k_accuracy_score(
                    y_true,
                    y_pred,
                    k=top_k_val,
                    labels=labels,
                )
                result[f'y_top_{top_k_val}_accuracy'] = y_top_k_accuracy
        return loss, result

    def training_step(self, batch, batch_no):
        loss, result = self._run_step(batch, batch_no, train=True)
        for name, val in result.items():
            self.log(name, val, prog_bar=("accuracy" in name))
        return {
            "loss": loss,
            "log": {
                "c_accuracy": result['c_accuracy'],
                "c_auc": result['c_auc'],
                "c_f1": result['c_f1'],
                "y_accuracy": result['y_accuracy'],
                "y_auc": result['y_auc'],
                "y_f1": result['y_f1'],
                "concept_loss": result['concept_loss'],
                "task_loss": result['task_loss'],
                "loss": result['loss'],
                "avg_c_y_acc": result['avg_c_y_acc'],
            },
        }

    def validation_step(self, batch, batch_no):
        loss, result = self._run_step(batch, batch_no, train=False)
        for name, val in result.items():
            self.log("val_" + name, val, prog_bar=("accuracy" in name))
        return {
            "val_" + key: val
            for key, val in result.items()
        }

    def test_step(self, batch, batch_no):
        loss, result = self._run_step(batch, batch_no, train=False)
        for name, val in result.items():
            self.log("test_" + name, val, prog_bar=True)
        return result['loss']

    def configure_optimizers(self):
        if self.optimizer_name.lower() == "adam":
            optimizer = torch.optim.Adam(
                self.parameters(),
                lr=self.learning_rate,
                weight_decay=self.weight_decay,
            )
        else:
            optimizer = torch.optim.SGD(
                filter(lambda p: p.requires_grad, self.parameters()),
                lr=self.learning_rate,
                momentum=self.momentum,
                weight_decay=self.weight_decay,
            )
        lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer)
        return {
            "optimizer": optimizer,
            "lr_scheduler": lr_scheduler,
            "monitor": "loss",
        }


import cv2
import numpy as np
import logging
import time

class VideoPlayer:
    def __init__(self, channel_num, src, dims, full_screen=True):
        self.channel_num = channel_num
        self.src = src
        self.dims = dims
        cv2.namedWindow('Grating Player')

    def play(self, mask_iter, ppl, export=False):
        cap = cv2.VideoCapture(self.src)

        def resize_with_padding(img, target_dim):
            height, width, channels = img.shape
            t_width, t_height = target_dim
            w_ratio = t_width / width
            h_ratio = t_height / height
            ratio = min(w_ratio, h_ratio)
            img = cv2.resize(img, None, fx=ratio, fy=ratio)
            if w_ratio < h_ratio:
                tmp = np.zeros((self.dims[1], img.shape[1], 3), np.uint8)
                start = int((self.dims[1] - int(round(ratio * s_height))) / 2)
                tmp[start : start + img.shape[0], :] = img
                img = tmp
            return img

        indicator = list(np.int32(np.zeros(self.dims[0])))
        while cap.isOpened():
            start = time.time()
            ret, frame = cap.read()
            if not ret:
                break
            s_height, s_width, channel = frame.shape
            try:
                mask = next(mask_iter)
            except StopIteration:
                logging.critical('Core engine failed to supply mask, player corrupted')
                return None

            # Div frame into frames
            starts = list(np.int32(np.linspace(0, frame.shape[1], self.channel_num + 1)))
            head = starts[0: -1]
            tail = starts[1:]
            sub_frames = []
            for i, j in zip(head, tail):
                tmp = frame[:, i: j]
                sub_frames.append(cv2.resize(tmp, None, fx=1 / self.channel_num, fy=1))

            offset = 0
            padding = False
            tailer = ppl - int(ppl)
            mask = list(mask)
            for i in range(len(indicator)):
                j = (i + offset) % int(ppl)
                if padding:
                    padding = False
                    indicator[i] = 0
                    offset -= 1
                    tailer -= 1
                elif j >= len(mask) - 1:
                    if np.abs(tailer - 1) < 0.1 or tailer >= 1:
                        padding = True
                    indicator[i] = mask[j]
                    tailer += ppl - int(ppl)
                else:
                    indicator[i] = mask[j]

            widthes = []
            for marker in range(self.channel_num):
                widthes.append(indicator.count(marker + 1))

            layers = []
            for i in range(len(sub_frames)):
                img = resize_with_padding(sub_frames[i], (widthes[i], self.dims[1]))
                layers.append(img)

            # for i in range(2):
            #     cv2.imshow(str(i), sub_frames[i])
            #     print(i, indicator.count(i + 1))

            # Build new frame
            new_frame = np.zeros((self.dims[1], self.dims[0], 3), np.uint8)
            pointers = [0, ] * self.channel_num
            for i in range(len(indicator)):
                index = indicator[i] - 1
                if index != -1:
                    try:
                        new_frame[:, i] = layers[index][:, pointers[index]]
                    except IndexError as e:
                        # print(e)
                        pass
                    pointers[index] += 1
                else:
                    continue

            if export:
                yield new_frame
            else:
                cv2.imshow('Grating Player', new_frame)
                if cv2.waitKey(30) & 0xFF == 27:
                    break


import logging
import numpy as np
import os
import pandas as pd
from hyperopt.pyll.stochastic import sample
from pathlib import Path
from pprint import pformat

from .schedule import (SerialScheduler, ConcurrentScheduler, MPSScheduler,
                       MIGScheduler, HFTAScheduler)
from .algorithms import RandomSearch, Hyperband
from .utils import (handle_integers, attach_common_args,
                    rearrange_algorithm_kwargs, generate_fusible_param_flags,
                    generate_nonfusible_param)

def _assert_positive_int_or_none(v):
  assert (v is None) or (isinstance(v, int) and v > 0)

def _expand_dict_if_valid(d, k, v):
  if v is not None:
    d[k] = v

def tune_hyperparameters(
    space={},
    try_params_callback=None,
    dry_run_callback=None,
    mode='hfta',
    algorithm='hyperband',
    nonfusibles=[],
    dry_run_repeats=None,
    dry_run_epochs=None,
    dry_run_iters_per_epoch=None,
    metric=None,
    goal='min',
    algorithm_configs={},
    seed=0,
    outdir=None,
):
  assert len(space) > 0
  assert try_params_callback is not None
  assert dry_run_callback is not None
  assert mode in {'serial', 'concurrent', 'mps', 'mig', 'hfta'}
  assert algorithm in {'hyperband', 'random'}
  assert nonfusibles is not None
  _assert_positive_int_or_none(dry_run_repeats)
  _assert_positive_int_or_none(dry_run_epochs)
  _assert_positive_int_or_none(dry_run_iters_per_epoch)
  assert metric is not None and isinstance(metric, str)
  assert goal in {'min', 'max'}
  assert isinstance(algorithm_configs, dict)
  assert isinstance(seed, int)
  assert outdir is None or isinstance(outdir, str)

  if mode == 'serial':
    scheduler = SerialScheduler(try_params_callback)
  else:
    kwargs = {
        'try_params_callback': try_params_callback,
        'dry_run_callback': dry_run_callback,
        'nonfusibles': nonfusibles,
    }
    _expand_dict_if_valid(kwargs, 'dry_run_repeats', dry_run_repeats)
    _expand_dict_if_valid(kwargs, 'dry_run_epochs', dry_run_epochs)
    _expand_dict_if_valid(
        kwargs,
        'dry_run_iters_per_epoch',
        dry_run_iters_per_epoch,
    )
    schedulers = {
        'concurrent': ConcurrentScheduler,
        'mps': MPSScheduler,
        'mig': MIGScheduler,
        'hfta': HFTAScheduler,
    }
    scheduler = schedulers[mode](**kwargs)

  rng_state = np.random.RandomState(seed=seed)

  def get_params():
    params = sample(space, rng=rng_state)
    return handle_integers(params)

  tuners = {'hyperband': Hyperband, 'random': RandomSearch}
  algorithm_config = {}
  if algorithm in algorithm_configs:
    for k, v in algorithm_configs[algorithm].items():
      _expand_dict_if_valid(algorithm_config, k, v)
  # To ensure a fair comparison between Hyperband and Random Search,
  # also pass in hyperband related parameters to decide the size
  # of candidate set, and the number of training epochs.
  tuner = tuners[algorithm](
      get_params,
      scheduler,
      metric,
      goal=goal,
      **algorithm_config,
  )

  history, trajectory = tuner.run()
  logging.info("\n=========================================================")
  logging.info("Done! the final results are:")
  logging.info("{} total, best:\n".format(len(history)))

  for trial in sorted(
      history.values(),
      key=lambda trial: trial['result'][metric],
      reverse=True,
  )[:5]:
    logging.info("{} | {:.1f}iterations | run {} ".format(
        pformat(trial['result']),
        trial['iterations'],
        trial['id'],
    ))
    pformat(trial['params'])

  if outdir is not None:
    logging.info('Saving results to {} ...'.format(outdir))
    Path(outdir).mkdir(parents=True, exist_ok=True)
    pd.json_normalize(history.values()).to_csv(
        os.path.join(outdir, 'history.csv'))
    pd.json_normalize(trajectory).to_csv(os.path.join(outdir, 'trajectory.csv'))

  return history, trajectory


#!/usr/local/bin/python3
#--------------------------------------------------------------------------------------------------
import sys,requests,getopt
from typing import List, Tuple

TPTP2TFormReplyURL = "https://www.tptp.org/cgi-bin/TPTP2TFormReply"

USAGE = f"Usage: RemoteSOT <options> [<File name>]\n \
    <options> are ...\n \
    -v              --version    - print version information\n \
    -h --help       --help       - print this help\n \
    -q <quietness>  --quiet      - control amount of output\n \
    -p <print>      --print      - pp (problems), ps (solutions), pps (both)\n \
    -d '<domains>'  --domains    - list of domains in ''s\n \
    -f <form>       --form       - THF,TH1,TH0,TXF,TX1,TX0,TFF,TF1,TF0,FOF,CNF\n \
    -r <min>-<max>  --rating     - Min and Max in the range 0.00 to 1.00\n"

VERSION = f"{sys.argv[0]} version 1.0.0"

DEFAULT_URL_PARAMETERS = {
    "NoHTML": (None, '1'),
    "QuietnessOption1": (None, '-q2'),
    "PrintFlag1": (None, '-pps'),
    "Domains": (None,'X9X'),
    "Form": (None,'ANY')
    }
#--------------------------------------------------------------------------------------------------
def parseCommandLine(args: List[str]):

    urlParameters = DEFAULT_URL_PARAMETERS

    options, arguments = getopt.getopt(
#----Arguments
        args,
#----Short option definitions
        'vhq:p:d:f:r:',
#----Long option definitions
        ["version", "help", "quiet=", "print=", "domains=", "form=", "rating=" ])

    for option, parameter in options:
#DEBUG        print(f"Option is ==={option}=== Parameter is ==={parameter}===")
        if option in ("-v", "--version"):
            print(VERSION)
            sys.exit()
        if option in ("-h", "--help"):
            print(USAGE)
            sys.exit()
#    -q<quietness>   --quiet      - control amount of output\n \
        if option in ("-q", "--quiet"):
            urlParameters["QuietnessOption1"] = (None, '-q'+parameter)
        if option in ("-p", "--print"):
            urlParameters["PrintFlag1"] = (None, '-p'+parameter)
        if option in ("-d", "--domains"):
            urlParameters["Domains"] = (None, 'Domains '+parameter)
        if option in ("-f", "--form"):
            urlParameters["Form"] = (None, parameter)
        if option in ("-r", "--rating"):
            urlParameters["Rating"] = (None, 'Rating__')
            minMax = parameter.split("-")
            if len(minMax) != 2:
                print(USAGE)
                sys.exit()
            urlParameters["Rating1"] = (None, minMax[0])
            urlParameters["Rating2"] = (None, minMax[1])

    return urlParameters
#--------------------------------------------------------------------------------------------------
def main() -> None:

    args = sys.argv[1:]
    if not args:
        print(USAGE)
        sys.exit()
    urlParameters = parseCommandLine(args)

#DEBUG    print(urlParameters)
    requestResult = requests.post(TPTP2TFormReplyURL,files=urlParameters)
    print(requestResult.text)
#--------------------------------------------------------------------------------------------------
if __name__ == '__main__':
    main()
#--------------------------------------------------------------------------------------------------


import os.path
debug = os.path.exists("debug")

import logging
if debug:
	logging.basicConfig(level=logging.DEBUG)
else:
	logging.basicConfig(filename='app.log', filemode='w', level=logging.ERROR)

from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker as _sessionmaker

if debug:
	engine = create_engine('sqlite:///database.db')
else:
	engine = create_engine('mysql+mysqlconnector://root:kNG7Dm2yC6dGfAD3AVkYjtYh_@localhost/qmarket', pool_size=20)

sessionmaker = _sessionmaker(bind=engine)