'''Cross validation

bug tracker
-----------------------------
priority | name
-----------------------------
1        | rubi, how accuracy so high?? 64%!
2        | should use PURGED K-FOLD Cross Validation or  TimeSeriesSplit instead of standart split
3        | rubi, which normalize function to use?
'''

from __future__ import absolute_import, division, print_function, unicode_literals

import tensorflow as tf

from data_splitter.splitter_k_fold_purged import PurgedKFoldDataSplitter
from buld.utils import *

import keras
from keras.layers import Dense, Dropout
from keras.models import Sequential
from keras.optimizers import RMSprop

# import pandas.io.data as web

# https://towardsdatascience.com/deep-learning-for-beginners-practical-guide-with-python-and-keras-d295bfca4487
# https://www.youtube.com/watch?v=aircAruvnKk

batch_size = 128  # we cannot pass the entire data into network at once , so we divide it to batches . number of samples that we will pass through the network at 1 time and use for each epoch. default is 32
epochs = 100  # 50  #  iterations. on each, train all data, then evaluate, then adjust parameters (weights and biases)
# iterations  = 60000/128
# size_input = 10  # x_train.shape[1] # no of features
size_hidden = 512  # If a model has more hidden units (a higher-dimensional representation space), and/or more layers, then the network can learn more complex representations. However, it makes the network more computationally expensive and may lead to overfit
size_output = 2  # there are 3 classes (buy.sell. hold) or (green,red,hold)
name_output = ['Green bar', 'Red Bar']  # 'no direction Bar'
# iterations  = 60000/128
symbol = '^GSPC'  # ^GSPC=SP500 (3600->1970 or 12500 =2000) DJI(300, 1988)  QQQ(300, 2000) GOOG XLF XLV
skipDays = 3600  # 12500 total 13894 daily bars
percentTestSplit = 0.33  # 33% from data will go to test

print('\nLoading  data')
print('\n======================================')
# Define date range
# start_date, end_date='1970-01-03','2019-07-12'
# dates=pd.date_range(start_date,end_date)
# print("dates="  ,dates)
# print("date[0]=",dates[0])
#
# # Define stock symbols
# symbols = []#'TSLA', 'GOOG', 'FB']  # SPY will be added in get_data()
# import get_prices as hist
# hist.get_stock_data(symbol, start_date=start_date, end_date=end_date)
# process = DataProcessing("stock_prices.csv", 0.9)
# process.gen_test(10)
# process.gen_train(10)
#

# Get stock data
df_all = get_data_from_disc(symbol, skipDays)
print(df_all.tail())

# Slice and plot
# plot_selected(df_all, [  'Close', 'sma200'], shouldNormalize=True, symbol=symbol)

# Slice and plot
plot_selected(df_all, title='TA-price of ' + symbol + ' vs time', columns=['Close', 'sma200'], shouldNormalize=False,
              symbol=symbol)

plot_selected(df_all.tail(500), title='TA-sma 1,10,20,50,200 of ' + symbol + ' vs time',
              columns=['Close', 'sma10', 'sma20', 'sma50', 'sma200', 'sma400', 'bb_hi10', 'bb_lo10', 'bb_hi20',
                       'bb_lo20', 'bb_hi50', 'bb_lo200', 'bb_lo50', 'bb_hi200'], shouldNormalize=False, symbol=symbol)

plot_selected(df_all.tail(500), title='TA-range sma,bband of ' + symbol + ' vs time',
              columns=['range_sma', 'log_sma20', 'log_sma50', 'log_sma200', 'log_sma400', 'rel_bol_hi10',
                       'rel_bol_hi20', 'rel_bol_hi200', 'rel_bol_hi50'], shouldNormalize=False, symbol=symbol)
plot_selected(df_all.tail(500), title='TA-rsi,stoc of ' + symbol + ' vs time',
              columns=['rsi10', 'rsi20', 'rsi50', 'rsi200', 'stoc10', 'stoc20', 'stoc50', 'stoc200'],
              shouldNormalize=False, symbol=symbol)

# plot_selected(df, ['Date','Close']                                    , start_date, end_date, shouldNormalize=False)
elements = df_all.size
shape = df_all.shape

print('\nInput Data')
print('\n======================================')
df_data = df_all.loc[:,
          ['sma10', 'sma20', 'sma50', 'sma200', 'sma400', 'range_sma', 'log_sma20', 'log_sma50', 'log_sma200',
           'log_sma400', 'bb_hi10', 'bb_lo10', 'bb_hi20', 'bb_lo20', 'bb_hi50', 'bb_lo50', 'bb_hi200', 'bb_lo200',
           'rel_bol_hi10', 'rel_bol_lo10', 'rel_bol_hi20', 'rel_bol_lo20', 'rel_bol_hi50', 'rel_bol_lo50',
           'rel_bol_hi200', 'rel_bol_lo200', 'rsi10', 'rsi20', 'rsi50', 'rsi200', 'stoc10', 'stoc20', 'stoc50',
           'stoc200']]
print('columns=', df_data.columns)
print('\ndata describe=\n', df_data.describe())
print('shape=', str(shape), " elements=" + str(elements), ' rows=', str(shape[0]))

print('\nOutput Data (Labeling)')
print('\n======================================')
df_y = df_all['isUp']  # np.random.randint(0,2,size=(shape[0], ))
# y = np.random.randint(0,2,size=(shape[0], ))
# print(y)
# df_y = pd.DataFrame()#, columns=list('is_up'))
# df_y['isUp'] = y
print(df_y)

ts_cv = PurgedKFoldDataSplitter    (n_splits=5, gap_percentage=2.5)
#ts_cv = StratifiedKFold(n_splits=5, shuffle=False)

# ts_cv = TimeSeriesSplit(n_splits=5)
cv_scores = []
for train_indices, test_indices in ts_cv.split(df_data.values):
    x_train = df_data.values[train_indices]
    x_test = df_data.values[test_indices]
    print('x_train shape=', str(x_train.shape))
    print('x_test shape=', str(x_test.shape))

    print('Observations: %d' % (len(x_train) + len(x_test)))
    print('Training Observations: %d' % (len(x_train)))
    print('Testing Observations: %d' % (len(x_test)))

    x_train = tf.keras.utils.normalize(x_train, axis=1)
    x_test = tf.keras.utils.normalize(x_test, axis=1)
    print('\ntrain data', x_train.shape)
    print('\ntest data', x_test.shape)

    y_train = df_y.values[train_indices]
    y_test = df_y.values[test_indices]
    print('y_train shape=', str(y_train.shape))
    print('y_test shape=', str(y_test.shape))

    y_train = keras.utils.to_categorical(y_train, size_output)
    y_test = keras.utils.to_categorical(y_test, size_output)
    print('\ntrain labels', y_train.shape)
    print('\ntest labels', y_test.shape)

    size_input = x_train.shape[1]  # no of features

    # create model
    model = Sequential()  # stack of layers
    # model.add(tf.keras.layers.Flatten())
    model.add(Dense(size_hidden, activation='relu', input_shape=(size_input,)))
    model.add(Dropout(0.2))  # for generalization
    model.add(Dense(size_hidden, activation='relu'))
    model.add(Dropout(0.2))  # regularization technic by removing some nodes
    model.add(Dense(size_output,
                    activation='softmax'))  # last layer always has softmax(except for regession problems and  binary- 2 classes where sigmoid is enough)
    # For binary classification, softmax & sigmoid should give the same results, because softmax is a generalization of sigmoid for a larger number of classes.
    # softmax:  loss: 0.3099 - acc: 0.8489 - val_loss: 0.2929 - val_acc: 0.8249
    # sigmoid:  loss: 0.2999 - acc: 0.8482 - val_loss: 0.1671 - val_acc: 0.9863

    # Prints a string summary of the  neural network.')
    model.summary()

    # Compile model
    model.compile(loss='categorical_crossentropy',  # measure how accurate the model during training
                  optimizer=RMSprop(),  # this is how model is updated based on data and loss function
                  metrics=['accuracy'])

    # Fit the model
    # model.fit(df_data[train], df_y[train], epochs=150, batch_size=10, verbose=0)
    model.fit(x_train, y_train, epochs=epochs, batch_size=batch_size, verbose=1)

    # history = model.fit(x_train
    #                     , y_train
    #                     , batch_size=batch_size
    #                     , epochs=epochs
    #                     , validation_data=(x_test, y_test)
    #                     , verbose=1)

    # evaluate the model
    scores = model.evaluate(x_test, y_test, verbose=0)
    print("----end of itteration: %s: %.2f%%" % (model.metrics_names[1], scores[1] * 100))
    cv_scores.append(scores[1] * 100)

print("-------total accuracy : %.2f%% (+/- %.2f%%)" % (np.mean(cv_scores), np.std(cv_scores)))


from os import scandir, rename, mkdir
from os.path import splitext, exists, join
from shutil import move
from time import sleep
import sys
import time
import logging
import os
import watchdog
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
from datetime import datetime

# ! FILL IN BELOW IF DIRECTORY OF DESKTOP FOLDER IS DIFFERENT !

# ? folder to track e.g. Windows: "C:\\Users\\, os.getlogin(), \\Downloads"    ! os.getlogin() CALLS USER NAME FROM OS !
source_dir = "".join(("C:\\Users\\", os.getlogin(), "\\OneDrive\\Desktop"))
dest_dir_sfx = "".join(("C:\\Users\\", os.getlogin(), "\\OneDrive\\Desktop\\SFX"))
dest_dir_music = "".join(("C:\\Users\\", os.getlogin(), "\\OneDrive\\Desktop\\MUSIC"))
dest_dir_video = "".join(("C:\\Users\\", os.getlogin(), "\\OneDrive\\Desktop\\VIDEO"))
dest_dir_image = "".join(("C:\\Users\\", os.getlogin(), "\\OneDrive\\Desktop\\IMAGE"))
dest_dir_documents = "".join(("C:\\Users\\", os.getlogin(), "\\OneDrive\\Desktop\\DOCUMENTS"))
dest_dir_software = "".join(("C:\\Users\\", os.getlogin(), "\\OneDrive\\Desktop\\SOFTWARE"))
dest_dir_blender = "".join(("C:\\Users\\", os.getlogin(), "\\OneDrive\\Desktop\\BLENDER"))
dest_dir_script = "".join(("C:\\Users\\", os.getlogin(), "\\OneDrive\\Desktop\\SCRIPTS"))

# ? supported image types
image_extensions = [".jpg", ".jpeg", ".jpe", ".jif", ".jfif", ".jfi", ".png", ".gif", ".webp", ".tiff", ".tif", ".psd", ".raw", ".arw", ".cr2", ".nrw",
                    ".k25", ".bmp", ".dib", ".heif", ".heic", ".ind", ".indd", ".indt", ".jp2", ".j2k", ".jpf", ".jpf", ".jpx", ".jpm", ".mj2", ".svg", ".svgz", ".ai", ".eps", ".ico"]
# ? supported Video types
video_extensions = [".webm", ".mpg", ".mp2", ".mpeg", ".mpe", ".mpv", ".ogg",
                    ".mp4", ".mp4v", ".m4v", ".avi", ".wmv", ".mov", ".qt", ".flv", ".swf", ".avchd"]
# ? supported Audio types
audio_extensions = [".m4a", ".flac", "mp3", ".wav", ".wma", ".aac"]
# ? supported Document types
document_extensions = [".doc", ".docx", ".odt",
                       ".pdf", ".xls", ".xlsx", ".ppt", ".pptx"]
# ? supported Software types
software_extensions = [".exe", ".rar", ".zip",
                       ".bat", ".msi"]
# ? supported Blender types
blender_extensions = [".blend", ".fbx", ".stl",
                       ".obj", ".ply", "x3d"]
# ? supported Script types
script_extensions = [".py", ".php", ".java",
                       ".html"]

def make_unique(dest, name):
    filename, extension = splitext(name)
    counter = 1
    now = datetime.now()
    current_time = now.strftime("%Hh%Mm")

    # * IF FILE EXISTS, ADDS NUMBER TO THE END OF THE FILENAME
    while exists(f"{dest}/{name}"):
        name = f"{filename}({str(counter)}[{current_time}]){extension}"
        counter += 1

    return name

def move_file(dest, entry, name):
    if exists(f"{dest}/{name}"):
        unique_name = make_unique(dest, name)
        oldName = join(dest, name)
        newName = join(dest, unique_name)
        rename(oldName, newName)
    move(entry, dest)

# BLENDER #

pathblender = "C:\\Users\\andre\\OneDrive\\Desktop\\BLENDER"
# Check whether the specified path exists or not
isExist = os.path.exists(pathblender)

if not isExist:

  # Create a new directory because it does not exist
  os.makedirs(pathblender)
  print("The new directory is created!")

# VIDEO #

pathvideo = "C:\\Users\\andre\\OneDrive\\Desktop\\VIDEO"
# Check whether the specified path exists or not
isExist = os.path.exists(pathvideo)

if not isExist:

  # Create a new directory because it does not exist
  os.makedirs(pathvideo)
  print("The new directory is created!")

# IMAGE #

pathimage = "C:\\Users\\andre\\OneDrive\\Desktop\\IMAGE"
# Check whether the specified path exists or not
isExist = os.path.exists(pathimage)

if not isExist:

  # Create a new directory because it does not exist
  os.makedirs(pathimage)
  print("The new directory is created!")

# SFX #

pathsfx = "C:\\Users\\andre\\OneDrive\\Desktop\\SFX"
# Check whether the specified path exists or not
isExist = os.path.exists(pathsfx)

if not isExist:

  # Create a new directory because it does not exist
  os.makedirs(pathsfx)
  print("The new directory is created!")

# MUSIC #

pathmusic = "C:\\Users\\andre\\OneDrive\\Desktop\\MUSIC"
# Check whether the specified path exists or not
isExist = os.path.exists(pathmusic)

if not isExist:

  # Create a new directory because it does not exist
  os.makedirs(pathmusic)
  print("The new directory is created!")

# SOFTWARE #

pathsoftware = "C:\\Users\\andre\\OneDrive\\Desktop\\SOFTWARE"
# Check whether the specified path exists or not
isExist = os.path.exists(pathsoftware)

if not isExist:

  # Create a new directory because it does not exist
  os.makedirs(pathsoftware)
  print("The new directory is created!")

# DOCUMENTS #

pathdocument = "C:\\Users\\andre\\OneDrive\\Desktop\\DOCUMENTS"
# Check whether the specified path exists or not
isExist = os.path.exists(pathdocument)

if not isExist:

  # Create a new directory because it does not exist
  os.makedirs(pathdocument)
  print("The new directory is created!")

# SCRIPTS #

pathscript = "C:\\Users\\andre\\OneDrive\\Desktop\\SCRIPTS"
# Check whether the specified path exists or not
isExist = os.path.exists(pathscript)

if not isExist:

  # Create a new directory because it does not exist
  os.makedirs(pathscript)
  print("The new directory is created!")

class MoverHandler(FileSystemEventHandler):
    # ? THIS FUNCTION WILL RUN WHENEVER THERE IS A CHANGE IN "source_dir"
    # ? .upper is for not missing out on files with uppercase extensions
    def on_modified(self, event):
        with scandir(source_dir) as entries:
            for entry in entries:
                name = entry.name
                self.check_audio_files(entry, name)
                self.check_video_files(entry, name)
                self.check_image_files(entry, name)
                self.check_document_files(entry, name)
                self.check_software_files(entry, name)
                self.check_blender_files(entry, name)
                self.check_script_files(entry, name)

    def check_audio_files(self, entry, name):  # * Checks all Audio Files
        for audio_extension in audio_extensions:
            if name.endswith(audio_extension) or name.endswith(audio_extension.upper()):
                if entry.stat().st_size < 3_000_000 or "SFX" in name:  # ? 3Megabytes
                    dest = dest_dir_sfx
                else:
                    dest = dest_dir_music
                move_file(dest, entry, name)
                logging.info(f"Moved audio file: {name}")

    def check_video_files(self, entry, name):  # * Checks all Video Files
        for video_extension in video_extensions:
            if name.endswith(video_extension) or name.endswith(video_extension.upper()):
                move_file(dest_dir_video, entry, name)
                logging.info(f"Moved video file: {name}")

    def check_image_files(self, entry, name):  # * Checks all Image Files
        for image_extension in image_extensions:
            if name.endswith(image_extension) or name.endswith(image_extension.upper()):
                move_file(dest_dir_image, entry, name)
                logging.info(f"Moved image file: {name}")

    def check_document_files(self, entry, name):  # * Checks all Document Files
        for documents_extension in document_extensions:
            if name.endswith(documents_extension) or name.endswith(documents_extension.upper()):
                move_file(dest_dir_documents, entry, name)
                logging.info(f"Moved document file: {name}")

    def check_software_files(self, entry, name):  # * Checks all Software Files
        for software_extension in software_extensions:
            if name.endswith(software_extension) or name.endswith(software_extension.upper()):
                move_file(dest_dir_software, entry, name)
                logging.info(f"Moved software file: {name}")

    def check_blender_files(self, entry, name):  # * Checks all Software Files
        for blender_extension in blender_extensions:
            if name.endswith(blender_extension) or name.endswith(blender_extension.upper()):
                move_file(dest_dir_blender, entry, name)
                logging.info(f"Moved blender file: {name}")

    def check_script_files(self, entry, name):  # * Checks all Script Files
        for script_extension in script_extensions:
            if name.endswith(script_extension) or name.endswith(script_extension.upper()):
                move_file(dest_dir_script, entry, name)
                logging.info(f"Moved script file: {name}")

# ! NO NEED TO CHANGE BELOW CODE
if __name__ == "__main__":
    logging.basicConfig(level=logging.INFO,
                        format='%(asctime)s - %(message)s',
                        datefmt='%Y-%m-%d %H:%M:%S')
    path = source_dir
    event_handler = MoverHandler()
    observer = Observer()
    observer.schedule(event_handler, path, recursive=True)
    observer.start()
    try:
        while True:
            sleep(10)
    except KeyboardInterrupt:
        observer.stop()
    observer.join()


import gym
from ppo import PPO

env = gym.make('Pendulum-v1')
hyperparameters = {
    'n_steps': 128,
    'n_timesteps_per_batch': 4800,
    'max_timesteps_per_episode': 1600,
    'n_updates_per_iteration': 5,
    'gamma': 0.99,
    'lr': 2.5e-4,
    'clip_range': 0.2,
    'anneal_lr': False,
    'cuda': True
}
model = PPO(env)
model.learn(100000)

obs = env.reset()
done = False
while not done:
    env.render()
    action = model.get_action(obs)
    obs, reward, done, _ = env.step(action)
    #time.sleep(0.01)


import json
import shutil
import pytest
import textwrap
try:
    from .utils import get_test_id, build_example, run_example
    from . import circuits
except BaseException:
    from utils import get_test_id, build_example, run_example
    import circuits

def test_1():
    name = f'ckt_{get_test_id()}'
    netlist = textwrap.dedent(f"""\
        .subckt {name} vbias2 vccx
        mp29 v4 vbias2 v2 vccx p w=2.16e-6 m=1 nf=12
        mp33 vbias2 vbias2 vbias1 vccx p w=1.44e-6 m=1 nf=8
        .ends {name}
        """)
    setup = textwrap.dedent("""\
        POWER = vccx
        GND =
        """)
    constraints = []
    example = build_example(name, netlist, setup, constraints)
    ckt_dir, run_dir = run_example(example, cleanup=False)

    with (run_dir / '1_topology' / '__primitives__.json').open('rt') as fp:
        primitives = json.load(fp)
        for key, _ in primitives.items():
            assert key.startswith('PMOS') or key.startswith('DCL'), f"Incorrect subcircuit identification {key}"

    shutil.rmtree(run_dir)
    shutil.rmtree(ckt_dir)


from django.contrib.auth.models import User
from django.db.models.query_utils import Q
from django.http.response import HttpResponse
from rest_framework import viewsets

from bookmarks.models import Bookmark
from bookmarks.serializers import UserSerializer, BookmarkSerializer

def home(request):
    return HttpResponse('Hello, World!')

class UserViewSet(viewsets.ModelViewSet):
    serializer_class = UserSerializer
    queryset = User.objects.all()

class BookmarkViewSet(viewsets.ModelViewSet):
    serializer_class = BookmarkSerializer
    def get_queryset(self):
        queryset = Bookmark.objects.all()
        scope = self.request.query_params.get('scope', None)
        if scope == 'all' :
            return queryset.exclude(~Q(user = self.request.user.id), isPublic = False)
        if scope == 'others' :
            return queryset.exclude(user = self.request.user).exclude(isPublic = False)
        # show only mine
        return queryset.filter(user = self.request.user.id)


import numpy as np
import pytest

import pineappl

class TestOrder:
    def test_init(self):
        args = (2, 1, 0, 1)
        o = pineappl.grid.Order(*args)

        assert isinstance(o, pineappl.grid.Order)
        assert isinstance(o.raw, pineappl.pineappl.PyOrder)
        assert o.as_tuple() == args

class TestGrid:
    def fake_grid(self, bins=None):
        lumis = [pineappl.lumi.LumiEntry([(1, 21, 0.1)])]
        orders = [pineappl.grid.Order(3, 0, 0, 0)]
        bin_limits = np.array([1e-7, 1e-3, 1] if bins is None else bins, dtype=float)
        subgrid_params = pineappl.subgrid.SubgridParams()
        g = pineappl.grid.Grid.create(lumis, orders, bin_limits, subgrid_params)
        return g

    def test_init(self):
        g = self.fake_grid()
        assert isinstance(g, pineappl.grid.Grid)
        assert isinstance(g.raw, pineappl.pineappl.PyGrid)
        # orders
        assert len(g.orders()) == 1
        assert g.orders()[0].as_tuple() == (3, 0, 0, 0)

    def test_set_subgrid(self):
        g = self.fake_grid()

        # DIS grid
        xs = np.linspace(0.1, 1.0, 5)
        vs = np.random.rand(len(xs))
        subgrid = pineappl.import_only_subgrid.ImportOnlySubgridV1(
            vs[np.newaxis, :, np.newaxis],
            np.array([90.0]),
            np.array(xs),
            np.array([1.0]),
        )
        g.set_subgrid(0, 0, 0, subgrid)

        # let's mix it for fun with an hadronic one
        x1s = np.linspace(0.1, 1, 2)
        x2s = np.linspace(0.5, 1, 2)
        Q2s = np.linspace(10, 20, 2)
        subgrid = pineappl.import_only_subgrid.ImportOnlySubgridV1(
            np.random.rand(len(Q2s), len(x1s), len(x2s)), Q2s, x1s, x2s
        )
        g.set_subgrid(0, 1, 0, subgrid)
        g.optimize()

    def test_set_key_value(self):
        g = self.fake_grid()
        g.set_key_value("bla", "blub")
        g.set_key_value('"', "'")
        g.set_key_value("äöü", "ß\\")

    def test_bins(self):
        g = self.fake_grid()
        # 1D
        normalizations = [1.0] * 2
        limits = [(1, 1), (2, 2)]
        remapper = pineappl.bin.BinRemapper(normalizations, limits)
        g.set_remapper(remapper)
        assert g.bin_dimensions() == 1
        np.testing.assert_allclose(g.bin_left(0), [1, 2])
        np.testing.assert_allclose(g.bin_right(0), [1, 2])
        # 2D
        limits = [(1, 2), (2, 3), (2, 4), (3, 5)]
        remapper = pineappl.bin.BinRemapper(normalizations, limits)
        g.set_remapper(remapper)
        assert g.bin_dimensions() == 2
        np.testing.assert_allclose(g.bin_left(0), [1, 2])
        np.testing.assert_allclose(g.bin_right(0), [2, 4])
        np.testing.assert_allclose(g.bin_left(1), [2, 3])
        np.testing.assert_allclose(g.bin_right(1), [3, 5])

    def test_convolute_with_one(self):
        g = self.fake_grid()

        # DIS grid
        xs = np.linspace(0.5, 1.0, 5)
        vs = xs.copy()
        subgrid = pineappl.import_only_subgrid.ImportOnlySubgridV1(
            vs[np.newaxis, :, np.newaxis],
            np.array([90.0]),
            xs,
            np.array([1.0]),
        )
        g.set_subgrid(0, 0, 0, subgrid)
        np.testing.assert_allclose(
            g.convolute_with_one(2212, lambda pid, x, q2: 0.0, lambda q2: 0.0),
            [0.0] * 2,
        )
        np.testing.assert_allclose(
            g.convolute_with_one(2212, lambda pid, x, q2: 1, lambda q2: 1.0),
            [5e6 / 9999, 0.0],
        )
        np.testing.assert_allclose(
            g.convolute_with_one(2212, lambda pid, x, q2: 1, lambda q2: 2.0),
            [2**3 * 5e6 / 9999, 0.0],
        )

    def test_axes(self):
        g = self.fake_grid()

        # add 2 DIS grids
        xs = np.linspace(0.5, 1.0, 5)
        vs = np.random.rand(len(xs))
        subgrid = pineappl.import_only_subgrid.ImportOnlySubgridV1(
            vs[np.newaxis, :, np.newaxis],
            [90.0],
            xs,
            np.array([1.0]),
        )
        g.set_subgrid(0, 0, 0, subgrid)
        vs2 = np.random.rand(len(xs))
        subgrid = pineappl.import_only_subgrid.ImportOnlySubgridV1(
            vs2[np.newaxis, :, np.newaxis],
            [100.0],
            xs,
            np.array([1.0]),
        )
        g.set_subgrid(0, 1, 0, subgrid)
        # now get the thing
        ei = g.axes()

        np.testing.assert_allclose(ei[0], xs)
        np.testing.assert_allclose(ei[1], [])
        np.testing.assert_allclose(ei[2], [90.0, 100.0])

    def test_io(self, tmp_path):
        g = self.fake_grid()
        p = tmp_path / "test.pineappl"
        p.write_text("")
        g.write(str(p))
        gg = pineappl.grid.Grid.read(p)
        assert isinstance(gg, pineappl.grid.Grid)
        _ = pineappl.grid.Grid.read(str(p))

    def test_convolute_eko(self):
        g = self.fake_grid()
        fake_eko = {
            "q2_ref": 1.0,
            "targetpids": [1],
            "targetgrid": [0.1, 1.0],
            "inputpids": [1],
            "inputgrid": [0.1, 1.0],
            "interpolation_xgrid": [0.1, 1.0],
            "Q2grid": {90: {"operators": np.random.rand(1, 2, 1, 2), "alphas": 1.0}},
        }
        g.set_key_value("lumi_id_types", "pdg_mc_ids")
        #  fk = g.convolute_eko(fake_eko)
        #  assert isinstance(fk.raw, pineappl.pineappl.PyFkTable)

    def test_fill(self):
        g = self.fake_grid()
        g.fill(0.5, 0.5, 10.0, 0, 0.01, 0, 10.0)
        res = g.convolute_with_one(2212, lambda pid, x, q2: x, lambda q2: 1.0)
        pytest.approx(res) == 0.0

    def test_fill_array(self):
        g = self.fake_grid()
        g.fill_array(
            np.array([0.5, 1.0]),
            np.array([0.5, 1.0]),
            np.array([0.5, 1.0]),
            0,
            np.array([1e-3, 1e-2]),
            0,
            np.array([10.0, 100.0]),
        )
        res = g.convolute_with_one(2212, lambda pid, x, q2: x, lambda q2: 1.0)
        pytest.approx(res) == 0.0

    def test_fill_all(self):
        g = self.fake_grid()
        g.fill_all(1.0, 1.0, 1.0, 0, 1e-2, np.array([10.0]))
        res = g.convolute_with_one(2212, lambda pid, x, q2: x, lambda q2: 1.0)
        pytest.approx(res) == 0.0

    def test_merge(self):
        g = self.fake_grid([1, 2, 3])
        g1 = self.fake_grid([3, 4, 5])
        assert g.bins() == 2
        assert g1.bins() == 2

        g.merge(g1)
        assert g.bins() == 4

        g2 = self.fake_grid([1, 2, 3])
        g3 = self.fake_grid([1, 2, 3])
        assert g2.bins() == 2
        assert g3.bins() == 2

        g2.merge(g3)
        assert g2.bins() == 2

        g4 = self.fake_grid([2, 3, 4])
        g5 = self.fake_grid([4, 5, 6])
        assert g4.bins() == 2
        assert g5.bins() == 2

        with pytest.raises(ValueError, match="NonConsecutiveBins"):
            g2.merge(g4)

        with pytest.raises(ValueError, match="NonConsecutiveBins"):
            g2.merge(g5)


import os
import sys
import hashlib
import pickle
from tqdm import tqdm
from pytube import YouTube
from video_wrangling_config import grab_params

def download_videos(params_key, video_dir, hash_dict_filename):
    if not os.path.exists(video_dir):
        os.makedirs(video_dir)

    p = grab_params(params_key)
    thingies = [] #(category, URL, hash(URL)) triplets
    categories = sorted(p.video_urls.keys())
    for category in categories:
        for video_url in p.video_urls[category]:
            video_hash = hashlib.md5(video_url.encode('utf8')).hexdigest()
            thingies.append((category, video_url, video_hash))

    #check for collisions
    all_hashes = [x[2] for x in thingies]
    assert(len(set(all_hashes)) == len(all_hashes))

    #make hash dict
    hash_dict = {x[2] : x[1] for x in thingies}
    with open(hash_dict_filename, 'wb') as f:
        pickle.dump(hash_dict, f)

    for category, video_url, video_hash in tqdm(thingies):
        yt = YouTube(video_url)
        best_stream = None
        for res in p.resolutions:
            my_streams = yt.streams.filter(**(p.pytube_kwargs), res=res)
            if len(my_streams) > 0:
                best_stream = my_streams[0] #arbitrarily pick first one
                break

        if best_stream is None:
            print('Could not download from URL "%s" in category "%s"'%(video_url, category))
            continue

        filename = category + '-' + video_hash + '-' + str(best_stream.itag)
        best_stream.download(output_path=video_dir, filename=filename)

def usage():
    print('Usage: python download_videos.py <params_key> <video_dir> <hash_dict_filename>')

if __name__ == '__main__':
    download_videos(*(sys.argv[1:]))


# -*- coding: utf-8 -*-
"""
Created on Wed Aug 10 11:58:35 2016

@author: Eward Comyn-Platt
"""

import netCDF4 as nc
from SoilTools import BrooksCorey as BC
from SoilTools import VanGenucthen as VG

Input_Dir='/users/eow/edwcom/GREENHOUSE/SOIL_PROPERTIES/datasets/'
Comp_File=Input_Dir+'Merged_Soil_Composition_CHESSgrid.nc'

Output_Dir=Input_Dir
OUT_BC_file=Output_Dir+'SoilProperties_BC_CHESSgrid.nc'
OUT_VG_file=Output_Dir+'SoilProperties_VG_CHESSgrid.nc'

# Read Soil Composition File
INPUT_vars=['sand','silt','clay','carbon']
COMP_data={}
COMPinf=nc.Dataset(Comp_File,'r')
for var in INPUT_vars:
    COMP_data[var]=COMPinf.variables[var][:]
COMPinf.close()

# Compute the Brooks and Corey Soil Properties
BC_Properties=BC.get_BC_soil_properties(COMP_data['clay'],\
                                        COMP_data['sand'],\
                                        COMP_data['silt'] )

# Compute the Brooks and Corey Soil Properties
VG_Properties=VG.get_VG_soil_properties(COMP_data['clay'],        \
                                        COMP_data['sand'],        \
                                        COMP_data['silt'],        \
                                        OC_PC=COMP_data['carbon'],\
                                        RETURN_SOIL_TEXTURE=True  )

print(BC_Properties)

import matplotlib.pyplot as plt

plotdata=COMP_data['sand']

plt.figure(figsize=(15,15))
for iSD in range(4):
    plt.subplot(2,2,iSD+1)
    plt.imshow(plotdata[iSD,:],origin='bottom')
    plt.colorbar()

plt.show()


from typing import Tuple, List, Optional
import numpy as np
import heapq
import torch
import math

from federeco.config import DEVICE, TOPK

def _precision(predicted, actual):
    prec = [value for value in predicted if value in actual]
    prec = float(len(prec)) / float(len(predicted))
    return prec

def _apk(rank_list: List, item: int) -> float:
    """
    Computes the average precision at k.
    Parameters
    ----------
    actual : list
        A list of actual items to be predicted
    predicted : list
        An ordered list of predicted items
    k : int, default = 10
        Number of predictions to consider
    Returns:
    -------
    score : float
        The average precision at k.
    """
    predicted = rank_list
    actual = [item]
    if not predicted or not actual:
        return 0.0

    score = 0.0
    true_positives = 0.0

    for i, p in enumerate(predicted):
        if p in actual and p not in predicted[:i]:
            max_ix = min(i + 1, len(predicted))
            score += _precision(predicted[:max_ix], actual)
            true_positives += 1

    if score == 0.0:
        return 0.0
    return score / true_positives

def _ark(rank_list: List, item: int):
    """
    Computes the average recall at k.
    Parameters
    ----------
    actual : list
        A list of actual items to be predicted
    predicted : list
        An ordered list of predicted items
    k : int, default = 10
        Number of predictions to consider
    Returns:
    -------
    score : float
        The average recall at k.
    """
    score = 0.0
    num_hits = 0.0
    predicted = rank_list
    actual = [item]
    for i,p in enumerate(predicted):
        if p in actual and p not in predicted[:i]:
            num_hits += 1.0
            score += num_hits / (i+1.0)

    if not actual:
        return 0.0

    return score / len(actual)

def get_metrics(rank_list: List, item: int) -> Tuple[int, float, float, float]:
    """
    Used for calculating hit rate & normalized discounted cumulative gain (ndcg)
    :param rank_list: Top-k list of recommendations
    :param item: item we are trying to match with `rank_list`
    :return: tuple containing 1/0 indicating hit/no hit & ndcg & ap@k & ar@k
    """
    if item not in rank_list:
        return 0, 0, 0, 0
    return 1, math.log(2) / math.log(rank_list.index(item) + 2), _apk(rank_list, item), _ark(rank_list, item)

def evaluate_model(model: torch.nn.Module,
                   users: List[int], items: List[int], negatives: List[List[int]],
                   k: Optional[int] = TOPK) -> Tuple[float, float]:
    """
    calculates hit rate and normalized discounted cumulative gain for each user across each item in `negatives`
    returns average of top-k list of hit rates and ndcgs
    """

    hits, ndcgs, apks , arks = list(), list(), list(), list()
    for user, item, neg in zip(users, items, negatives):

        item_input = neg + [item]

        with torch.no_grad():
            item_input_gpu = torch.tensor(np.array(item_input), dtype=torch.int, device=DEVICE)
            user_input = torch.tensor(np.full(len(item_input), user, dtype='int32'), dtype=torch.int, device=DEVICE)
            pred, _ = model(user_input, item_input_gpu)
            pred = pred.cpu().numpy().tolist()

        map_item_score = dict(zip(item_input, pred))
        rank_list = heapq.nlargest(k, map_item_score, key=map_item_score.get)
        hr, ndcg, apk, ark = get_metrics(rank_list, item)
        hits.append(hr)
        ndcgs.append(ndcg)
        apks.append(apk)
        arks.append(ark)

    return np.array(hits).mean(), np.array(ndcgs).mean(), np.array(apks).mean(), np.array(arks).mean()


import pygame
from pygame.locals import *

pygame.init()

screen_width: int = 700
screen_height: int = 700

screen: pygame.Surface = pygame.display.set_mode((screen_width, screen_height), 0, 32)
pygame.display.set_caption("Using Mouse")
screen.fill((0, 0, 0))

sprite1: pygame.Surface = pygame.image.load('./images/butterfly.png')
sprite1 = pygame.transform.scale(sprite1, (64, 64))
sprite_width: int = sprite1.get_width()
sprite_height: int = sprite1.get_height()

x: int = 0
y: int = 0

clock: int = pygame.time.Clock()
game_over: bool = False
while not game_over:
    dt = clock.tick(100)
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            game_over = True
        elif event.type == pygame.MOUSEMOTION:
            x, y = event.pos
            x -= sprite_width / 2
            y -= sprite_height / 2

    pressed = pygame.key.get_pressed()
    if pressed[K_UP]:
        y -= 0.5 * dt
    if pressed[K_DOWN]:
        y += 0.5 * dt
    if pressed[K_RIGHT]:
        x += 0.5 * dt
    if pressed[K_LEFT]:
        x -= 0.5 * dt
    if pressed[K_SPACE]:
        x, y = 0, 0

    x = min(screen_width - sprite_width, x)
    x = max(0, x)
    y = min(screen_height - sprite_height, y)
    y = max(0, y)

    screen.fill((128, 0, 255))
    screen.blit(sprite1, (x, y))
    pygame.display.update()
pygame.quit()


from transposition.ngramTransposition import decrypt_horizontal, decrypt_vertical, convert_to_tuple
from random import randrange
from copy import deepcopy
from numpy.random import permutation
from itertools import combinations
from substitution.manualSubDecoder import color
from englishDetection import ngramScore, englishScore

#decrypts a ciphertext by using a custom algorithm
#Algorithm doesn't work 100% of the time, somewhat unpredictable
#It is kind of like bogosort: https://en.wikipedia.org/wiki/Bogosort

def decrypt(ciphertext, lowerBound, upperBound, n=1, direction='H', num_of_decryptions=10, num_of_times=200,
            fitness_score=ngramScore.ngram_score(ngramScore.QUADGRAMS).score): #this is the default fitness score

    decryptMessage = decrypt_horizontal if direction.upper() == 'H' else decrypt_vertical

    scores_list = []

    #generate two random indexes
    for key in range(lowerBound, upperBound + 1):
        #max num of allowed failures (new score not greater than old)
        #randomly choose a starting initial permutation (the "best" permutation only initially)
        best_perm = list(permutation(key))
        #initial "best" is decryption with the initial permutation
        best_plaintext = decryptMessage(key=tuple(best_perm), message=ciphertext)
        best_score = fitness_score(best_plaintext)
        #add initial as an entry
        scores_list.append((best_score, best_perm, best_plaintext))
        g = get_switched_permutations(best_perm)
        for i in range(num_of_times):
           entries = []
           for j in range(len(g)):
               new_perm = switch(best_perm, g[j][0], g[j][1])
               new_plaintext = decryptMessage(key=tuple(new_perm), message=ciphertext)
               new_score = fitness_score(new_plaintext)
               entry = (new_score, new_perm, new_plaintext)
               entries.append(entry)

           entries.sort(key=lambda x: x[0], reverse=True)
           if not entries[0][0] > best_score: #if this seems to be a possible decryption, add it to the scores-list
               best_entry = (best_score, best_perm, best_plaintext)
               #print it out
               print("%-10.0f %-50s %-10s" % (best_score, best_perm, best_plaintext))
               if not best_entry in scores_list:
                   scores_list.append(best_entry)
               #re-randomize best values
               best_perm = list(permutation(key))
               best_plaintext = decryptMessage(key=tuple(best_perm), message=ciphertext)
               best_score = fitness_score(best_plaintext)

           else:
                best_score, best_perm, best_plaintext = entries[0][0], entries[0][1], entries[0][2]

    print()
    scores_list.sort(key=lambda x: x[0], reverse=True)
    return scores_list[:num_of_decryptions + 1] if len(scores_list) <= num_of_decryptions else scores_list

def print_best_solutions(scores_list):
    print(color.RED + color.BOLD + " BEST SOLUTIONS: " + color.END)
    print((color.BOLD + "%10s %15s %54s" + color.END) % ("Score:", "Key:", "Message:"))
    for i in range(10):
        try:
            print("%-5d %-15.0f %-50s %-10s" % (i+1, scores_list[i][0], str(tuple(scores_list[i][1])), scores_list[i][2]))
        except IndexError:
            break

#returns list with switched elements at two distinct indexes in a list
def switch(li, pos1, pos2):
    return_li = deepcopy(li)
    return_li[pos1], return_li[pos2] = return_li[pos2], return_li[pos1]
    return return_li

#returns a list containing all the possible index swaps on a permutation
def get_switched_permutations(perm):
    return list(combinations(range(len(perm)), 2))

def main():
    message = input("Enter a message: ")
    remove = input("Remove spaces? [y/n]: ")
    if remove == 'y':
        message = ''.join(message.split())
    direction = input("horizontal or vertical <h/v>?: ").upper()
    n = int(input("n = "))
    lower = int(input("Lower bound: "))
    upper = int(input("Upper bound: "))
    num_of_times = int(input("How many iterations for each permutation length?: "))

    print_best_solutions(decrypt(message, lower, upper, n, direction, num_of_times,
    fitness_score= lambda x: englishScore.english_word_score(x, englishScore.COMMON_WORDS, 2)))
    # fitness score can be changed! Sometimes works better with ngram logarithmic probabilities, n = 3

if __name__ == '__main__':
    main()


from __future__ import annotations

from pathlib import Path

import pytest

from poetry.repositories.parsers.pypi_search_parser import Result
from poetry.repositories.parsers.pypi_search_parser import SearchResultParser

FIXTURES_DIRECTORY = Path(__file__).parent.parent / "fixtures" / "pypi.org" / "search"

@pytest.fixture
def search_page_data() -> str:
    with FIXTURES_DIRECTORY.joinpath("search.html").open(encoding="utf-8") as f:
        return f.read()

def test_search_parser(search_page_data: str) -> None:
    parser = SearchResultParser()
    parser.feed(search_page_data)
    assert parser.results == [
        Result(
            name="SQLAlchemy",
            version="1.3.10",
            description="Database Abstraction Library",
        ),
        Result(
            name="SQLAlchemy-Dao",
            version="1.3.1",
            description="Simple wrapper for sqlalchemy.",
        ),
        Result(
            name="graphene-sqlalchemy",
            version="2.2.2",
            description="Graphene SQLAlchemy integration",
        ),
        Result(
            name="SQLAlchemy-UTCDateTime",
            version="1.0.4",
            description=(
                "Convert to/from timezone aware datetimes when storing in a DBMS"
            ),
        ),
        Result(
            name="paginate_sqlalchemy",
            version="0.3.0",
            description="Extension to paginate.Page that supports SQLAlchemy queries",
        ),
        Result(
            name="sqlalchemy_audit",
            version="0.1.0",
            description=(
                "sqlalchemy-audit provides an easy way to set up revision "
                "tracking for your data."
            ),
        ),
        Result(
            name="transmogrify.sqlalchemy",
            version="1.0.2",
            description="Feed data from SQLAlchemy into a transmogrifier pipeline",
        ),
        Result(
            name="sqlalchemy_schemadisplay",
            version="1.3",
            description="Turn SQLAlchemy DB Model into a graph",
        ),
        Result(name="sqlalchemy_traversal", version="0.5.2", description="UNKNOWN"),
        Result(
            name="sqlalchemy-filters",
            version="0.10.0",
            description="A library to filter SQLAlchemy queries.",
        ),
        Result(
            name="SQLAlchemy-wrap",
            version="2.1.7",
            description="Python wrapper for the CircleCI API",
        ),
        Result(
            name="sqlalchemy-nav",
            version="0.0.2",
            description=(
                "SQLAlchemy-Nav provides SQLAlchemy Mixins for creating "
                "navigation bars compatible with Bootstrap"
            ),
        ),
        Result(
            name="sqlalchemy-repr",
            version="0.0.1",
            description="Automatically generates pretty repr of a SQLAlchemy model.",
        ),
        Result(
            name="sqlalchemy-diff",
            version="0.1.3",
            description="Compare two database schemas using sqlalchemy.",
        ),
        Result(
            name="SQLAlchemy-Equivalence",
            version="0.1.1",
            description=(
                "Provides natural equivalence support for SQLAlchemy "
                "declarative models."
            ),
        ),
        Result(
            name="Broadway-SQLAlchemy",
            version="0.0.1",
            description="A broadway extension wrapping Flask-SQLAlchemy",
        ),
        Result(
            name="jsonql-sqlalchemy",
            version="1.0.1",
            description="Simple JSON-Based CRUD Query Language for SQLAlchemy",
        ),
        Result(
            name="sqlalchemy-plus",
            version="0.2.0",
            description="Create Views and Materialized Views with SqlAlchemy",
        ),
        Result(
            name="CherryPy-SQLAlchemy",
            version="0.5.3",
            description="Use SQLAlchemy with CherryPy",
        ),
        Result(
            name="sqlalchemy_sqlany",
            version="1.0.3",
            description="SAP Sybase SQL Anywhere dialect for SQLAlchemy",
        ),
    ]


# This code uses TensorFlow and Keras to build and train a deep learning model for image classification. It loads a pre-trained MobileNetV2 model and adds custom layers. The model is compiled and trained on the training data, validated on the validation data, and evaluated on the test data. The PlotCallback class creates a plot of the accuracy and loss of the model during training. The ReduceLROnPlateau and EarlyStopping callbacks are used to adjust the learning rate and stop training early if the validation loss does not improve. The final trained model is saved to a file.

###################################
# Author: Sharjeel Arif
# SID: 200388331
# 08th March 2023
###################################

# REVISION HISTORY

# 08 MAR 23: Created file to train machine learning model

# 08 MAR 23: Followed steps of setting up training model on deeplizard

# 09 MAR 23: Was able to train first model
# Model was scratch. First model did not use transfer learning

# 11 Mar 23: Consulted Dr. Bais regarding accuracy of a trained model
# Suggested I use transfer learning on a pre-existing model
# Rather than training a model from scratch

# 12 MAR 23: Implemented changes to import VGG19 Model and use transfer learning to train

# 15 MAR 23: Implemented changes to import VGG16 model and use transfer learning to train

# 20 MAR 23: Implemented changes to import MobileNetV2 model and use transfer learning to train

# 21 MAR 23: Trained first few iterations off 600 size dataset. Inaccurate

# 23 MAR 23: Consulted Dr. Bais to increase accuracy of model
# Was advised to increase dataset

# 29 Mar 23: Increased dataset using augmentation

# 30 Mar 23: Trained new MobileNetV2 model

# 30 Mar 23: Tested Model, Adjusted Parameters (Added dense and dropout layers and unfroze layers), Trained another

# 30 Mar 23: Tested Model, Adjusted Parameters (Learning Rate), Trained another

# 30 Mar 23: Tested Model, Adjusted Parameters (Adjusted frozen layers), Trained another

# 31 Mar 23: Tested Model, Adjusted Parameters (Adjusted frozen layers), Trained another

import tensorflow as tf
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.layers import Dense, Dropout
from tensorflow.keras.models import Model
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.applications.mobilenet_v2 import MobileNetV2
from tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping
import matplotlib.pyplot as plt
import numpy as np

class PlotCallback(tf.keras.callbacks.Callback):
    def on_train_begin(self, logs=None):
        self.acc = []
        self.val_acc = []
        self.loss = []
        self.val_loss = []

    def on_epoch_end(self, epoch, logs=None):
        self.acc.append(logs.get('accuracy'))
        self.val_acc.append(logs.get('val_accuracy'))
        self.loss.append(logs.get('loss'))
        self.val_loss.append(logs.get('val_loss'))

        plt.figure(figsize=(10, 5))

        plt.subplot(1, 2, 1)
        plt.plot(np.arange(epoch + 1), self.acc, label='Train')
        plt.plot(np.arange(epoch + 1), self.val_acc, label='Validation')
        plt.title('Model accuracy')
        plt.ylabel('Accuracy')
        plt.xlabel('Epoch')
        plt.legend()
        plt.grid(True)

        plt.subplot(1, 2, 2)
        plt.plot(np.arange(epoch + 1), self.loss, label='Train')
        plt.plot(np.arange(epoch + 1), self.val_loss, label='Validation')
        plt.title('Model loss')
        plt.ylabel('Loss')
        plt.xlabel('Epoch')
        plt.legend()
        plt.grid(True)

        plt.tight_layout()
        plt.show(block=False)
        plt.pause(1)
        plt.close()

    def save_plot(self, file_name='training_plot4.png'):
        plt.figure(figsize=(10, 5))

        plt.subplot(1, 2, 1)
        plt.plot(self.acc, label='Train')
        plt.plot(self.val_acc, label='Validation')
        plt.title('Model accuracy')
        plt.ylabel('Accuracy')
        plt.xlabel('Epoch')
        plt.legend()
        plt.grid(True)

        plt.subplot(1, 2, 2)
        plt.plot(self.loss, label='Train')
        plt.plot(self.val_loss, label='Validation')
        plt.title('Model loss')
        plt.ylabel('Loss')
        plt.xlabel('Epoch')
        plt.legend()
        plt.grid(True)

        plt.tight_layout()
        plt.savefig(file_name)
        plt.close()

# Define the paths to the training, validation, and test directories
train_dir = 'dataset/train'
val_dir = 'dataset/val'
test_dir = 'dataset/test'

# Define the image size and batch size
img_size = (224, 224)
batch_size = 16

# Define the number of output classes
num_classes = 4

# Define the MobileNetV2 model with pre-trained weights from ImageNet
base_model = MobileNetV2(input_shape=(img_size[0], img_size[1], 3), include_top=False, weights='imagenet')

# Freeze all layers in the base model
for layer in base_model.layers:
    layer.trainable = False

# Unfreeze the last 30 layers
for layer in base_model.layers[-35:]:
    layer.trainable = True

# Add custom layers on top of the base model for your specific use case
x = base_model.output
x = tf.keras.layers.GlobalAveragePooling2D()(x)
x = Dense(128, activation='relu')(x)
x = Dropout(0.5)(x)
x = Dense(64, activation='relu')(x)  # Additional Dense layer
x = Dropout(0.5)(x)  # Additional Dropout layer
x = Dense(32, activation='relu')(x)  # New dense layer
x = Dropout(0.5)(x)  # New dropout layer
predictions = Dense(num_classes, activation='softmax')(x)

# Create the model
model = Model(inputs=base_model.input, outputs=predictions)

# Compile the model with the desired optimizer, loss function, and metrics
model.compile(optimizer=Adam(lr=1e-4), loss='categorical_crossentropy', metrics=['accuracy'])

# Print a summary of the model architecture
model.summary()

# Define the data generators for training, validation, and test sets
datagen = ImageDataGenerator(
    rescale=1./255,
    validation_split=0.2
)

train_generator = datagen.flow_from_directory(
    train_dir,
    target_size=img_size,
    batch_size=batch_size,
    class_mode='categorical',
    subset='training'
)

val_generator = datagen.flow_from_directory(
    val_dir,
    target_size=img_size,
    batch_size=batch_size,
    class_mode='categorical',
    subset='validation'
)

test_generator = datagen.flow_from_directory(
    test_dir,
    target_size=img_size,
    batch_size=batch_size,
    class_mode='categorical'
)

# Define the ReduceLROnPlateau and EarlyStopping callbacks
reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=3, min_lr=1e-6)
early_stopping = EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)

# Train the model on the training data, and validate it on the validation data
plot_callback = PlotCallback()

history = model.fit(
    train_generator,
    epochs=30,
    validation_data=val_generator,
    callbacks=[reduce_lr, early_stopping, plot_callback]
)

# Save final plot
plot_callback.save_plot('final_training_plot4.png')

# Evaluate the model on the test data
test_loss, test_acc = model.evaluate(test_generator)
print('Test accuracy:', test_acc)

# Save the trained model to a file (optional)
model.save('baby_posture_modelv4.h5')


import xlrd,pymysql,time
from xlutils.copy import copy
import traceback
## ##读取目录下文件的第一页
##确文件的第六列跟第七列分别是离职与交接人的email
## ##
TIMEFORMAT="%Y-%m-%d %X"
data =xlrd.open_workbook('YW05-离职流程报表 (32).xls')
table = data.sheets()[0] ##通过索引顺序获取
##table = data.sheet_by_index(0) #通过索引顺序获取
##table =data.sheet_by_name(u'Sheet1')#通过名称获取 ##  获取整行和整列的值（数组）

try:
    ##测试环境and '@' in table.row_values(line)[9]
##        conn_online=pymysql.connect(host='****',user='****',passwd='****',db='****',port=3306,charset='utf8')
    ##线上环境
        conn_online=pymysql.connect(host='*******',user='****',passwd='****',db='****',port=3306,charset='utf8')
        cur_online=conn_online.cursor()#获取一个游标
##    cur_test=conn_test.cursor()
except  Exception as  e:
          print("发生异常",e)

for line in range(1,table.nrows):
    if table.row_values(line)[4] :
        quit_name =  table.row_values(line)[3]
        quit_email =  table.row_values(line)[5]
        jie_name = table.row_values(line)[6]
        jie_email = table.row_values(line)[7]

        try:
            sql_check_quit='select * from t_customer_aid_quit where quit_customer_email= "' +quit_email+'"'
            cur_online.execute(sql_check_quit)
            data=cur_online.fetchall()
            if len(data) != 0 and data:###处理已经离职过的数据
##                table.row_values(line)[13]='离职账号'+quit_email+'已经离过职';
##                wb = copy(table)
##                ws = wb.get_sheet(0)
##                ws.write(line,13,'离职账号'+quit_email+'  已经存在')
                sql_check_quit_2='select * from t_customer_aid where email= "' +jie_email+'"'
                cur_online.execute(sql_check_quit_2)
                data_2=cur_online.fetchall()
                if  len(data_2)==0:
                        print('接收人'+jie_email+' 不存在，未做离职处理')
                        continue
                if data_2 and data_2[0][7] ==3:
                    print('交接人'+data_2[0][2]+data_2[0][1]+'是分总，未做离职处理   ')
                    continue
                if data_2  and data_2[0][1] != jie_name:
                        if data_2[0][1] != jie_name+'1':
                            print('接收人：'+data_2[0][2]+':'+data_2[0][1]+'与 excel ：'+jie_email+':'+jie_name+' 姓名不一致!未做离职处理')
                            continue
                print("UPDATE t_customer_aid_quit SET  take_customer_id='"+str(data_2[0][0])+"', take_customer_name='"+data_2[0][1]+"', take_customer_email='"+data_2[0][2]+"',quit_time='"+time.strftime( TIMEFORMAT, time.localtime( ) )+"' ,account_process_status='0', relation_process_status='1', task_process_status='0' WHERE id='"+str(data[0][0])+"';")
                continue
##            sql_onlie1 ='select id,name,email from t_customer_aid where email="'+jie_email+'"'
##            status = cur_online.execute(sql_online1)
##            data=cur_online.fetchall()
##            if len(data) == 0:
##                 print('接收人账号'+jie_email +' 不存在')
##                 continue
###未离职处理
            sql_online ='select a.id,a.name,a.email,b.id,b.name,b.email,"0", "1", "0", "1",a.role from t_customer_aid a,(select id,name,email from t_customer_aid where email="'+jie_email+'") b  where a.email="'+quit_email+'"'
            status = cur_online.execute(sql_online)
            ##          print('sql_online: ')
            ##          print(sql_test)
            data=cur_online.fetchall()
            if len(data) == 0:
                print('接收人'+jie_email+' 或关闭助手人'+quit_email+' 不存在，未做离职处理')
                continue
            for d in data :
                if d[1] != quit_name:
                    print('关闭助手权限人：'+d[2]+':'+d[1]+'与 excel ：'+quit_email+':'+quit_name+' 姓名不一致')
                    continue
                if d[4] != jie_name:
                    print('接收人：'+d[4]+':'+d[5]+'与 excel ：'+jie_email+':'+jie_name+' 姓名不一致')
                    continue
##                if d[10] == 3:
##                        sql_online = ''
##                        if
##                    print('交接人'+d[4]+':'+d[5]+'是分总，未做离职处理   ')
##                    continue
                sql_test='insert into t_customer_aid_quit(quit_customer_id,quit_customer_name,quit_customer_email,take_customer_id,take_customer_name,take_customer_email,quit_time,account_process_status,relation_process_status,task_process_status,create_time,state) '
                sql_test +='values("'+str(d[0])+'","'+str(d[1])+'","'+str(d[2])+'","'+str(d[3])+'","'+str(d[4])+'","'+str(d[5])+'","'+\
                time.strftime( TIMEFORMAT, time.localtime( ) )+'","'+str(d[6])+'","'+str(d[7])+'","'+str(d[8])+'","'+time.strftime( TIMEFORMAT, time.localtime( ) )+'","'+str(d[9])+'")'
                ##              cur_test.execute(sql_test)
                ##              conn_test.commit()
                ##              print('sql_test: ')
                print(sql_test+';')
        except  Exception as  e:
            print("发生异常",e)
            traceback.print_exc()
    else:
        print("email没有@")

cur_online .close()#关闭游标
##cur_test.close()
conn_online.close()#释放数据库资源
##conn_test.close()
print("总计数量："+str(line))

##print(table.col_values(1))


#!/usr/bin/env python
# coding: utf-8

# ### 09 cats and dog

# In[1]:

# cats_and_dogs classification model with CNN
# train : 2000 images [cat(1000) + dog(1000)]  , size는 다름
# validation : 1000 images [cat(500) + dog(500)] , size는 다름

# In[3]:

import tensorflow as tf
import urllib
import zipfile
from tensorflow.keras.optimizers import RMSprop
from tensorflow.keras.preprocessing.image import ImageDataGenerator

# In[4]:

# cats_and_dogs 데이터셋 다운로드 , Windows용
_TRAIN_URL = "https://storage.googleapis.com/mledu-datasets/cats_and_dogs_filtered.zip"
urllib.request.urlretrieve(_TRAIN_URL, 'tmp/cats_and_dogs_filtered.zip')

# In[6]:

#압축해제
local_zip = 'tmp/cats_and_dogs_filtered.zip'
zip_ref = zipfile.ZipFile(local_zip,'r')
zip_ref.extractall('tmp/')
zip_ref.close()

# In[7]:

# 이미지 데이터 경로 설정
import os

base_dir = 'tmp/cats_and_dogs_filtered'

train_dir = os.path.join(base_dir, 'train')
validation_dir = os.path.join(base_dir, 'validation')

# Directory with our training cat/dog pictures
train_cats_dir = os.path.join(train_dir, 'cats')
train_dogs_dir = os.path.join(train_dir, 'dogs')

# Directory with our validation cat/dog pictures
validation_cats_dir = os.path.join(validation_dir, 'cats')
validation_dogs_dir = os.path.join(validation_dir, 'dogs')
train_cat_fnames = os.listdir( train_cats_dir )
train_dog_fnames = os.listdir( train_dogs_dir )

print(train_cat_fnames[:10])
print(train_dog_fnames[:10])

print('total training cat images :', len(os.listdir(train_cats_dir ) ))
print('total training dog images :', len(os.listdir(train_dogs_dir ) ))

print('total validation cat images :', len(os.listdir( validation_cats_dir ) ))
print('total validation dog images :', len(os.listdir( validation_dogs_dir ) ))

# In[11]:

# 데이터 시각화
import matplotlib.image as mpimg
import matplotlib.pyplot as plt

# Parameters for our graph; we'll output images in a 4x4 configuration
nrows = 4
ncols = 4

pic_index = 0 # Index for iterating over images ,시작 인덱스

# Set up matplotlib fig, and size it to fit 4x4 pics
fig = plt.gcf()
fig.set_size_inches(ncols*3, nrows*3)

pic_index+=8

next_cat_pix = [os.path.join(train_cats_dir, fname)
                for fname in train_cat_fnames[ pic_index-8:pic_index]
               ]
next_dog_pix = [os.path.join(train_dogs_dir, fname)
                for fname in train_dog_fnames[ pic_index-8:pic_index]
               ]

for i, img_path in enumerate(next_cat_pix+next_dog_pix):
    # Set up subplot; subplot indices start at 1
    sp = plt.subplot(nrows, ncols, i + 1)
    sp.axis('Off') # Don't show axes (or gridlines)

    img = mpimg.imread(img_path)
    plt.imshow(img)

plt.show()

# ### CNN model 구현

# In[17]:

model = tf.keras.Sequential([
    tf.keras.layers.Conv2D(16,(3,3),activation='relu',input_shape=(150,150,3)),
    tf.keras.layers.MaxPool2D(2,2),
    tf.keras.layers.Conv2D(32,(3,3),activation='relu'),
    tf.keras.layers.MaxPool2D(2,2),
    tf.keras.layers.Conv2D(64,(3,3),activation='relu'),
    tf.keras.layers.MaxPool2D(2,2),
    tf.keras.layers.Flatten(),
    tf.keras.layers.Dense(512,activation='relu'),
    tf.keras.layers.Dense(1,activation='sigmoid')
])
model.compile(optimizer=RMSprop(learning_rate=0.001),
             loss='binary_crossentropy',
             metrics=['accuracy'])
model.summary()

# ### 텐서플로 이미지 증강 구현

# In[18]:

# rescale: 1./255

train_datagen = ImageDataGenerator(rescale=1./255)
#print(type(train_datagen))

validation_datagen = ImageDataGenerator(rescale=1./255)

#train data의 generator
train_generator = train_datagen.flow_from_directory(
                  train_dir,
                  target_size=(150,150), # resize될 크기
                  batch_size=20,
                  class_mode='binary' # 2진 분류
)

#validation data의 generator
validation_generator = validation_datagen.flow_from_directory(
                  validation_dir,
                  target_size=(150,150), # resize될 크기
                  batch_size=20,
                  class_mode='binary' # 2진 분류
)

print(train_generator)

# In[19]:

# 학습
history = model.fit(
                train_generator, # x
                steps_per_epoch=100, # 2000/20 = 8.xxx, train image number/train batch_size
                epochs=15,
                validation_data=validation_generator,
                validation_steps=50 # 1000/20
)

# In[20]:

# 학습 결과 시각화
#-----------------------------------------------------------
# Retrieve a list of list results on training and test data
# sets for each training epoch
#-----------------------------------------------------------
acc      = history.history[     'accuracy' ]
val_acc  = history.history[ 'val_accuracy' ]
loss     = history.history[    'loss' ]
val_loss = history.history['val_loss' ]

epochs   = range(len(acc)) # Get number of epochs

#------------------------------------------------
# Plot training and validation accuracy per epoch
#------------------------------------------------
plt.plot(epochs, acc, 'g', label='Training accuracy')
plt.plot(epochs, val_acc, 'r', label='Validation accuracy')
plt.title('Training and validation accuracy')
plt.legend()
plt.figure()

# ### 이미지 분류 예측

# In[23]:

import numpy as np
from tensorflow.keras.preprocessing import image
def image_predict(file_name):
    path = 'cats_and_dogs예측용이미지/' + file_name
    img = image.load_img(path, target_size=(150, 150))
    x = image.img_to_array(img)   # 2차원 ndarray로 변환
    x = np.expand_dims(x, axis=0) # 3차원으로 변환
    images = np.vstack([x])       # 4차원으로 변환
    classes = model.predict(images, batch_size=10)
    # print(classes[0],end=' ')
    if classes[0] == 1.0:
        print(file_name ,': dog')
    else:
        print(file_name ,': cat')
image_predict('cat_01.jpg')
image_predict('cat_02.jpg')
image_predict('cat_03.jpg')
image_predict('cat_04.jpg')
image_predict('cat_05.jpg')

image_predict('dog_01.jpg')
image_predict('dog_02.jpg')
image_predict('dog_03.jpg')
image_predict('dog_04.jpg')
image_predict('dog_05.jpg')

image_predict('cat_dog_01.jpg')
image_predict('cat_dog_02.jpg')
image_predict('cat_dog_03.jpg')
image_predict('cat_dog_04.jpg')
image_predict('cat_dog_05.jpg')

# ### 모델 개선

# In[24]:

model = tf.keras.Sequential([
    tf.keras.layers.Conv2D(32,(3,3),activation='relu',input_shape=(150,150,3)),
    tf.keras.layers.MaxPool2D(2,2),
    tf.keras.layers.Conv2D(64,(3,3),activation='relu'),
    tf.keras.layers.MaxPool2D(2,2),
    tf.keras.layers.Conv2D(128,(3,3),activation='relu'),
    tf.keras.layers.MaxPool2D(2,2),
    tf.keras.layers.Conv2D(128,(3,3),activation='relu'),
    tf.keras.layers.MaxPool2D(2,2),
    tf.keras.layers.Dropout(0.5),

    tf.keras.layers.Flatten(),
    tf.keras.layers.Dense(512,activation='relu'),
    tf.keras.layers.Dense(1,activation='sigmoid')
])
model.compile(optimizer=RMSprop(learning_rate=0.0001), # 1e-4
             loss='binary_crossentropy',
             metrics=['accuracy'])
model.summary()

# In[25]:

# 이미지 증강

train_datagen = ImageDataGenerator(rescale=1./255,
                                  rotation_range=40,
                                  width_shift_range=0.2,
                                  height_shift_range=0.2,
                                  shear_range=0.2,
                                  zoom_range=0.2,
                                  horizontal_flip=True)
#print(type(train_datagen))

validation_datagen = ImageDataGenerator(rescale=1./255,
                                  rotation_range=40,
                                  width_shift_range=0.2,
                                  height_shift_range=0.2,
                                  shear_range=0.2,
                                  zoom_range=0.2,
                                  horizontal_flip=True)

#train data의 generator
train_generator = train_datagen.flow_from_directory(
                  train_dir,
                  target_size=(150,150), # resize될 크기
                  batch_size=20,
                  class_mode='binary', # 2진 분류

)

#validation data의 generator
validation_generator = validation_datagen.flow_from_directory(
                  validation_dir,
                  target_size=(150,150), # resize될 크기
                  batch_size=20,
                  class_mode='binary', # 2진 분류

)

print(train_generator)

# In[26]:

# 학습
history = model.fit(
                train_generator, # x
                steps_per_epoch=100, # 2000/20 = 8.xxx, train image number/train batch_size
                epochs=30,
                validation_data=validation_generator,
                validation_steps=50 # 1000/20
)

# 성능이 약간 개선됨
# 100 epochs로 학습
# 기타 : 이미지의 shape을 (300,300,3)로 할 경우 정확도 감소됨

# In[27]:

# 시각화
import matplotlib.pyplot as plt
acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']

epochs = range(len(acc))

plt.plot(epochs, acc, 'bo', label='Training accuracy')
plt.plot(epochs, val_acc, 'r', label='Validation accuracy')
plt.title('Training and validation accuracy')

plt.figure()

plt.plot(epochs, loss, 'bo', label='Training Loss')
plt.plot(epochs, val_loss, 'r', label='Validation Loss')
plt.title('Training and validation loss')
plt.legend()

plt.show()

# In[ ]:


from django.contrib.auth.models import User
from django.shortcuts import render, HttpResponse, redirect
# Create your views here.
from blog.forms import PostCreatForm, PostModifyForm
from blog.models import Post

def main_view(request):
    posts = Post.objects.order_by('-created_date')
    context = {
        'posts':posts
    }
    return render(request,'post/post-list.html' ,context=context)

def post_add_view(request):
    if request.method == "GET":
        form = PostCreatForm()
        context = {
            'form': form
        }
        return render(request, 'post/post-add.html', context=context)

    elif request.method == "POST":
        form = PostCreatForm(request.POST)
        if form.is_valid():
            title = form.cleaned_data['title']
            text = form.cleaned_data['text']
            user = User.objects.first().id
            Post.objects.create(author_id=user,title = title, text = text)
            return redirect('main_view')
        else:
            context = {
                'form':form,
            }
            return render(request, 'post/post-add.html',context=context)

def post_modify_view(request,pk):
    post = Post.objects.get(id=pk)
    if request.method == "GET":
        form = PostModifyForm(initial={'title': post.title, 'text':post.text})
        context = {
            'form': form
        }
        return render(request, 'post/post-modify.html', context=context)

    elif request.method == "POST":
        form = PostModifyForm(request.POST)
        if form.is_valid():
            title = form.cleaned_data['title']
            text = form.cleaned_data['text']
            post.title=title
            post.text=text
            post.save()
            return redirect('main_view')
        else:
            context = {
                'form':form,
            }
            return render(request, 'post/post-modify.html',context=context)


from cStringIO import StringIO
import json
import unittest

from etl import __main__ as main

from fixtures import TEST_CONTRIBUTIONS_LIST

class TestMain(unittest.TestCase):
    pass

    def test_main_module(self):

        input_file = StringIO(TEST_CONTRIBUTIONS_LIST)
        output_file = StringIO()
        main.main(input_file, output_file)

        output_file.seek(0)
        output = [
            json.loads(row)
            for row in output_file.readlines()
        ]

        excepted = [
            {"Contribution Date": "2/2/2004", "Contributor Name": "Inc, Caldera   Management", "Contributor Address": "4260 Hwy 1, Rehoboth, DE 19971"},
            {"Contribution Date": "9/22/2004", "Contributor Name": "Conset-vancy/LLC, Rehoboth   Bay", "Contributor Address": "1207 Delaware AveWilinin too/DE 19806, DE"},
            {"Contribution Date": "9/17/2004", "Contributor Name": "lingua, James.A", "Contributor Address": "28 The Circle.Georgetown/DE 19947, DE"}
        ]

        self.assertEqual(output, excepted)


#===============================================================================
# UserCheckpoint action layer
#===============================================================================
from util.geo import bounding_box, proximity_sort, distance_between_points
from sqlalchemy.sql.expression import and_, or_, union_all, alias, desc
from action.user import get_friends
from collections import namedtuple
from action.checkpoint import CHECKPOINT_TYPES
from action.common import _get_2_weeks_date_before
from sqlalchemy.orm.util import aliased
from action.like import get_total_likes
import datetime
from action.notification import delete_notifications_w_user_checkpoint

def checkpoint_proximity(user_checkpoints, lat, lon):
    """
    returns a list of a dict containing usercheckpoint id, checkpoint ids, and their proximity
    from the given lat, lon coordinate
    """

    lis = [{"user_checkpoint_id" : ucp.id,
            "checkpoint_id": ucp.checkpoint.id,
            "proximity_in_km": distance_between_points(lat, lon, ucp.checkpoint.latitude, ucp.checkpoint.longitude),
            } for ucp in user_checkpoints]
    return lis

def get_user_checkpoint(id):
    """
    gets UserCheckpoint record by id
    """
    from db import UserCheckpoint, db
    cp = UserCheckpoint.query.filter_by(id=id)
    if cp.count() > 0:
        return cp.first()
    return None

def get_user_checkpoint_attr(user_obj, checkpoint_obj):
    """
    gets UserCheckpoint record given supplied args
    """
    from db import UserCheckpoint, db
    cp = UserCheckpoint.query.filter_by(user_id = user_obj.id, checkpoint_id = checkpoint_obj.id)
    if cp.count() > 0:
        return cp.first()
    return None

def remove_checkpoint_from_user(user_obj, checkpoint_obj):
    from db import db

    #ensure that user does own the user_checkpoint
    user_checkpoint_obj = get_user_checkpoint_attr(user_obj, checkpoint_obj)

    #delete notifications
    delete_notifications_w_user_checkpoint(user_checkpoint_obj)

    if not user_checkpoint_obj is None:
        delete_notifications_w_user_checkpoint(user_checkpoint_obj)
        db.session.delete(user_checkpoint_obj)
        db.session.commit()

def add_checkpoint_to_user(user_obj, checkpoint_obj):
    """
    Adds a record to the db table to reflect an addition of an NEW Checkpoint
    to a user's repertoir of Checkpoints
    """
    ucp = get_user_checkpoint_attr(user_obj, checkpoint_obj)
    if not ucp is None:
        return ucp

    from db import UserCheckpoint, db

    user_checkpoint = UserCheckpoint()
    user_checkpoint.user_id = user_obj.id
    user_checkpoint.checkpoint_id = checkpoint_obj.id
    user_checkpoint.date_added = datetime.datetime.now()

    db.session.add(user_checkpoint)
    db.session.commit()

    return user_checkpoint

def add_existing_checkpoint_to_user(user_obj, user_checkpoint_obj):
    """
    Duplicates a record of an instance of <<UserCheckpoint>> and gives it to
    the provided <<User>>.

    Also recursively duplicates corresponding <<UserCheckpointOptions>> and gives it to the
    duplicated instance of <<UserCheckpoint>>
    """

    ucp = get_user_checkpoint_attr(user_obj, user_checkpoint_obj.checkpoint)
    if not ucp is None:
        return ucp

    from db import UserCheckpoint, UserCheckpointOptions, db

    #duplicate UserCheckpoint
    duplicated_ucp = UserCheckpoint()
    duplicated_ucp.user_id = user_obj.id
    duplicated_ucp.checkpoint_id = user_checkpoint_obj.checkpoint_id
    duplicated_ucp.date_added = datetime.datetime.now()
    db.session.add(duplicated_ucp)

    #duplicate UserCheckpointOptions
    options = UserCheckpointOptions.query.filter_by(user_checkpoint_id=user_checkpoint_obj.id)
    for opt in options:
        duplicated_opt = UserCheckpointOptions()
        duplicated_opt.user_checkpoint_id = duplicated_ucp.id
        duplicated_opt.name = opt.name
        duplicated_opt.value = opt.value
        db.session.add(duplicated_opt)

    db.session.commit()

    return duplicated_ucp

def get_nearby_checkpoints(user_obj, point_coord, radius_in_kilometres):
    """
    get UserCheckpoints in a given radius sorted according to proximity
    returns in tuple, (friends_user_checkpoints_list, anonymous_user_checkpoints_list)
    """
    from db import UserCheckpoint, db, Checkpoint

    #bounding box
    lat, lon = point_coord[0], point_coord[1]

    dlat, dlon = bounding_box(lat, lon, radius_in_kilometres)
    min_lat, max_lat = lat-dlat, lat+dlat
    min_lon, max_lon = lon-dlon, lon+dlon

    radius_cond = and_(Checkpoint.latitude <= max_lat,
                       Checkpoint.latitude >= min_lat,
                       Checkpoint.longitude <= max_lon,
                       Checkpoint.longitude >= min_lon
                       )

    ucp_in_radius = (db.session.query(UserCheckpoint).
                     join(UserCheckpoint.checkpoint).
                     filter(radius_cond).
                     filter(Checkpoint.demo == False)
                     )

    #removing dupes and making sure that the oldest creator always gets credited
    unduped_ucp = {}
    all_ucp = ucp_in_radius.all()
    for ucp in all_ucp:
        key = ucp.checkpoint.id
        if key in unduped_ucp:
            compared_ucp = unduped_ucp[key]
            if ucp.user_id == user_obj.id:
                unduped_ucp[key] = ucp
            elif ucp < compared_ucp and compared_ucp.user_id != user_obj.id: #older
                unduped_ucp[key] = ucp
        else: unduped_ucp[key] = ucp
    all_ucp = [v for k,v in unduped_ucp.iteritems()]

    ucp_namedtuples = _checkpoints_to_location_namedtuples(all_ucp)
    sorted_ucp = proximity_sort((lat, lon), ucp_namedtuples, ucp_in_radius.count())

    #separate into friends and anon ucp
    friend_list = get_friends(user_obj)
    friends = []
    anon = []
    for ucp in sorted_ucp:
        if ucp.user_checkpoint.user in friend_list:
            friends += [ucp.user_checkpoint]
        else:
            anon += [ucp.user_checkpoint]

    return friends, anon

def sort_checkpoints(ucp_lis, sort_method, **kwargs):
    """
    sorts checkpoints according to method supplied.
    methods available are: newest, nearest, popular

    if "nearest" method is chosen, longitude/latitude kw args should be provided.
    """
    sorted_ucp = None

    if sort_method == "popular":
        sorted_ucp = sorted(ucp_lis,
                            key=lambda ucp: get_total_likes(ucp),
                            reverse=True)

    elif sort_method == "nearest":
        lat, lon = kwargs["latitude"], kwargs["longitude"]
        ucp_namedtuples = _checkpoints_to_location_namedtuples(ucp_lis)
        sorted_ucp = proximity_sort((lat, lon), ucp_namedtuples, len(ucp_namedtuples))
        sorted_ucp = [ucp[1] for ucp in sorted_ucp]
    else: #newest
        sorted_ucp = sorted(ucp_lis, key=lambda ucp: ucp.checkpoint.date_created, reverse=True)

    return sorted_ucp

def get_my_checkpoints(user_obj):
    """
    get all UserCheckpoints belonging to a user
    """
    from db import UserCheckpoint
    ucp = UserCheckpoint.query.filter_by(user_id=user_obj.id)
    return ucp.all()

def get_recent_friend_user_checkpoints(user_obj, limit = None):
    """
    (faux notification) returns Checkpoints that were recently created by friends
    """
    if limit == None:
        limit = 8

    from db import UserCheckpoint, db, FacebookUser, User, FriendConnection, Checkpoint

    FriendUserCheckpoint, FriendFacebookUser, FriendUser = aliased(UserCheckpoint), aliased(FacebookUser), aliased(User)
    q = (db.session.query(UserCheckpoint).
         join(Checkpoint, Checkpoint.id == UserCheckpoint.checkpoint_id).
         join(FriendUser, FriendUser.id == UserCheckpoint.user_id).
         join(FriendFacebookUser, FriendFacebookUser.id == FriendUser.facebook_user_id).
         join(FriendConnection, FriendConnection.fb_user_to == FriendFacebookUser.id).
         join(FacebookUser, FacebookUser.id == FriendConnection.fb_user_from).
         join(User, User.facebook_user_id == FacebookUser.id).
         filter(User.id == user_obj.id).
         filter(Checkpoint.creator == FriendUser.id).
         filter(Checkpoint.demo == False)
         )

    return q.order_by(desc(Checkpoint.date_created)).limit(limit).all()

def search_user_checkpoints(user_obj, search_term):
    """
    Searches both my UserCheckpoints and my friends' with a literal search term.
    Searches a Checkpoint's name, description, and User's (Facebook) name (union)
    """
    from db import UserCheckpoint, db, FacebookUser, User, FriendConnection, Checkpoint
    search_term_with_wildcard = "%"+search_term+"%"

    #query that contain my own and my friends' usercheckpoints
    FriendUserCheckpoint, FriendFacebookUser, FriendUser = aliased(UserCheckpoint), aliased(FacebookUser), aliased(User)
    ucp = (db.session.query(UserCheckpoint).
           join(Checkpoint, Checkpoint.id == UserCheckpoint.checkpoint_id).
           join(FriendUser, FriendUser.id == UserCheckpoint.user_id).
           join(FriendFacebookUser, FriendFacebookUser.id == FriendUser.facebook_user_id).
           join(FriendConnection, FriendConnection.fb_user_to == FriendFacebookUser.id).
           join(FacebookUser, FacebookUser.id == FriendConnection.fb_user_from).
           join(User, User.facebook_user_id == FacebookUser.id).
           filter(or_(User.id == user_obj.id, FriendUser.id == user_obj.id)).
           filter(Checkpoint.demo == False)
           )

    name_search = ucp.filter(Checkpoint.name.ilike(search_term_with_wildcard))
    desc_search = ucp.filter(Checkpoint.description.ilike(search_term_with_wildcard))
    fb_name_search = ucp.filter(FriendFacebookUser.name.ilike(search_term_with_wildcard))

    result = name_search.union(desc_search).union(fb_name_search)

    return result.all()

def user_checkpoint_sanify(ucp_collection):
    """
    sanify a <<UserCheckpoint>> collection for jsonify-ication
    by separating it by Checkpoint Types
    """
    separated = {}
    for ucp in ucp_collection:
        if ucp.checkpoint.type.lower() in CHECKPOINT_TYPES:
            if ucp.checkpoint.type.lower() in separated:
                separated[ucp.checkpoint.type.lower()] += [ucp.serialize]
            else:
                separated[ucp.checkpoint.type.lower()] = [ucp.serialize]
    return separated

def _checkpoints_to_location_namedtuples(lis_of_cp):
    """
    converts a collection of user checkpoint objects to location namedtuples (usually) for proximity sorting
    """
    Obj = namedtuple("Obj", ("location", "user_checkpoint"))
    lis = []
    for cp in lis_of_cp:
        lis += [Obj((float(cp.checkpoint.latitude), float(cp.checkpoint.longitude)), cp)]
    return lis


import argparse
import numpy as np
from Phase2.Modules import VisDescParser as vd

def parse_args_process():
    argument_parse = argparse.ArgumentParser()
    argument_parse.add_argument('--dim_red_algo', help='The model you want to use(PCA/SVD/LDA)', type=str, required=True,
                                choices=['PCA', 'SVD', 'LDA'])
    argument_parse.add_argument('--k', help='The number semantics you want to find', type=int, required=True)
    argument_parse.add_argument('--loc_id', help='Enter the location id -', type=str, required=True,
                                )
    argument_parse.add_argument('--vis_model', help='Enter the Visual descriptor model -', type=str, required=True,
                                )
    args = argument_parse.parse_args()
    return args

def clean_ouput():
    f = open("task4output.txt", "w+")
    f.close()

def write_output(line):
    f = open("task4output.txt", "a+")
    f.write(line+"\n")
    f.close()

def print_output(result , locInKSem, KSemInFet):
    write_output("Input :"+str(args))
    write_output("Location in K semantics")
    for sem in locInKSem:
        write_output(np.array2string(sem))
    write_output("K semantics in features")
    for fet in KSemInFet:
        write_output(np.array2string(fet))
    write_output("Top K locations")
    for locId in result:
        write_output(str(locId))

# Main starts here
clean_ouput()
args = parse_args_process()
obj = vd.VisDescParser()
result , locInKSem, KSemInFet = obj.getTask4Items(args.dim_red_algo, args.k, args.loc_id, args.vis_model ,5)
print_output(result , locInKSem, KSemInFet)


from bokeh.plotting import figure, show
from bokeh.models import HoverTool
from cds_reader import source_para_vis_2, data_para_vis_2

quakers_aroma = figure(title="Quakers: Grãos imaturos prejudicam o aroma do café",
                       x_axis_label="Quakers",
                       y_axis_label="Nota do Aroma: 1 a 10",
                       width=513,
                       height=483
                       )

#Plotar os circulos
dados = source_para_vis_2(data_para_vis_2)
alpha = source_para_vis_2(data_para_vis_2).data['Alpha']

quakers_aroma.circle(source = dados, x='Quakers', y ='Aroma', fill_color='Blue',
                     line_color='Black',
                     fill_alpha = 'Alpha',
                     legend_label="Pontuação geral: Maior para menor, mais escuro para mais claro",
                     size=10
                     )

hover = HoverTool(tooltips=[("Quakers", "@Quakers"), ("Aroma", "@Aroma"), ("Pontuação Geral", "@Overall")])
quakers_aroma.add_tools(hover)

# Retirando logo do Bokeh
quakers_aroma.toolbar.logo = None

show(quakers_aroma)


import cv2
import numpy as np
from cv2 import aruco
import camara

DICCIONARIO = aruco.getPredefinedDictionary(aruco.DICT_5X5_50)
lena = cv2.imread("opencv/lena.tif")
h, w, _ = lena.shape

vlena = np.array([[0,0],[w,0],[w,h],[0,h]])

cap = cv2.VideoCapture(0)

if not cap.isOpened():
    print("No se puede abrir la cámara")
    exit()
while True:
    ret, frame = cap.read()

    if not ret:
        print("No he podido leer el frame")
        break

    gris = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
    bboxs, ids, rechazados = aruco.detectMarkers(gris, DICCIONARIO)

    if(ids is not None):
        aruco.drawDetectedMarkers(frame, bboxs, ids)
        for i in range(len(ids)):
            vertices = bboxs[i][0].astype(int)

            cv2.putText(frame, str(ids[i][0]), vertices[1], cv2.FONT_HERSHEY_SIMPLEX, 1, (0,0,0), 2)

            matriz, _ = cv2.findHomography(vlena, vertices)
            warplena = cv2.warpPerspective(lena, matriz, (frame.shape[1], frame.shape[0]))

            #cv2.fillConvexPoly(frame, vertices, (0,0,0))
            #frame = frame + warplena

    cv2.imshow('WEBCAM', frame)

    if cv2.waitKey(1) == ord(' '):
        break

cap.release()
cv2.destroyWindow('WEBCAM')


from django.http import HttpResponse
from django.http import HttpResponseRedirect
from django.template import loader
from django.db.models import Sum
from .models import Account, AccountList
from transfers.models import Transfer
from transactions.models import Transaction
from django.shortcuts import render
from .forms import GetDateForm
from datetime import date
import numpy as np

def index(request):
    account_balance_array=[]
    template = loader.get_template('accounts/index.html')
    all_accounts = Account.objects.all()

    print (all_accounts)
    get_accounts = Account.objects.all()
    #print (get_accounts)
    #for account_list in get_account_list:
    #    print (account_list.account_name)
    #    print (account_list.account_type)
    #    print (account_list.initial_balance)
    #    print (account_list.date)

    #for account in get_accounts:
    #    print ('-------------------')
    #    print (account.balance_desc)
    #    print (account.account_name)
    #    print (account.balance)
    #    print (account.balance_date)
    #    print ('------------------')

    total_cash = 0
    for account_list in get_account_list:

        #get account name, account type and balance date (today) to load into array
        account_name = account_list.account_name
        account_id = account_list.id

        print ('-----------------')
        print (account_name)
        print ('------------------')
        balance_date = Account.objects.filter(account_name=account).last()
        print (balance_date)

        #get all transactions for current account
        #account_transactions = Transaction.objects.filter(account_name=account.id, trans_date__lte=balance_date).aggregate(sum=Sum('amount'))['sum'] or 0.00
        #get the current account balance (at the moment this is just the initial balance - need to get all transactions from current day forward??)
        #account_balance = Account.objects.filter(account=account_id).last()
        #print (account_balance)
        #get any transfers (likely need to only get stuff from current balance date..otherwise this may mess up if we write a new balance)
       # account_incoming_transfers = Transfer.objects.filter(incoming_account__pk=account.id, transfer_date__lte=balance_date).aggregate(sum=Sum('transfer_amount'))['sum'] or 0.00
       # account_outgoing_transfers = Transfer.objects.filter(outgoing_account__pk=account.id, transfer_date__lte=balance_date).aggregate(sum=Sum('transfer_amount'))['sum'] or 0.00
        #calculate the new balance
        #new_account_balance = account_balance + account_transactions + account_incoming_transfers - account_outgoing_transfers

        #calculate cash position by summing the new balance of each account
        #total_cash = new_account_balance + total_cash
        #build the array for display
        #account_balance_array.append((account_name, account_balance, balance_date))
        print ('yo')
    #card_total_object = Account.objects.filter(account_type__contains="credit card").values('balance')
    #card_total = card_total_object[0]['balance']
    #other_account_total = Account.objects.exclude(account_type__contains="credit card").aggregate(sum=Sum('balance'))['sum'] or 0.00
    context = {
        'total_cash': total_cash,
        'account_balance_array' : account_balance_array    }
    return HttpResponse(template.render(context, request))

def detail(request, account_id):
    template = loader.get_template ('accounts/details.html')
    selected_account = Account.objects.filter(id=account_id).values('account_list')
    acct_name = selected_account[0]['account_list']
    trans_for_account_id = Transaction.objects.filter(account_list=1)
    transaction_total = Transaction.objects.filter(account_list=1).aggregate(sum=Sum('amount'))['sum'] or 0.00
    print (acct_name)
    context ={
        'acct_name': acct_name,
        'trans_for_account_id': trans_for_account_id,
        'transaction_total': transaction_total
    }
    return HttpResponse((template.render(context,request)))


import easyocr
import cv2

reader = easyocr.Reader(['en', 'ko'])
def fn_get_text(path):
    img = cv2.imread(path, cv2.IMREAD_GRAYSCALE)
    height, width = img.shape
    result = reader.readtext(img)
    print(result)
    result_dataset = []
    for bbox, text, prob in result:
        if prob > 0.1:
            result_dataset.append({'text':text, 'bbox':bbox, 'prob':prob})
    for i, v in enumerate(result_dataset):
        print(v['text'], v['bbox'], v['prob'])
    draw_bbox(path, result_dataset)
def draw_bbox(path, data):
    img = cv2.imread(path)
    for d in data:
        bbox = d['bbox']
        start_point = (int(bbox[0][0]), int(bbox[0][1]))  # Convert to integers
        end_point = (int(bbox[2][0]), int(bbox[2][1]))  # Convert to integers
        #사각형 그리기
        color = (0, 255, 0)
        thinkness = 2
        img = cv2.rectangle(img, start_point, end_point, color, thinkness)
    cv2.imwrite('output.jpg', img)

if __name__ == '__main__':
    fn_get_text('text.jpg')


import pygame,sys
from pygame.locals import*
# Variables
BLUE = (130, 202, 250)
WHITE = (255, 255, 255)
BLACK = (0, 0, 0)
LIGHTPURPLE = (240,206,253)
PINK = (255,147,214)
GREY = (222,225,223)
DARKBLUE = (24,32,83)

def searchGame(gamewindow):

    # Set the name of Display
    pygame.display.set_caption("Search and Find Game")
    # Set Background - Using a picture
    gamewindow.fill(DARKBLUE)
    menuBackground = pygame.transform.scale(pygame.image.load(
        "images/hidden background.png"), (1200, 650))
    gamewindow.blit(menuBackground, (0, 0))

    # Title
    fontTitle = pygame.font.Font('freesansbold.ttf', 25)
    title = fontTitle.render('Search & Find Game', True, WHITE)
    Title = title.get_rect()
    Title.center = (660, 40)
    gamewindow.blit(title, Title)
    pygame.display.update()

    # Create this loop so that user can click on the 'x' button to exit
    while True:
        for event in pygame.event.get():
            if event.type == QUIT:
                pygame.quit()
                sys.exit()

def searchFindInstructions(gamewindow):
    pygame.display.set_caption("Search & Find Game Instructions")
    gamewindow.fill(GREY)
    instructionFont = pygame.font.Font('freesansbold.ttf', 40)
    instruction = instructionFont.render('Search & Find Game Instructions', True, BLACK)
    Instruction = instruction.get_rect()
    Instruction.center = (600, 80)
    gamewindow.blit(instruction, Instruction)
    pygame.display.update()

    # Create this loop so that user can click on the 'x' button to exit
    while True:
        for event in pygame.event.get():
            if event.type == QUIT:
                pygame.quit()
                sys.exit()

        pygame.event.get(pygame.MOUSEBUTTONDOWN)
        mouse = pygame.mouse.get_pos()
        click = pygame.mouse.get_pressed()

        if 800 + 150 > mouse[0] > 800 and 600 + 50 > mouse[1] > 600:
            pygame.draw.rect(gamewindow, LIGHTPURPLE, (800, 600, 150, 50))
            if click[0] == 1:
                searchGame(gamewindow)
        else:
            pygame.draw.rect(gamewindow, PINK, (800, 600, 150, 50))

        mazeFont = pygame.font.Font('freesansbold.ttf', 20)
        mazeWord = mazeFont.render('START GAME', True, BLACK)
        MazeWord = mazeWord.get_rect()
        MazeWord.center = ((800 + (150 / 2)), (600 + (50 / 2)))
        gamewindow.blit(mazeWord, MazeWord)

        pygame.display.update()


#!/usr/bin/env python3
import requests
import argparse
from typing import cast, Union, Any

def readSecret(secretPath):
    with open(secretPath) as f:
            mySecret = f.readline().strip()
            return mySecret

def main():
    parser = argparse.ArgumentParser(description="Argument Parser")
    parser.add_argument("domain", type=str, help="Domain to Delete")
    args = parser.parse_args()

    api_url = "https://{}.console.ves.volterra.io/api/config/dns/namespaces/system/dns_zones/{}".format(readSecret(".secrets/.consoleDomain"), args.domain)

    api_headers = {
        "Authorization" :   readSecret(".secrets/.apiToken"),
        "Accept"        :   "application/json"
    }

    body = "{{\n\t\"fail_if_referred\": true,\n\t\"name\": {}\n\t\"namespace\": \"system\"\n}}".format(args.domain)
    print(body)
    deleteZone = requests.delete(api_url, verify=False, headers=api_headers).content.decode("utf8")
    print(deleteZone)

############################ GLOBALS ###############################

if __name__ == "__main__":
    main()


from django.shortcuts import render
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import joblib
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import BaggingClassifier
from sklearn.preprocessing import LabelEncoder, MinMaxScaler
from sklearn.svm import SVC
# # Create your views here
#
def index(request):
    return render(request, 'home.html')

def text_uppercase(text):
    return text.upper()

def baglogr_prediction(col1,col2,col3,col4,col5,col6,col7,col8,col9,col10,col11):
    filename = 'classification/models/baggingClass_jb.aiml'
    loaded_model_v2 = joblib.load(filename)
    predictrun = [[int(col1),int(col2),int(col3),int(col4),int(col5),int(col6),int(col7),int(col8),int(col9),float(col10),int(col11)]]
    predicted = loaded_model_v2.predict(predictrun)
    return predicted

def classification(request):

    form_posted = False
    dictsdia = {0: "Subject has a normal heart.", 1: "Subject has a high probability of having a disease."}
    map_sex={'F': 0, 'M': 1}
    map_cpt={"ASY": 0, "ATA": 1, "NAP": 2, "TA": 3}
    map_recg={"LVH": 0, "NORMAL": 1, "ST": 2}
    map_ea={'N': 0, 'Y': 1}
    map_sts={"DOWN": 0, "FLAT": 1, "UP": 2}

    if request.method == 'POST':

        form_posted = True

        col1 = request.POST['age']
        col2 = text_uppercase(request.POST['sex'])
        col3 = text_uppercase(request.POST['cpt'])
        col4 = request.POST['rbp']
        col5 = request.POST['chol']
        col6 = request.POST['fbs']
        col7 = text_uppercase(request.POST['recg'])
        col8 = request.POST['mhr']
        col9 = text_uppercase(request.POST['ea'])
        col10 = request.POST['op']
        col11 = text_uppercase(request.POST['sts'])

        col2 = map_sex[col2[0]]
        col3 = map_cpt[col3]
        col7 = map_recg[col7]

        col9 = map_ea[col9[0]]
        col11 = map_sts[col11]

    else:
        col1 = None
        col2 = None
        col3 = None
        col4 = None
        col5 = None
        col6 = None
        col7 = None
        col8 = None
        col9 = None
        col10 = None
        col11 = None

    if form_posted:
        predicted = baglogr_prediction(col1,col2,col3,col4,col5,col6,col7,col8,col9,col10,col11)
        predicted = dictsdia[predicted[0]]

    context = {
        "is_posted": form_posted,
        'prediction' : 'None' if form_posted is False else predicted,
        "col1": col1,
        "col2": col2,
        "col3": col3,
        "col4": col4,
        "col5": col5,
        "col6": col6,
        "col7": col7,
        "col8": col8,
        "col9": col9,
        "col10": col10,
        "col11": col11
    }

    return render(request, 'dave.html', context)

def bike_input(request):

    return render(request, 'bikeinput.html')

def bike_output(request):
    # Get the text

    seasontext = request.GET.get('season')
    mnthtext = request.GET.get('mnth')
    holitext = request.GET.get('holiday')
    weekdtext = request.GET.get('weekday')
    weasittext = request.GET.get('weathersit')
    temptext = request.GET.get('temp')
    atemptext = request.GET.get('atemp')
    humtext =  request.GET.get('hum')
    wndspdtext  = request.GET.get('windspeed')

    sea = int(seasontext)
    mnth = int(mnthtext)
    holiday = int(holitext)
    weekday = int(weekdtext)
    weathersit = int(weasittext)
    temp = float(temptext)
    atemp = float(atemptext)
    hum = float(humtext)
    windspeed = float(wndspdtext)
    filename = 'classification/models/bike_model.aiml'
    loaded_model_v2 = joblib.load(filename)
    sampletest = [[sea, mnth, holiday, weekday, weathersit, temp, atemp, hum, windspeed]]
    predicted = loaded_model_v2.predict(sampletest)

    params = {'Category': round(predicted[0])}
    return render(request, 'bike.html', params)


import os
import os.path
import xml.etree.ElementTree as ET
import glob

class_names = ['person with helmet','person','helmet','person without helmet']
xmlpath='C:/Users/18251/Desktop/TrainImage2/train_annot_folder'
txtpath='C:/Users/18251/Desktop/TrainImage2/txt_annot'

def xml_to_txt(xmlpath,txtpath):

    os.chdir(xmlpath)
    annotations = os.listdir('.')
    annotations = glob.glob(str(annotations)+'*.xml')

    file_save = 'train' + '.txt'
    file_txt = os.path.join(txtpath, file_save)
    f_w = open(file_txt, 'w')

    for i,file in enumerate(annotations):

        in_file = open(file)
        tree=ET.parse(in_file)
        root = tree.getroot()

        filename = root.find('filename').text

        for obj in root.iter('object'):
                current = list()
                name = obj.find('name').text

                class_num = class_names.index(name)

                xmlbox = obj.find('bndbox')

                x1 = xmlbox.find('xmin').text
                x2 = xmlbox.find('xmax').text
                y1 = xmlbox.find('ymin').text
                y2 = xmlbox.find('ymax').text

                f_w.write('VOCdevkit/VOC2007/JPEGImages/'+filename+'.jpg '+x1+','+y1+','+x2+','+y2+','+str(class_num)+'\n')

xml_to_txt(xmlpath,txtpath)

# C:/Users/18251/Desktop/TrainImage2/train_annot_folder
# C:/Users/18251/Desktop/TrainImage2/txt_annot