"""
Project: Learn how to use PyTorch Lighnting on a simple toy problem
Author: Dr. Arrykrishna Mootoovaloo
Reference: PyTorch Lighnting website
Date: December 2022
"""
from ml_collections import ConfigDict
from pl_bolts.transforms.dataset_normalizations import cifar10_normalization
from pl_bolts.datamodules import CIFAR10DataModule
import torchvision

def data_module(cfg: ConfigDict):
    """The data module for the CIFAR 10 data

    Args:
        config (ConfigDict): the main configuration file.
    """
    train_transforms = torchvision.transforms.Compose(
        [
            torchvision.transforms.RandomCrop(
                cfg.trans.crop_size, cfg.trans.padding),
            torchvision.transforms.RandomHorizontalFlip(),
            torchvision.transforms.ToTensor(),
            cifar10_normalization()
        ]
    )

    test_transforms = torchvision.transforms.Compose(
        [
            torchvision.transforms.ToTensor(),
            cifar10_normalization()
        ]
    )

    module = CIFAR10DataModule(
        data_dir=cfg.paths.data,
        batch_size=cfg.training.batch_size,
        num_workers=cfg.training.num_workers,
        train_transforms=train_transforms,
        test_transforms=test_transforms,
        val_transforms=test_transforms
    )

    return module


import os
import shutil
import json
import wx
from datetime import datetime as time
import numpy as np
import sys
import re
def simp_pack_path(path):
  return re.sub(r'\s+','_',path)
def simp_nmsp_path(path):
  return 'ac.g.' + re.sub(r'\.ac\.g$', '', re.sub(r'(\s+|-)','_',path))
def verify_pack_root(path):
  tpath = os.path.abspath(path)
  if not os.path.exists(tpath):
    os.makedirs(tpath)
  if not os.path.isdir(os.path.join(tpath, 'data','ac','tags','functions','api')):
    os.makedirs(os.path.join(tpath, 'data','ac','tags','functions','api'))
  if not os.path.isdir(os.path.join(tpath, 'data','minecraft','tags','functions')):
    os.makedirs(os.path.join(tpath, 'data','minecraft','tags','functions'))
  space = simp_nmsp_path(os.path.basename(os.path.normpath(path)))
  nmsp = os.path.join(tpath, 'data', space)
  if not os.path.isdir(os.path.join(nmsp,'functions')):
    os.makedirs(os.path.join(nmsp,'functions'))
  if not os.path.isdir(os.path.join(nmsp,'tags','items')):
    os.makedirs(os.path.join(nmsp,'tags','items'))
  if not os.path.isdir(os.path.join(nmsp,'predicates')):
    os.makedirs(os.path.join(nmsp,'predicates'))
  if not os.path.isfile(os.path.join(tpath, 'pack.mcmeta')):
    with open(os.path.join(tpath, 'pack.mcmeta'),'w+') as f:
      f.write('{"pack":{"pack_format":6,"description":"generated recipepack for radiish\'s autocrafting datapack"}}')
  if not os.path.isfile(os.path.join(tpath, 'data', 'ac', 'tags', 'functions', 'api','craft.json')):
    with open(os.path.join(tpath, 'data', 'ac', 'tags', 'functions', 'api','craft.json'),'w+') as f:
      f.write(f'{{"replace":false,"values":["{space}:recipes"]}}')
  if not os.path.isfile(os.path.join(tpath, 'data', 'minecraft', 'tags', 'functions','load.json')):
    with open(os.path.join(tpath, 'data', 'minecraft', 'tags', 'functions','load.json'),'w+') as f:
      f.write(f'{{"replace":false,"values":["{space}:load"]}}')
  return '\\\\?\\' + tpath
def pack_nmsp(path):
  return os.path.join(os.path.abspath(path), 'data', simp_nmsp_path(os.path.basename(os.path.normpath(path))))
def mcpath(path):
  return path.replace('\\','/')
supress = False
slots = ['3','4','5','12','13','14','21','22','23']
not_table = ['smoking','stonecutting','smelting','blasting','campfire_cooking']

g_log = ""
g_gui = None
def resource_path(relative_path):
  try:
    base_path = sys._MEIPASS
  except Exception:
    base_path = os.path.abspath(".")

  return os.path.join(base_path, relative_path)
def log_print(msg):
  global g_gui
  g_gui.log.SetValue(g_gui.log.GetValue() + '\n' + msg)
  g_gui.Layout()

def clear_contents(path):
  shutil.rmtree(path)
  os.mkdir(path)

def nested_paths(dest, ignore = []):
  result = []
  listdir = os.listdir(dest)
  for f in listdir:
    if f in ignore:
      continue
    if os.path.isfile(os.path.join(dest, f)):
      result.append(os.path.join(dest, f))
    else:
      nest = nested_paths(os.path.join(dest, f))
      nest.extend(result)
      result = nest
  return result

class kitem:
  def __init__(self,value):
    self.value = value.split(':')[1]
    self.namespace = value.split(':')[0]
    self.both = value
    self.item = True
class ktag:
  def __init__(self,value):
    self.value = value.split(':')[1]
    self.namespace = value.split(':')[0]
    self.both = value
    self.item = False
def create_predicate(tag,root,tagname = None):
  if tagname == None:
    tagname = f'{tag.namespace}:{tag.value}'
  pathi = os.path.join(root,'predicates','tags',tag.namespace)
  if not os.path.isdir(pathi):
    os.makedirs(pathi)
  with open(os.path.join(pathi, tag.value) + '.json','w+') as f:
    f.write(f'{{"condition":"minecraft:entity_properties","entity":"this","predicate":{{"equipment":{{"feet":{{"tag":"{tagname}"}}}}}}}}')
def create_tag(options,root,namespace):
  item = kitem(options[0])
  pathi2 = os.path.join(root,'tags','items',item.namespace)
  if not os.path.isdir(pathi2):
    os.makedirs(pathi2)
  strops = ''
  for op in options:
    strops += f'"{op}"'
    if op != options[len(options)-1]:
      strops += ','
  with open(os.path.join(pathi2,item.value) + '.json','w+') as f:
    f.write(f'{{"replace":false,"values":[{strops}]}}')
  create_predicate(item,root,f'{namespace}:{item.namespace}/{item.value}')
def write_functions(keys_in,out_id,out_count,root,namespace):
  path = os.path.join(root,'functions','recipes')
  old_path = path
  if os.path.isfile(old_path + '.mcfunction') == False:
    open(old_path + '.mcfunction','w+').close()
  if os.path.isdir(path) == False:
    os.mkdir(path)
  for i in range(8):
    if i > 0:
      old_path = path
    path += '/' + keys_in[i].value
    func_path = 'recipes/' + '/'.join(path.split('/')[1:])
    if os.path.isdir('\\'.join(path.split('/'))) == False:
      os.mkdir('\\'.join(path.split('/')))
      open('\\'.join(path.split('/')) + '.mcfunction','w+').close()
      with open('\\'.join(old_path.split('/')) + '.mcfunction','a+') as file:
        file.write(f'#{keys_in[i].both}\n')
        if keys_in[i].item:
          if keys_in[i].both == 'builtin:null':
            file.write(f'execute unless data block ~ ~1 ~ {{Items:[{{Slot:{slots[i]}b}}]}} run function {namespace}:{func_path}\n')
          else:
            file.write(f'execute if data block ~ ~1 ~ {{Items:[{{Slot:{slots[i]}b,id:"{keys_in[i].both}"}}]}} run function {namespace}:{func_path}\n')
        else:
          file.write('data modify entity @s ArmorItems[0] set value {id:"minecraft:jigsaw",Count:1b}\n')
          file.write(f'data modify entity @s ArmorItems[0].id set from block ~ ~1 ~ Items[{{Slot:{slots[i]}b}}].id\n')
          file.write(f'execute if predicate {namespace}:tags/{keys_in[i].namespace}/{keys_in[i].value} run function {namespace}:{func_path}\n')
  with open('\\'.join(path.split('/')) + '.mcfunction','a+') as file:
    file.write(f'#{keys_in[8].both}\n')
    if keys_in[8].item:
      if keys_in[8].both == 'builtin:null':
        file.write(f'execute unless data block ~ ~1 ~ {{Items:[{{Slot:{slots[8]}b}}]}} run function {namespace}:{func_path}/{keys_in[8].value}\n')
      else:
        file.write(f'execute if data block ~ ~1 ~ {{Items:[{{Slot:{slots[8]}b,id:"{keys_in[8].both}"}}]}} run function {namespace}:{func_path}/{keys_in[8].value}\n')
    else:
      file.write('data modify entity @s ArmorItems[0] set value {id:"minecraft:jigsaw",Count:1b}\n')
      file.write(f'data modify entity @s ArmorItems[0].id set from block ~ ~1 ~ Items[{{Slot:{slots[8]}b}}].id\n')
      file.write(f'execute if predicate {namespace}:tags/{keys_in[8].namespace}/{keys_in[8].value} run function {namespace}:{func_path}/{keys_in[8].value}\n')
  with open('\\'.join(path.split('/')) + '\\' + keys_in[8].value+'.mcfunction','w+') as file:
    file.write(f'data modify block ~ ~1 ~ Items[{{Slot:0b}}] set value {{Slot:0b,id:"{out_id}",Count:{out_count}b}}\n')
    file.write('tag @s add ac.offering\n')
    file.write('data modify entity @s ArmorItems[1].tag.ac_last set from block ~ ~1 ~ Items\n')
    for i in range(9):
      if keys_in[i].both == 'builtin:null':
        file.write(f'data modify entity @s ArmorItems[1].tag.ac_last append value {{Slot:{slots[i]}b,id:"minecraft:structure_void",Count:1b,tag:{{ac_gui:1b,CustomModelData:2b,display:{{Name:\'""\'}}}}}}\n')

def create_recipe(path,root,namespace):
  with open(path) as file:
    data = json.load(file)
    ctype = data['type'].split(':')[-1]
    if ctype == 'crafting_shaped':
      x = len(data['pattern'][0])
      y = len(data['pattern'])
      pattern = np.zeros((y,x))
      keys = [' ']
      key_values = [kitem('builtin:null')]
      for xi in range(x):
        for yi in range(y):
          key = data['pattern'][yi][xi]
          if key in keys:
            pattern[yi][xi] = keys.index(key)
          else:
            pattern[yi][xi] = len(keys)
            keys.append(key)
            if 'item' in data['key'][key]:
              key_values.append(kitem(data['key'][key]['item']))
            elif 'tag' in data['key'][key]:
              key_values.append(ktag(data['key'][key]['tag']))
            elif isinstance(data['key'][key],list):
              ops = []
              for i in data['key'][key]:
                if 'item' in i:
                  ops.append(i['item'])
                elif 'tag' in i:
                  ops.append('#' + i['tag'])
              create_tag(ops,root,namespace)
              key_values.append(ktag(ops[0]))
            else:
              key_values.append(kitem('builtin:null'))
      pattern = np.pad(pattern, ((0,3-y),(0,3-x)),mode='constant',constant_values=0)
      keys_in = []
      for yi in range(3):
        for xi in range(3):
          app = key_values[int(pattern[yi][xi])]
          keys_in.append(app)
      result_id = data['result']['item']
      if "count" in data['result']:
        result_count = data['result']['count']
      else:
        result_count = 1
      write_functions(keys_in,result_id,result_count,root,namespace)
      return True
    elif ctype == 'crafting_shapeless':
      result_id = data['result']['item']
      if 'count' in data['result']:
        result_count = data['result']['count']
      else:
        result_count = 1
      keys_in = []
      ingreds = data['ingredients']
      i = 0
      for i in range(len(ingreds)):
        if 'item' in ingreds[i]:
          keys_in.append(kitem(ingreds[i]['item']))
        elif 'tag' in ingreds[i]:
          keys_in.append(ktag(ingreds[i]['tag']))
        elif isinstance(ingreds[i],list):
          ops = []
          for i2 in ingreds[i]:
            if 'item' in i2:
              ops.append(i2['item'])
          create_tag(ops,root,namespace)
          op = f'{namespace}:' + ops[0][ops[0].index(':') + 1:]
          keys_in.append(ktag(op))
        else:
          keys_in.append(kitem('builtin:null'))
      for i2 in range(9-i):
        keys_in.append(kitem('builtin:null'))
      write_functions(keys_in,result_id,result_count,root,namespace)
      return True
    else:
      if not supress:
        if ctype == 'crafting_special':
          log_print(f'invalid recipe. unable to create recipe ({os.path.basename(path)}) because its features are not emulatable by the current system ({ctype})')
        elif ctype in not_table:
          log_print(f'invalid recipe. unable to create recipe ({os.path.basename(path)}) because it is implemented by other means ({ctype})')
        else:
          log_print(f'invalid recipe. unable to create recipe ({os.path.basename(path)}) beacuse of an incompatible crafting_type ({ctype})')
        return False
def create_from_dir(path,root,namespace):
  listdir = nested_paths(path)
  for f in listdir:
    if os.path.splitext(f)[1] == '.json':
      create_recipe(f,root,namespace)

class gui(wx.Frame):

  def __init__(self, *args, **kwargs):
    super(gui, self).__init__(*args, **kwargs)

    self.SetSize((560, 640))
    self.SetTitle('recipe generator')
    self.Centre()

    menubar = wx.MenuBar()
    fileMenu = wx.Menu()

    qmi = wx.MenuItem(fileMenu, wx.ID_CLOSE, '&quit\tCtrl+Q')
    fileMenu.Append(qmi)
    self.Bind(wx.EVT_MENU, self.quit, id=wx.ID_CLOSE)

    imi = wx.MenuItem(fileMenu, -1, '&import (source)\tCtrl+I')
    fileMenu.Append(imi)
    self.Bind(wx.EVT_MENU, self.onimport, imi)
    omi = wx.MenuItem(fileMenu, -1, '&open (destination)\tCtrl+O')
    fileMenu.Append(omi)
    self.Bind(wx.EVT_MENU, self.open, omi)

    rmi = wx.MenuItem(fileMenu, 3, '&reset all recipes')
    fileMenu.Append(rmi)
    self.Bind(wx.EVT_MENU, self.reset, id=3)

    menubar.Append(fileMenu, '&File')
    self.SetMenuBar(menubar)

    panel = wx.Panel(self)

    icon = wx.Icon()
    icon.CopyFromBitmap(wx.Bitmap("icon.ico", wx.BITMAP_TYPE_ICO))
    self.SetIcon(icon)

    wx.StaticText(panel, -1, label='source path:', pos=(20, 0), size=(500, 20))
    self.input = wx.TextCtrl(panel, -1, value='[path to your recipes datapack]', pos=(20, 20), size=(500, 40), style = wx.TE_MULTILINE)

    wx.StaticText(panel, -1, label='destination path:', pos=(20, 60), size=(500, 20))
    self.output = wx.TextCtrl(panel, -1, value='[path to your newly created datapack folder]', pos=(20, 80), size=(500, 40), style = wx.TE_MULTILINE)

    panel.Bind(wx.EVT_BUTTON, self.generate, wx.Button(panel, -1, label='generate reicpes', pos=(200,140), size=(160,40)))

    wx.StaticText(panel, -1, label='log:', pos=(20, 180), size=(500, 20))
    self.log = wx.TextCtrl(panel, -1, value=f'[generator opened at {time.now().strftime("%Y-%m-%d %H:%M:%S")}]', pos=(20, 200), size=(500, 260), style = wx.TE_MULTILINE | wx.TE_READONLY)

    self.Bind(wx.EVT_CLOSE, self.quit, self)
  def quit(self,e):
    log_print(f'[generator closed at {time.now().strftime("%Y-%m-%d %H:%M:%S")}]')
    with open('last.log','w+') as log:
      log.write(self.log.GetValue())
    self.Destroy()
  def onimport(self, e):
    with wx.DirDialog (None, "choose source directory", "", wx.DD_DEFAULT_STYLE | wx.DD_DIR_MUST_EXIST) as dirDialog:
      if dirDialog.ShowModal() == wx.ID_CANCEL:
        return
      path = dirDialog.GetPath()
      self.input.SetValue(path)
      if self.output.GetValue() == '[path to your newly created datapack folder]':
        self.output.SetValue(os.path.join(os.path.dirname(path), os.path.basename(os.path.normpath(path))) + '.ac.g')
  def open(self, e):
    with wx.DirDialog (None, "choose destination directory", "", wx.DD_DEFAULT_STYLE | wx.DD_DIR_MUST_EXIST) as dirDialog:
      if dirDialog.ShowModal() == wx.ID_CANCEL:
        return
      self.output.SetValue(dirDialog.GetPath())
  def reset(self, e):
    with wx.MessageDialog(self,'are you sure you want to delete all recipes?','reset all recipes?',wx.YES_NO) as dlg:
      if dlg.ShowModal() == wx.ID_YES:
        root = pack_nmsp(self.output.GetValue())
        if os.path.exists(root):
          log_print('i sure hope you know what you\'re doing!')
          length = len(nested_paths(os.path.join(root,'ac','functions','autocraft','recipes')))
          clear_contents(os.path.join(root,'ac','functions','autocraft','recipes'))
          clear_contents(os.path.join(root,'ac','predicates','tags'))
          clear_contents(os.path.join(root,'ac','tags','items'))
          open(os.path.join(root,'ac','functions','autocraft','recipes.mcfunction'),'w+').close()
          log_print(f'deleted {length} files')
        else:
          log_print(f'invalid path. the path specified does not exist. ({root[4:]})')
  def generate(self,e):
    path = self.input.GetValue()
    dest = verify_pack_root(self.output.GetValue())
    namespace = simp_nmsp_path(os.path.basename(os.path.normpath(dest)))
    dest = os.path.join(dest,'data',namespace)
    if not os.path.isdir(os.path.join(dest,'recipes')):
      log_print('no \'recipes\' folder found. creating new folder')

    if os.path.isdir(os.path.join(path,'data')):
      path = os.path.normpath(os.path.join(path,'data'))
      tags = [(os.path.join(path, f, 'tags', 'items'),f) for f in os.listdir(path) if os.path.isdir(os.path.join(path, f, 'tags', 'items'))]
      all_tags = []
      namespaces = []
      for f in tags:
        nest = nested_paths(f[0])
        nest.extend(all_tags)
        for i in range(len(nest)):
          namespaces.append(f[1])
        all_tags = nest
      for i in range(len(all_tags)):
        tagname = os.path.splitext(os.path.join(*all_tags[i][len(path):].split(os.sep)[4:]))[0]
        create_predicate(ktag(f'{namespaces[i]}:{tagname}'),dest)

      paths = [os.path.join(path, f, 'recipes') for f in os.listdir(path) if os.path.isdir(os.path.join(path, f, 'recipes'))]
      for f in paths:
        create_from_dir(f,dest,namespace)
      with open(os.path.join(dest,'functions','load.mcfunction'),'w+') as f:
        f.write(f'tellraw @a[tag=debug] ["",{{"color":"yellow","translate":"debug.prefix","bold":true}}," ","loaded {len(paths)} recipes from {namespace[5:]}"]')
    else:
      log_print('invalid path. make sure you are using the datapack and not the recipes or data folder')
app = wx.App()
g_gui = gui(None)
g_gui.Show()
app.MainLoop()


# 최솟값 찾기
import sys
from collections import deque
input = sys.stdin.readline

N, L = map(int, input().split())
A = list(map(int, input().split()))
D = [0 for _ in range(N)]
Q = deque()
for i in range(N):
    # step1: A[i]보다 더 큰 값이 덱에 있다면, 최솟값의 가능성이 없기 때문에 pop
    while Q and Q[-1][1] > A[i]:
            Q.pop()
    # step2: 덱에 A[i] 추가
    Q.append([i, A[i]])
    # step3: window size를 벗어나면 popleft
    if Q[0][0] <= i-L:
        Q.popleft()
    print(Q[0][1], end=' ')


import argparse
import os
import shutil
import socket
import stat
import sys
from socket import *
from pathlib import Path

from progress.bar import IncrementalBar
import donkeycar as dk
from donkeycar.management.joystick_creator import CreateJoystick
from donkeycar.management.tub import TubManager
from donkeycar.utils import *

import time
from datetime import datetime
import requests
import tempfile
import tarfile
from requests_toolbelt.multipart.encoder import MultipartEncoder
import netifaces
from donkeycar import __version__

PACKAGE_PATH = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
TEMPLATES_PATH = os.path.join(PACKAGE_PATH, 'templates')

HELP_CONFIG = 'location of config file to use. default: ./config.py'

def make_dir(path):
    real_path = os.path.expanduser(path)
    print('making dir ', real_path)
    if not os.path.exists(real_path):
        os.makedirs(real_path)
    return real_path

def load_config(config_path):

    '''
    load a config from the given path
    '''
    conf = os.path.expanduser(config_path)

    if not os.path.exists(conf):
        print("No config file at location: %s. Add --config to specify\
                location or run from dir containing config.py." % conf)
        return None

    try:
        cfg = dk.load_config(conf)
    except:
        print("Exception while loading config from", conf)
        return None

    return cfg

class BaseCommand(object):
    pass

class CreateCar(BaseCommand):

    def parse_args(self, args):
        parser = argparse.ArgumentParser(prog='createcar', usage='%(prog)s [options]')
        parser.add_argument('--path', default=None, help='path where to create car folder')
        parser.add_argument('--template', default=None, help='name of car template to use')
        parser.add_argument('--overwrite', action='store_true', help='should replace existing files')
        parsed_args = parser.parse_args(args)
        return parsed_args

    def run(self, args):
        args = self.parse_args(args)
        self.create_car(path=args.path, template=args.template, overwrite=args.overwrite)

    def create_car(self, path, template='complete', overwrite=False):
        """
        This script sets up the folder structure for donkey to work.
        It must run without donkey installed so that people installing with
        docker can build the folder structure for docker to mount to.
        """

        # these are neeeded incase None is passed as path
        path = path or '~/mycar'
        template = template or 'complete'
        print("Creating car folder: {}".format(path))
        path = make_dir(path)

        print("Creating data & model folders.")
        folders = ['models', 'data', 'logs']
        folder_paths = [os.path.join(path, f) for f in folders]
        for fp in folder_paths:
            make_dir(fp)

        # add car application and config files if they don't exist
        app_template_path = os.path.join(TEMPLATES_PATH, template+'.py')
        config_template_path = os.path.join(TEMPLATES_PATH, 'cfg_' + template + '.py')
        myconfig_template_path = os.path.join(TEMPLATES_PATH, 'myconfig.py')
        train_template_path = os.path.join(TEMPLATES_PATH, 'train.py')
        calibrate_template_path = os.path.join(TEMPLATES_PATH, 'calibrate.py')
        car_app_path = os.path.join(path, 'manage.py')
        car_config_path = os.path.join(path, 'config.py')
        mycar_config_path = os.path.join(path, 'myconfig.py')
        train_app_path = os.path.join(path, 'train.py')
        calibrate_app_path = os.path.join(path, 'calibrate.py')

        if os.path.exists(car_app_path) and not overwrite:
            print('Car app already exists. Delete it and rerun createcar to replace.')
        else:
            print("Copying car application template: {}".format(template))
            shutil.copyfile(app_template_path, car_app_path)
            os.chmod(car_app_path, stat.S_IRWXU)

        if os.path.exists(car_config_path) and not overwrite:
            print('Car config already exists. Delete it and rerun createcar to replace.')
        else:
            print("Copying car config defaults. Adjust these before starting your car.")
            shutil.copyfile(config_template_path, car_config_path)

        if os.path.exists(train_app_path) and not overwrite:
            print('Train already exists. Delete it and rerun createcar to replace.')
        else:
            print("Copying train script. Adjust these before starting your car.")
            shutil.copyfile(train_template_path, train_app_path)
            os.chmod(train_app_path, stat.S_IRWXU)

        if os.path.exists(calibrate_app_path) and not overwrite:
            print('Calibrate already exists. Delete it and rerun createcar to replace.')
        else:
            print("Copying calibrate script. Adjust these before starting your car.")
            shutil.copyfile(calibrate_template_path, calibrate_app_path)
            os.chmod(calibrate_app_path, stat.S_IRWXU)

        if not os.path.exists(mycar_config_path):
            print("Copying my car config overrides")
            shutil.copyfile(myconfig_template_path, mycar_config_path)
            # now copy file contents from config to myconfig, with all lines
            # commented out.
            cfg = open(car_config_path, "rt")
            mcfg = open(mycar_config_path, "at")
            copy = False
            for line in cfg:
                if "import os" in line:
                    copy = True
                if copy:
                    mcfg.write("# " + line)
            cfg.close()
            mcfg.close()

        print("Donkey setup complete.")

class UpdateCar(BaseCommand):
    '''
    always run in the base ~/mycar dir to get latest
    '''

    def parse_args(self, args):
        parser = argparse.ArgumentParser(prog='update', usage='%(prog)s [options]')
        parser.add_argument('--template', default=None, help='name of car template to use')
        parsed_args = parser.parse_args(args)
        return parsed_args

    def run(self, args):
        args = self.parse_args(args)
        cc = CreateCar()
        cc.create_car(path=".", overwrite=True, template=args.template)

class FindCar(BaseCommand):
    def parse_args(self, args):
        pass

    def run(self, args):
        print('Looking up your computer IP address...')
        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        s.connect(("8.8.8.8",80))
        ip = s.getsockname()[0]
        print('Your IP address: %s ' %s.getsockname()[0])
        s.close()

        print("Finding your car's IP address...")
        cmd = "sudo nmap -sP " + ip + "/24 | awk '/^Nmap/{ip=$NF}/B8:27:EB/{print ip}'"
        cmdRPi4 = "sudo nmap -sP " + ip + "/24 | awk '/^Nmap/{ip=$NF}/DC:A6:32/{print ip}'"
        print("Your car's ip address is:" )
        os.system(cmd)
        os.system(cmdRPi4)

class CalibrateCar(BaseCommand):

    def parse_args(self, args):
        parser = argparse.ArgumentParser(prog='calibrate', usage='%(prog)s [options]')
        parser.add_argument('--channel', help="The channel you'd like to calibrate [0-15]")
        parser.add_argument('--address', default='0x40', help="The i2c address you'd like to calibrate [default 0x40]")
        parser.add_argument('--bus', default=None, help="The i2c bus you'd like to calibrate [default autodetect]")
        parser.add_argument('--pwmFreq', default=60, help="The frequency to use for the PWM")
        parser.add_argument('--arduino', dest='arduino', action='store_true', help='Use arduino pin for PWM (calibrate pin=<channel>)')
        parser.set_defaults(arduino=False)
        parsed_args = parser.parse_args(args)
        return parsed_args

    def run(self, args):
        args = self.parse_args(args)
        channel = int(args.channel)

        if args.arduino == True:
            from donkeycar.parts.actuator import ArduinoFirmata

            arduino_controller = ArduinoFirmata(servo_pin=channel)
            print('init Arduino PWM on pin %d' %(channel))
            input_prompt = "Enter a PWM setting to test ('q' for quit) (0-180): "
        else:
            from donkeycar.parts.actuator import PCA9685
            from donkeycar.parts.sombrero import Sombrero

            s = Sombrero()

            busnum = None
            if args.bus:
                busnum = int(args.bus)
            address = int(args.address, 16)
            print('init PCA9685 on channel %d address %s bus %s' %(channel, str(hex(address)), str(busnum)))
            freq = int(args.pwmFreq)
            print("Using PWM freq: {}".format(freq))
            c = PCA9685(channel, address=address, busnum=busnum, frequency=freq)
            input_prompt = "Enter a PWM setting to test ('q' for quit) (0-1500): "
            print()
        while True:
            try:
                val = input(input_prompt)
                if val == 'q' or val == 'Q':
                    break
                pmw = int(val)
                if args.arduino == True:
                    arduino_controller.set_pulse(channel,pmw)
                else:
                    c.run(pmw)
            except KeyboardInterrupt:
                print("\nKeyboardInterrupt received, exit.")
                break
            except Exception as ex:
                print("Oops, {}".format(ex))

class MakeMovieShell(BaseCommand):
    '''
    take the make movie args and then call make movie command
    with lazy imports
    '''
    def __init__(self):
        self.deg_to_rad = math.pi / 180.0

    def parse_args(self, args):
        parser = argparse.ArgumentParser(prog='makemovie')
        parser.add_argument('--tub', help='The tub to make movie from')
        parser.add_argument('--out', default='tub_movie.mp4', help='The movie filename to create. default: tub_movie.mp4')
        parser.add_argument('--config', default='./config.py', help=HELP_CONFIG)
        parser.add_argument('--model', default=None, help='the model to use to show control outputs')
        parser.add_argument('--type', default=None, required=False, help='the model type to load')
        parser.add_argument('--salient', action="store_true", help='should we overlay salient map showing activations')
        parser.add_argument('--start', type=int, default=0, help='first frame to process')
        parser.add_argument('--end', type=int, default=-1, help='last frame to process')
        parser.add_argument('--scale', type=int, default=2, help='make image frame output larger by X mult')
        parser.add_argument('--draw-user-input', default=True, action='store_false', help='show user input on the video')
        parsed_args = parser.parse_args(args)
        return parsed_args, parser

    def run(self, args):
        '''
        Load the images from a tub and create a movie from them.
        Movie
        '''
        args, parser = self.parse_args(args)

        from donkeycar.management.makemovie import MakeMovie

        mm = MakeMovie()
        mm.run(args, parser)

class ShowCnnActivations(BaseCommand):

    def __init__(self):
        import matplotlib.pyplot as plt
        self.plt = plt

    def get_activations(self, image_path, model_path, cfg):
        '''
        Extracts features from an image

        returns activations/features
        '''
        from tensorflow.python.keras.models import load_model, Model

        model_path = os.path.expanduser(model_path)
        image_path = os.path.expanduser(image_path)

        model = load_model(model_path, compile=False)
        image = load_image(image_path, cfg)[None, ...]

        conv_layer_names = self.get_conv_layers(model)
        input_layer = model.get_layer(name='img_in').input
        activations = []
        for conv_layer_name in conv_layer_names:
            output_layer = model.get_layer(name=conv_layer_name).output

            layer_model = Model(inputs=[input_layer], outputs=[output_layer])
            activations.append(layer_model.predict(image)[0])
        return activations

    def create_figure(self, activations):
        import math
        cols = 6

        for i, layer in enumerate(activations):
            fig = self.plt.figure()
            fig.suptitle('Layer {}'.format(i+1))

            print('layer {} shape: {}'.format(i+1, layer.shape))
            feature_maps = layer.shape[2]
            rows = math.ceil(feature_maps / cols)

            for j in range(feature_maps):
                self.plt.subplot(rows, cols, j + 1)

                self.plt.imshow(layer[:, :, j])

        self.plt.show()

    def get_conv_layers(self, model):
        conv_layers = []
        for layer in model.layers:
            if layer.__class__.__name__ == 'Conv2D':
                conv_layers.append(layer.name)
        return conv_layers

    def parse_args(self, args):
        parser = argparse.ArgumentParser(prog='cnnactivations', usage='%(prog)s [options]')
        parser.add_argument('--image', help='path to image')
        parser.add_argument('--model', default=None, help='path to model')
        parser.add_argument('--config', default='./config.py', help=HELP_CONFIG)

        parsed_args = parser.parse_args(args)
        return parsed_args

    def run(self, args):
        args = self.parse_args(args)
        cfg = load_config(args.config)
        activations = self.get_activations(args.image, args.model, cfg)
        self.create_figure(activations)

class ShowHistogram(BaseCommand):

    def parse_args(self, args):
        parser = argparse.ArgumentParser(prog='tubhist', usage='%(prog)s [options]')
        parser.add_argument('--tub', nargs='+', help='paths to tubs')
        parser.add_argument('--record', default=None, help='name of record to create histogram')
        parser.add_argument('--out', default=None, help='path where to save histogram end with .png')
        parsed_args = parser.parse_args(args)
        return parsed_args

    def show_histogram(self, tub_paths, record_name, out):
        '''
        Produce a histogram of record type frequency in the given tub
        '''
        from matplotlib import pyplot as plt
        from donkeycar.parts.datastore import TubGroup
        from donkeycar.parts.tub_v2 import Tub
        import pandas as pd
        output = out or os.path.basename(tub_paths)

        base_path = Path(os.path.expanduser(tub_paths)).absolute().as_posix()
        tub = Tub(base_path)
        records = list(tub)
        user_angles = []
        user_throttles = []

        for record in records:
            user_angle = float(record["user/angle"])
            user_throttle = float(record["user/throttle"])

            user_angles.append(user_angle)
            user_throttles.append(user_throttle)

        df = pd.DataFrame({'user_angle': user_angles, 'user_throttle': user_throttles})

        if record_name is not None:
            df[record_name].hist(bins=50)
        else:
            df.hist(bins=50)

        try:
            if out is not None:
                filename = output
            else:
                if record_name is not None:
                    filename = output + '_hist_%s.png' % record_name.replace('/', '_')
                else:
                    filename = output + '_hist.png'
            plt.savefig(filename)
            print('saving image to:', filename)
        except Exception as e:
            print(e)
        # plt.show()

    def run(self, args):
        args = self.parse_args(args)
        args.tub = ','.join(args.tub)
        self.show_histogram(args.tub, args.record, args.out)

class ShowPredictionPlots(BaseCommand):

    def plot_predictions(self, cfg, tub_paths, model_path, limit, model_type):
        '''
        Plot model predictions for angle and throttle against data from tubs.

        '''
        import matplotlib.pyplot as plt
        import pandas as pd

        model_path = os.path.expanduser(model_path)
        model = dk.utils.get_model_by_type(model_type, cfg)
        # This just gets us the text for the plot title:
        if model_type is None:
            model_type = cfg.DEFAULT_MODEL_TYPE
        model.load(model_path)

        user_angles = []
        user_throttles = []
        pilot_angles = []
        pilot_throttles = []

        from donkeycar.parts.tub_v2 import Tub
        from pathlib import Path

        base_path = Path(os.path.expanduser(tub_paths)).absolute().as_posix()
        tub = Tub(base_path)
        records = list(tub)
        records = records[:limit]
        bar = IncrementalBar('Inferencing', max=len(records))

        for record in records:
            img_filename = os.path.join(base_path, Tub.images(), record['cam/image_array'])
            img = load_image(img_filename, cfg)
            user_angle = float(record["user/angle"])
            user_throttle = float(record["user/throttle"])
            pilot_angle, pilot_throttle = model.run(img)

            user_angles.append(user_angle)
            user_throttles.append(user_throttle)
            pilot_angles.append(pilot_angle)
            pilot_throttles.append(pilot_throttle)
            bar.next()

        angles_df = pd.DataFrame({'user_angle': user_angles, 'pilot_angle': pilot_angles})
        throttles_df = pd.DataFrame({'user_throttle': user_throttles, 'pilot_throttle': pilot_throttles})

        fig = plt.figure()

        title = "Model Predictions\nTubs: " + tub_paths + "\nModel: " + model_path + "\nType: " + model_type
        fig.suptitle(title)

        ax1 = fig.add_subplot(211)
        ax2 = fig.add_subplot(212)

        angles_df.plot(ax=ax1)
        throttles_df.plot(ax=ax2)

        ax1.legend(loc=4)
        ax2.legend(loc=4)

        plt.savefig(model_path + '_pred.png')
        plt.show()

    def parse_args(self, args):
        parser = argparse.ArgumentParser(prog='tubplot', usage='%(prog)s [options]')
        parser.add_argument('--tub', nargs='+', help='The tub to make plot from')
        parser.add_argument('--model', default=None, help='model for predictions')
        parser.add_argument('--limit', type=int, default=1000, help='how many records to process')
        parser.add_argument('--type', default=None, help='model type')
        parser.add_argument('--config', default='./config.py', help=HELP_CONFIG)
        parsed_args = parser.parse_args(args)
        return parsed_args

    def run(self, args):
        args = self.parse_args(args)
        args.tub = ','.join(args.tub)
        cfg = load_config(args.config)
        self.plot_predictions(cfg, args.tub, args.model, args.limit, args.type)

class Train(BaseCommand):

    def parse_args(self, args):
        HELP_FRAMEWORK = 'the AI framework to use (tensorflow|pytorch). ' \
                         'Defaults to config.DEFAULT_AI_FRAMEWORK'
        parser = argparse.ArgumentParser(prog='train', usage='%(prog)s [options]')
        parser.add_argument('--tub', nargs='+', help='tub data for training')
        parser.add_argument('--model', default=None, help='output model name')
        parser.add_argument('--type', default=None, help='model type')
        parser.add_argument('--config', default='./config.py', help=HELP_CONFIG)
        parser.add_argument('--framework',
                            choices=['tensorflow', 'pytorch', None],
                            required=False,
                            help=HELP_FRAMEWORK)
        parser.add_argument('--checkpoint', type=str,
                            help='location of checkpoint to resume training from')
        parser.add_argument('--transfer', type=str, help='transfer model')
        parser.add_argument('--comment', type=str,
                            help='comment added to model database - use '
                                 'double quotes for multiple words')
        parsed_args = parser.parse_args(args)
        return parsed_args

    def run(self, args):
        args = self.parse_args(args)
        args.tub = ','.join(args.tub)
        cfg = load_config(args.config)
        framework = args.framework if args.framework \
            else getattr(cfg, 'DEFAULT_AI_FRAMEWORK', 'tensorflow')

        if framework == 'tensorflow':
            from donkeycar.pipeline.training import train
            train(cfg, args.tub, args.model, args.type, args.transfer,
                  args.comment)
        elif framework == 'pytorch':
            from donkeycar.parts.pytorch.torch_train import train
            train(cfg, args.tub, args.model, args.type,
                  checkpoint_path=args.checkpoint)
        else:
            print(f"Unrecognized framework: {framework}. Please specify one of "
                  f"'tensorflow' or 'pytorch'")

class TrainRemote(BaseCommand):
    """
    train models on a remote server
    """
    # example command:
    #     donkey trainremote --tub ~/mycar/data/tub_4_21-06-11/ ~/mycar/data/tub_3_21-06-11/ --path ~/mycar
    #     --url https://hq.robocarstore.com/train/submit_job --get https://hq.robocarstore.com/train/refresh_job_statuses --model modelname

    WLAN="wlan0"
    HOTSPOT_IF_NAME="uap0"
    REFRESH_JOB_STATUS_URL = "https://hq.robocarstore.com/train/refresh_job_statuses"
    SUBMIT_JOB_URL = "https://hq.robocarstore.com/train/submit_job"
    TIMEOUT = 900 # limit the training to 15min

    def parse_args(self, args):
        parser = argparse.ArgumentParser(prog='train', usage='%(prog)s [options]')
        parser.add_argument('--tub', nargs='+', help='tub data for training')
        parser.add_argument('--path', default=None, help='path where to create car folder')
        parser.add_argument('--url', default=None, help='url of the remote server to submit job')
        parser.add_argument('--get', default=None, help='url of the remote server to get statuses')
        parser.add_argument('--model', default=None, help='output model name')

        parsed_args = parser.parse_args(args)
        return parsed_args

    def generate_tub_archive(self, tub_paths, carapp_path):
        print("generating tub archive")
        f = tempfile.NamedTemporaryFile(mode='w+b', suffix='.tar.gz', delete=False)

        with tarfile.open(fileobj=f, mode='w:gz') as tar:
            for tub_path in tub_paths:
                p = Path(tub_path)
                tar.add(p, arcname=p.name)
            tar.add(f"{carapp_path}/myconfig.py", arcname="myconfig.py")

        f.close()

        return f.name

    def get_wlan_mac_address(self, wlan):
        interfaces = netifaces.interfaces()
        if wlan not in interfaces:
            # logger.error(f"Network interface is not properly configured. {interface} does not exists.")
            return "None"
        addrs =  netifaces.ifaddresses(wlan)

        if addrs is None:
            return "None"

        if (netifaces.AF_LINK in addrs) and (len(addrs[netifaces.AF_LINK]) == 1):
            return addrs[netifaces.AF_LINK][0]['addr']
        else:
            return "None"

    def submit_job(self, tub_paths, carapp_path, submit_job_url):
        filename = self.generate_tub_archive(tub_paths, carapp_path)
        mp_encoder = MultipartEncoder(
            fields={
                'device_id': self.get_wlan_mac_address(self.WLAN),
                'hostname' : gethostname(),
                'tub_archive_file': ('file.tar.gz', open(filename, 'rb'), 'application/gzip'),
                'donkeycar_version': str(__version__)
            }
        )
        print(f"URL submitted to: {submit_job_url}")
        print(f"Data to submit: {mp_encoder}")
        r = requests.post(
            submit_job_url,
            data=mp_encoder,  # The MultipartEncoder is posted as data, don't use files=...!
            # The MultipartEncoder provides the content-type header with the boundary:
            headers={'Content-Type': mp_encoder.content_type}
        )
        print(f"Submission response: HTTP {r.status_code}")
        if (r.status_code == 200):
            # if HTTP 200 OK
            if ("job_uuid" in r.json()):
                try:
                    uuid = r.json()['job_uuid']
                    print(f"Submitted a Training Job to the remote server {submit_job_url}\n uuid: {uuid}")
                    return uuid
                except Exception as e:
                    print(e)
                    raise Exception("Failed to call submit job")
            else:
                raise Exception("Failed to call submit job")
        else:
            raise Exception("Failed to call submit job")

    def get_latest_job_status_from_hq(self, refresh_job_statuses, job_uuids):
        print(f"Getting lastest job status for uuid {job_uuids}")
        # job_uuids = [job_uuid for job_uuid in job_uuids if job_uuid]
        response = requests.post(refresh_job_statuses, data={"job_uuids": job_uuids})
        if response.status_code == 200:
            return response.json()
        else:
            print(response.status_code)
            print(response.content)
            raise Exception("Problem requesting latest job status from hq")

    def run(self, args):
        args = self.parse_args(args)
        tub_paths = list(args.tub)
        car_path = make_dir(args.path or '~/mycar')
        if args.url:
            url = args.url
        else:
            url = self.SUBMIT_JOB_URL
        if args.get:
            url_status = args.get
        else:
            url_status = self.REFRESH_JOB_STATUS_URL

        uuid = self.submit_job(tub_paths, car_path, url)
        if uuid:
            if args.model:
                model_name = args.model
            else:
                model_name = f"job_{uuid}"

            start_time = time.time()
            run_time = 0
            checkagain = True
            while checkagain and (run_time < self.TIMEOUT):
                result = self.get_latest_job_status_from_hq(url_status, uuid)[0]
                checkagain = result['status'] == "SCHEDULED" or result['status'] == "TRAINING"
                print(f"Training Status:{result['status']} at {datetime.now().strftime('%Y-%m-%d, %H:%M:%S')}")

                # print(result)
                time.sleep(5)
                if result['status'] == "COMPLETED":
                    print("Training completed!!")
                    print(f"Model URL: {result['model_url']}")
                    print(f"Model Accuracy URL: {result['model_accuracy_url']}")
                    print(f"Model Movie URL: {result['model_movie_url']}")

                    if not os.path.isdir(f"{car_path}/movies/"):
                        print(f"Creating movie folder '{car_path}/movies/' ")
                        os.mkdir(f"{car_path}/movies/")

                    command = ["curl", "--fail", result['model_url'], "--output", f"{car_path}/models/{model_name}.h5"]
                    proc = subprocess.Popen(command)
                    command = ["curl", "--fail", result['model_accuracy_url'], "--output", f"{car_path}/models/{model_name}.png"]
                    proc = subprocess.Popen(command)
                    command = ["curl", "--fail", result['model_movie_url'], "--output", f"{car_path}/movies/{model_name}.mp4"]
                    proc = subprocess.Popen(command)
                    print("Downloaded model, accuracy graph and movie from the URLs")
                    break
                elif result['status'] in ["NO_CAPACITY", "NO_QUOTA", "SPOT_REQUEST_FAILED", "TIMEOUT"]:
                    raise Exception(f"Failed to train the submitted job\nError : {result['status']}")

                run_time = time.time()-start_time
                print(f"Time spent: {run_time} s")

class Gui(BaseCommand):
    def run(self, args):
        from donkeycar.management.kivy_ui import main
        main()

def execute_from_command_line():
    """
    This is the function linked to the "donkey" terminal command.
    """
    commands = {
        'createcar': CreateCar,
        'findcar': FindCar,
        'calibrate': CalibrateCar,
        'tubclean': TubManager,
        'tubhist': ShowHistogram,
        'tubplot': ShowPredictionPlots,
        'makemovie': MakeMovieShell,
        'createjs': CreateJoystick,
        'cnnactivations': ShowCnnActivations,
        'update': UpdateCar,
        'train': Train,
        'trainremote': TrainRemote,
        'ui': Gui,
    }

    args = sys.argv[:]

    if len(args) > 1 and args[1] in commands.keys():
        command = commands[args[1]]
        c = command()
        c.run(args[2:])
    else:
        dk.utils.eprint('Usage: The available commands are:')
        dk.utils.eprint(list(commands.keys()))

if __name__ == "__main__":
    execute_from_command_line()


import argparse

parser = argparse.ArgumentParser()
parser.add_argument("-W", type=int)
parser.add_argument("-w", nargs='+', type=int)
parser.add_argument("-n", type=int)
args = parser.parse_args()

def maxweight(W, w):
    """
    Function, which find maximum weight which can be stored in rucksack
    """
    w = [0] + w
    items = len(w)
    capacity = W + 1
    matrix = [[0 for i in range(items)] for i in range(capacity)]

    for i in range(1, items):
        for j in range(1, capacity):
            matrix[j][i] = matrix[j][i - 1]
            if w[i] <= j:
                val = matrix[j - w[i]][i - 1] + w[i]
                if matrix[j][i] < val:
                    matrix[j][i] = val
    return matrix[-1][-1]

    # read capacity, number of bars and weight of every bar

print(maxweight(args.W, args.w))


from ruxit.api.base_plugin import RemoteBasePlugin
import logging
from kafka_offset_reader import KafkaOffsetReader

logger = logging.getLogger(__name__)

class KafkaOffsetPlugin(RemoteBasePlugin):

    def initialize(self, **kwargs):
        self.kafka = KafkaOffsetReader(self.config)

    def query(self, **kwargs):
      highwaters = self.kafka.get_highwater()
      offsets = self.kafka.get_offsets()
      id = "Kafka Cluster" + str(self.config["bootstrap_servers"])
      group = self.topology_builder.create_group(identifier=id, group_name=id)
      # topic > group > partition

      for (topic, partition_offsets) in highwaters.items():
        device = group.create_device(identifier=str(topic))

        highwaters = dict()
        for (partition, offset) in partition_offsets.items():
          highwaters[partition.partition] = offset
          device.absolute(key="topic.highwater", dimensions={"partition": str(partition)}, value=offset)

        if topic in offsets:
          for (consumer_group, group_offsets) in offsets[topic].items():
            for (partition, partition_offset) in group_offsets.items():
              lag = highwaters.get(partition, partition_offset - 1) - partition_offset
              logger.debug("Lag calculated for " + topic + " - " + str(partition) + ": " + str(lag))
              device.absolute(key="consumer.lag", dimensions={"partition": str(partition), "consumer_group": str(consumer_group)}, value=lag)

              device.absolute(key="consumer.offset", dimensions={"partition": str(partition), "consumer_group": str(consumer_group)}, value=partition_offset)


import torch
import torch.nn as nn
import torch.nn.functional as F

class CharbonnierLoss(nn.Module):
    """L1 charbonnier loss."""

    def __init__(self, epsilon=1e-3, reduce=True):
        super(CharbonnierLoss, self).__init__()
        self.eps = epsilon * epsilon
        self.reduce = reduce

    def forward(self, X, Y):
        diff = torch.add(X, -Y)
        error = torch.sqrt(diff * diff + self.eps)
        if self.reduce:
            loss = torch.mean(error)
        else:
            loss = error
        return loss


'''
This script is to upgrade project's files from old-style (storage_version 1)
to new-style (storage_version 2), to well support multi-tenant use.
This procedure is forced to run on server start.
On server start:
1 Check all project's storage_version
2 If it is "1", upgrade it to 2:
2.1 move all files into new-style place
2.2 set project's storage_version to 2
3 Continue run
add in version 0.6.6
'''
from ..models import Project, session_scope
from ..storage import ProjectStorage
from ..config import Config

import logging

logger = logging.getLogger(__name__)

def upgrade(config=None):
    if config is None:
        config = Config()
    logger.info('Start upgrading projects storage')
    with session_scope() as session:
        for project in session.query(Project).all():
            if project.storage_version == 1:
                _move_project_files(project, config)
                project.storage_version = 2
                session.add(project)
                session.commit()

def _move_project_files(project, config):
    old_project_storage = ProjectStorage(data_dir='.', project=project)
    new_project_storage = ProjectStorage(data_dir=config.get('project_storage_dir'), project=project, storage_version=2)

    egg_version, eggf = old_project_storage.get_egg()
    new_project_storage.put_egg(eggf, egg_version)

    for spider in project.spiders:
        for job in spider.historical_jobs:
            try:
                job_items = old_project_storage.get_job_items(job)
            except IOError:
                job_items = None
            try:
                job_log = old_project_storage.get_job_log(job)
            except IOError:
                job_log = None
            new_project_storage.put_job_data(job, log_file=job_log, item_file=job_items)


"""empty message

Revision ID: 1d9b4ec155f7
Revises: cbd26a0de3b3
Create Date: 2018-03-23 16:06:05.444079

"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql

# revision identifiers, used by Alembic.
revision = '1d9b4ec155f7'
down_revision = 'cbd26a0de3b3'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('webinar', sa.Column('open_to_account_types', postgresql.ARRAY(sa.String(length=256)), nullable=True))
    op.add_column('webinar', sa.Column('open_to_all', sa.Boolean(), nullable=True))
    op.execute("UPDATE webinar SET open_to_all=true")
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('webinar', 'open_to_all')
    op.drop_column('webinar', 'open_to_account_types')
    # ### end Alembic commands ###


import cv2
from datetime import datetime

video = cv2.VideoCapture(
    r'C:\Users\abdul\Desktop\Python Videos\VID-20171220-WA0022.mp4')

# We need to set resolutions.
# so, convert them from float to integer.
frame_width = int(video.get(3))
frame_height = int(video.get(4))

size = (frame_width, frame_height)

# Below VideoWriter object will create
# a frame of above defined The output
# is stored in 'filename.avi' file.
result = cv2.VideoWriter('filename.avi',
                         cv2.VideoWriter_fourcc(*'MJPG'),
                         60, size)

# speed increase and text will be written
while True:
    ret, frame = video.read()  # read frame/image one by one

    now = datetime.now()
    current_time = now.strftime("%m/%d/%Y, %H:%M:%S")
    text = "Live Streaming"

    if not ret:
        break

    font = cv2.FONT_HERSHEY_SIMPLEX  # font to apply on text
    cv2.putText(frame, current_time, (50, 50), font,
                1, (0, 0, 255), 2)  # add text on frame
    result.write(frame)

video.release()  # release video capture object
result.release()  # release video capture object
cv2.destroyAllWindows()  # destroy all frame windows


import asyncio
import os
import sys
from contextlib import asynccontextmanager

import uvicorn
from fastapi import FastAPI
from fastapi.staticfiles import StaticFiles
from loguru import logger

from routers import example, home
from utils import config_util, logger_util

WORKING_DIR = os.path.realpath(os.path.dirname(__file__))
CLIENT_DIR = os.path.realpath(os.path.join(WORKING_DIR, "../client"))

@asynccontextmanager
async def lifespan(app: FastAPI):
    # Before FastAPI app start.
    logger.debug("Before FastAPI app start.")
    yield
    # Before FastAPI app shutdown.
    logger.debug("Before FastAPI app shutdown.")

app = FastAPI(lifespan=lifespan)

app.include_router(home.router)
app.include_router(example.router)

# Serve static files.
app.mount(
    "/",
    StaticFiles(directory=CLIENT_DIR, html=True),
    name="static",
)

async def main():
    uvicorn_config = uvicorn.Config(
        APP,
        settings.uvicorn.host,
        settings.uvicorn.port,
        log_level=settings.uvicorn.log_level,
    )
    logger_util.override_uvicorn_logger()
    server = uvicorn.Server(uvicorn_config)
    await server.serve()

if __name__ == "__main__":
    APP = "main:app"
    settings = config_util.get_settings(
        os.path.join(WORKING_DIR, "configs/settings.json"),
    )
    if settings is None:
        sys.exit(1)

    if settings.logger is not None:
        logger_util.setup_logger(settings.logger)

    if os.environ.get("MODE", "") == "dev":
        logger.info("In development mode.")
        uvicorn.run(
            APP,
            host=settings.uvicorn.host,
            port=settings.uvicorn.port,
            reload=True,
            log_level=settings.uvicorn.log_level,
        )
    else:
        asyncio.run(main())


from __future__ import annotations

import math
import random
import re
from abc import ABC, abstractmethod
from dataclasses import dataclass
import cv2 as cv
import sys

import numpy as np
import pytesseract
import requests

@dataclass
class ChainParam:
    src: np
    copySrc: np = None
    pre: np = None
    dst: np = None
    vehicle_no: str = ''

class Handler(ABC):
    @abstractmethod
    def set_next(self, handler: Handler) -> Handler:
        pass

    @abstractmethod
    def handle(self, param: ChainParam):
        pass

class AbstractHandler(Handler):
    _next_handler: Handler = None

    def set_next(self, handler: Handler) -> Handler:
        self._next_handler = handler

        return handler

    def handle(self, param: ChainParam):
        if self._next_handler:
            return self._next_handler.handle(param)

        return None

class PreProcess(AbstractHandler):
    def handle(self, param: ChainParam):
        gray = cv.cvtColor(param.src, cv.COLOR_BGR2GRAY)
        src_bin = cv.GaussianBlur(gray, (0, 0), 2, sigmaY=0, borderType=cv.BORDER_DEFAULT)
        src_bin = cv.adaptiveThreshold(src_bin, 255, cv.ADAPTIVE_THRESH_GAUSSIAN_C, cv.THRESH_BINARY_INV, 19, 4)
        param.pre = src_bin

        return super().handle(param)

class CutVehicleRegion(AbstractHandler):
    # src = None
    # pre = None

    def handle(self, param: ChainParam):
        min_pts = None
        min_area: int = sys.maxsize
        contours, _ = cv.findContours(param.pre, cv.RETR_LIST, cv.CHAIN_APPROX_NONE)
        for pts in contours:
            # 외곽선 근사화
            approx = cv.approxPolyDP(pts, cv.arcLength(pts, True) * 0.02, True)

            # 컨벡스가 아니고, 사각형이 아니면 무시
            # if not cv.isContourConvex(approx):
            #     continue
            # if len(approx) != 4:
            #     continue

            x, y, w, h = cv.boundingRect(pts)
            cv.rectangle(param.copySrc, (x, y), (x + w, y + h), (0, 0, 255), thickness=3)
            if h / w > 0.5:
                continue
            area = cv.contourArea(pts)
            if area < 80 * 80:
                continue
            # cv.drawContours(param.src, [approx], 0, (0, 0, 255), 3)
            # cv.rectangle(param.copySrc, (x, y), (x + w, y + h), (0, 0, 255), thickness=3)
            # cv.polylines(param.src, [approx], True, (0, 0, 255), 2, cv.LINE_AA)
            count = self.get_connected_components((x, y, w, h), param)
            if count < 4 or count > 20:
                continue

            if w * h < min_area:
                min_area = w * h
                min_pts = pts

        if min_pts is not None:
            x, y, w, h = cv.boundingRect(min_pts)
            param.dst = param.src[y:y + h, x + 20:x + w - 20]
            param.src = param.src[y:y + h, x + 20:x + w - 20]
            param.copySrc = param.src.copy()
            cv.rectangle(param.copySrc, (x, y), (x + w, y + h), (0, 0, 255), thickness=3)

        return super().handle(param)

    def get_connected_components(self, rect, param: ChainParam) -> int:
        component_count = 0
        x, y, w, h = rect
        mat = param.pre[y:y + h, x:x + w]
        contours, _ = cv.findContours(mat, cv.RETR_LIST, cv.CHAIN_APPROX_NONE)
        if len(contours) == 0:
            return 0
        for pts in contours:
            if cv.contourArea(pts) < 200:
                continue
            if not self.guess_vehicle_no(pts):
                continue

            similar_count = self.similar_contour(contours, pts)
            if similar_count < 3:
                continue

            component_count = component_count + 1

        return component_count

    @staticmethod
    def guess_vehicle_no(pts) -> bool:
        x, y, w, h = cv.boundingRect(pts)
        ratio = h / w
        if ratio < 1.0 or ratio > 4.0:
            return False
        return True

    @staticmethod
    def similar_contour(contours, pts) -> int:
        similar_count = 0
        x1, y1, w1, h1 = cv.boundingRect(pts)
        diagonal_length = math.sqrt(w1 * w1 + h1 * h1)
        for pts in contours:
            x2, y2, w2, h2 = cv.boundingRect(pts)
            area_diff: float = abs(w1 * h1 - w2 * h2) / (w1 * h1)
            width_diff: float = abs(w1 - w2) / w1
            height_diff: float = abs(h1 - h2) / h1

            if area_diff > 0.6 or width_diff > 0.8 or height_diff > 0.2:
                continue

            distance = math.sqrt((x1 - x2) ** 2 + (y1 - y2) ** 2)
            if distance > diagonal_length * 5:
                continue

            angle_diff = 0
            dx = abs(x1 - x2)
            dy = abs(y1 - y2)
            if dx == 0:
                angle_diff = 90
            else:
                angle_diff = math.atan(dy / dx) * (180 / math.pi)
            if angle_diff > 20.0:
                continue

            similar_count = similar_count + 1
        return similar_count

class GetVehicleNo(AbstractHandler):
    def handle(self, param: ChainParam):
        if param.dst is None:
            return super().handle(param)

        p = re.compile('[0-9]{2,3}[가-힣]{1}[0-9]{4}|[가-힣]{2}[0-9]{2}[가-힣]{1}[0-9]{4}')
        gray = cv.cvtColor(param.dst, cv.COLOR_BGR2GRAY)
        src_bin = cv.GaussianBlur(gray, (0, 0), 1, sigmaY=4, borderType=cv.BORDER_DEFAULT)
        thres, _ = cv.threshold(src_bin, 0, 255, cv.THRESH_BINARY | cv.THRESH_OTSU, dst=src_bin)

        src_bin = cv.erode(src_bin, None, iterations=2)

        src_bin = cv.copyMakeBorder(src_bin, 10, 10, 10, 10, cv.BORDER_CONSTANT, value=(0, 0, 0))
        vehicle_no = self.get_text_from_image(src_bin)
        if vehicle_no != '':
            m = p.match(vehicle_no)
            if m is not None:
                param.vehicle_no = m.group()
                return super().handle(param)

        cv.imshow('src_bin', src_bin)
        # cv.waitKey()
        vehicle_no = self.retry_vehicle_no(gray, thres, p)
        param.vehicle_no = vehicle_no
        print(vehicle_no)
        if vehicle_no == '' or len(vehicle_no) == 4:
            v = self.retry_vehicle_no(gray, thres, p, make_border=True)
            print(v)
            if vehicle_no == '' or len(v) >= 4:
                param.vehicle_no = v

        return super().handle(param)

    def retry_vehicle_no(self, gray, thres, p, iterations=10, make_border=False) -> str:
        for i in range(iterations):
            thres = thres + 10
            src_bin = cv.GaussianBlur(gray, (0, 0), 1, sigmaY=4, borderType=cv.BORDER_DEFAULT)
            thres, _ = cv.threshold(src_bin, thres, 255, cv.THRESH_BINARY, dst=src_bin)

            src_bin = cv.erode(src_bin, None, iterations=2)

            if make_border:
                src_bin = cv.copyMakeBorder(src_bin, 10, 10, 10, 10, cv.BORDER_CONSTANT, value=(0, 0, 0))
            vehicle_no = self.get_text_from_image(src_bin)
            if vehicle_no != '':
                m = p.match(vehicle_no)
                if m is not None:
                    return m.group()
                elif len(vehicle_no) > 4:
                    v = vehicle_no[-4:]
                    if v.isdigit():
                        return v

        return ''

    @staticmethod
    def get_text_from_image(src_bin) -> str:
        result: str = ''
        vehicle_no = pytesseract.image_to_string(src_bin, lang='kor', config='--oem 3 --psm 7')
        for i in range(len(vehicle_no)):
            v = vehicle_no[i]
            print(v)
            if ('가' <= v <= '힣') or v.isdigit():
                result = result + v

        return result

if __name__ == "__main__":
    # url = 'https://parkingcone.s3.ap-northeast-2.amazonaws.com/real/user_vehicle/2023/04/eaee1dfc75754d44046cfb9177c59c8b/1682679738_FVBUUC/f9c2755a3aa4cc06404b8e64f030c'
    # url = 'https://parkingcone.s3.ap-northeast-2.amazonaws.com/real/user_vehicle/2023/05/eaee1dfc75754d44046cfb9177c59c8b/1682679738_FVBUUC/7cccd183e821468e65b81b734'
    # url = 'https://parkingcone.s3.ap-northeast-2.amazonaws.com/real/user_vehicle/2023/04/eaee1dfc75754d44046cfb9177c59c8b/1682679738_FVBUUC/2cbdb867245618e515a61776b26dcd'
    url = 'https://parkingcone.s3.ap-northeast-2.amazonaws.com/real/user_vehicle/2023/05/bc8e73ed52dc49fe9bf95149b00a9f31/1683169300_DGSPYV/66d64a516a3950a1686ce524453b9d81'
    image_array = np.asarray(bytearray(requests.get(url).content), dtype=np.uint8)
    src = cv.imdecode(image_array, cv.IMREAD_COLOR)
    # src = cv.imread('../imgs/vehicle18.jpeg')
    if src is None:
        print('image read fail!!')
        sys.exit()

    chainParam = ChainParam(src)
    chainParam.copySrc = src.copy()

    pre = PreProcess()
    pre.set_next(CutVehicleRegion()).set_next(PreProcess()).set_next(CutVehicleRegion()).set_next(GetVehicleNo())
    pre.handle(chainParam)

    cv.imshow('pre', chainParam.pre)
    if chainParam.dst is not None:
        cv.imshow('dst', chainParam.dst)
    cv.imshow('src', chainParam.copySrc)
    print(chainParam.vehicle_no)

    cv.waitKey()
    cv.destroyAllWindows()


import customtkinter
import tkinter
import tksheet
import getperf
import SM3
from tkinter import filedialog

customtkinter.set_default_color_theme("dark-blue")
customtkinter.set_appearance_mode("light")

path = ""

def tk_center():
    screen_width = app.winfo_screenwidth()
    screen_height = app.winfo_screenheight()
    width = 400
    height = 300
    x = int(screen_width / 2 - width / 2)
    y = int(screen_height / 2 - height / 2)
    size = '{}x{}+{}+{}'.format(width, height, x, y)
    return size

class AboutWindow(customtkinter.CTkToplevel):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.geometry(tk_center())
        self.title("关于")
        self.label = customtkinter.CTkLabel(self, text="ToplevelWindow")
        self.label.pack(padx=20, pady=20)
        self.wm_attributes('-topmost',1)

class FileFrame(customtkinter.CTkFrame):
    def __init__(self, master, **kwargs):
        super().__init__(master, **kwargs)

        self.grid_columnconfigure(1, weight=1)
        self.grid_columnconfigure(0, weight=1)
        self.filepath = customtkinter.CTkTextbox(self, height=20)
        self.filepath.grid(row=0, column=0, padx=20, pady=10, sticky="ew")
        self.selectbutton = customtkinter.CTkButton(self, text="选择文件", command=self.FileSelect)
        self.selectbutton.grid(row=0, column=1, padx=20, pady=20, sticky="e")

    def FileSelect(self):
        global path
        path = filedialog.askopenfilename()
        self.filepath.delete(1.0, tkinter.END)
        self.filepath.insert(tkinter.END, path)

class inputFrame(customtkinter.CTkFrame):
    def __init__(self, input_name, master, **kwargs):
        super().__init__(master, **kwargs)

        self.grid_rowconfigure(0, weight=1)
        self.grid_rowconfigure(1, weight=1)
        self.grid_columnconfigure(0, weight=1)

        self.label = customtkinter.CTkLabel(self, text=input_name, height=20)
        self.label.grid(row=0, column=0, padx=20, pady=10, sticky="n")

        self.input_box = customtkinter.CTkTextbox(self)
        self.input_box.grid(row=1, column=0, padx=20, pady=20, sticky="nsew")

class SideFrame(customtkinter.CTkFrame):
    def __init__(self, master, **kwargs):
        super().__init__(master, **kwargs)

        self.about_window = None
        self.label1 = customtkinter.CTkLabel(self, text="工具箱")
        self.label1.grid(row=0, column=0, padx=20, pady=20)
        self.button1 = customtkinter.CTkButton(self, text="性能提取", command=master.ShowPerf)
        self.button1.grid(row=1, column=0, padx=20, pady=5)
        self.button2 = customtkinter.CTkButton(self, text="随机数检测", command=master.randomtest)
        self.button2.grid(row=2, column=0, padx=20, pady=5)
        self.button3 = customtkinter.CTkButton(self, text="SM3算法", command=master.SM3Hash)
        self.button3.grid(row=3, column=0, padx=20, pady=5)
        self.button4 = customtkinter.CTkButton(self, text="文本处理", command=master.SM3Hash)
        self.button4.grid(row=4, column=0, padx=20, pady=5)
        self.aboutbutton = customtkinter.CTkButton(self, text="关于", command=self.open_about)
        self.aboutbutton.grid(row=8, column=0, padx=20, pady=20, sticky="s")

    def open_about(self):
        if self.about_window is None or not self.about_window.winfo_exists():
            self.about_window = AboutWindow(self)

class PerfFrame(customtkinter.CTkFrame):
    def __init__(self, master, **kwargs):
        super().__init__(master, **kwargs)

        self.sheet = None
        self.grid_columnconfigure(0, weight=1)
        self.grid_rowconfigure(0, weight=1)

        self.files = FileFrame(self, height=60)
        self.files.grid(row=1, column=0, padx=20, sticky="sew")

        self.start = customtkinter.CTkButton(self, text="开始", command=self.getres)
        self.start.grid(row=0, pady=20, padx=40, sticky="se")

    def getres(self):
        global path
        res = getperf.get_perf(path)
        self.sheet = tksheet.Sheet(self, headers=["测试项", "源数据", "结果"], height=750, width=648,
                                   show_x_scrollbar=False,
                                   vertical_grid_to_end_of_window=True,
                                   horizontal_grid_to_end_of_window=True)
        self.sheet.enable_bindings()
        self.sheet.font(("Times New Roman", 16, "normal"))
        self.sheet.header_font(("Times New Roman", 16, "bold"))
        self.sheet.grid(row=0, column=0, padx=40, pady=40, sticky="nsw")
        for i in range(3):
            self.sheet.column_width(column=i, width=200)
            self.sheet.set_column_data(i, values=res[i], add_rows=True, redraw=False)

class RandomFrame(customtkinter.CTkFrame):
    def __init__(self, master, **kwargs):
        super().__init__(master, **kwargs)

        print("random")

class SM3Frame(customtkinter.CTkFrame):
    def __init__(self, master, **kwargs):
        super().__init__(master, **kwargs)

        self.grid_columnconfigure((0, 1), weight=1)
        self.grid_rowconfigure((0, 1), weight=1)
        self.message_input = inputFrame("消息原文", master=self)
        self.message_input.grid(row = 0, column=0, sticky = "nsew")
        self.key_input = inputFrame("HMAC密钥", master=self)
        self.key_input.grid(row=0, column=1, sticky = "nsew")
        self.hash_output = inputFrame("HASH结果", master=self)
        self.hash_output.grid(row=1, column=0, sticky = "nsew")
        self.hash_output = inputFrame("HMAC结果", master=self)
        self.hash_output.grid(row=1, column=1, sticky = "nsew")

        print("sm3")

class App(customtkinter.CTk):
    def __init__(self):
        super().__init__()

        self.sm3 = None
        self.random = None
        self.func = None
        self.title('MuTools')
        self.minsize(960, 720)
        self.grid_rowconfigure(0, weight=1)
        self.grid_columnconfigure(0, weight=0)
        self.grid_columnconfigure(1, weight=1)

        self.side = SideFrame(master=self)
        self.side.grid_rowconfigure(8, weight=1)
        self.side.grid(row=0, column=0, sticky="nsew")

    def ShowPerf(self):
        self.func = PerfFrame(master=self)
        self.func.grid(row=0, column=1, sticky="nsew")

    def randomtest(self):
        self.random = RandomFrame(master=self)
        self.random.grid(row=0, column=1, sticky="nsew")

    def SM3Hash(self):
        self.sm3 = SM3Frame(master=self, corner_radius=0)
        self.sm3.grid(row=0, column=1, sticky="nsew")

if __name__ == "__main__":
    app = App()
    app.mainloop()


__author__ = 'pborky'
__all__ = ['google_search', 'root_page']
from quirks.functional import partial, combinator
from quirks.iterable import itake

from .scraping import Chain, Form, Items, MySession , KeywordSet
from .addrutil import addrutil

import logging
logger = logging.getLogger(__name__)

class ScraperMetaclass(type):
    def __new__(mcs, name, bases, attrs):
        Meta = attrs.pop('Meta', None)
        scraping = getattr(Meta, 'scraping', None)
        session_init = getattr(Meta, 'session_init', {})
        #scrapping_init = getattr(Meta, 'scrapping_init', {})

        def _get_session(self, SessClass):
            return SessClass(**session_init)

        #def _scraping(self, session, *args, **kwargs):
        #    return scraping(session, **scrapping_init)

        attrs['_get_session'] = _get_session
        attrs['_scraping'] = scraping

        new_class = super(ScraperMetaclass, mcs).__new__(mcs, name, bases, attrs)
        return new_class

class Scraper(object):
    __metaclass__ =  ScraperMetaclass
    def __init__(self):
        super(Scraper,self).__init__()
        self._sess = self._get_session(MySession)
        #self._scraping = self._get_scrapping(self._sess)
    def __call__(self, *args, **kwargs):
        try:
            return self._scraping(self._sess, *args, **kwargs)
        except Exception as e:
            logger.error(e)
            logger.warning('Reloading webkit session.')
            try:
                self._sess.kill()
            except Exception:
                pass
            self._sess = self._get_session(MySession)
    def __del__(self):
        self._sess.kill()
        super(Scraper,self).__del__()

class GoogleSearch(Scraper):
    class Meta:
        scraping = Chain(
            Form('/',
                q = '//input[@name="q"]'
            ),
            Items(
                _li = '//li[@class="g"]//h3[@class="r"]/a',
                _next = '//a[@class="pn"]',
                url = lambda a: a.get_attr('href').decode('utf-8'),
                title = lambda a: a.text().decode('utf-8'),
            )
        )
        session_init = {
            'base_url': 'https://google.com/',
        }

class RootPage(Scraper):
    def __call__(self, url, *args, **kwargs):
        self._sess.base_url = url
        return super(RootPage, self).__call__(*args, **kwargs)
    class Meta:
        scraping = KeywordSet('/')

google_search = GoogleSearch()
engines = {
    'GOOG_S': google_search
}
root_page_keywords = RootPage()


import collections
import numpy as np
import tensorflow as tf
import tensorflow.contrib.seq2seq as seq2seq
from tensorflow.contrib.seq2seq.python.ops import helper as helper_py

from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.layers import core as layer_core
from tensorflow.python.layers import base as layers_base
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import control_flow_ops
from tensorflow.python.ops import embedding_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import rnn_cell_impl
from tensorflow.python.ops.distributions import categorical
from tensorflow.python.util import nest
from tensorflow.python.framework import tensor_shape

class BasicVectorDecoderOutput(
    collections.namedtuple("BasicVectorDecoderOutput",
                           ("rnn_output", "stop_vector", "stop_id"))):
  pass

class BasicVectorDecoder(seq2seq.BasicDecoder):
    """Basic sampling decoder with vector input / output."""

    def __init__(self, cell, helper, initial_state, output_layer=None):
        """Initialize BasicVectorDecoder.

        Args:
            cell: An `RNNCell` instance.
            helper: A `Helper` instance.
            initial_state: A (possibly nested tuple of...) tensors and TensorArrays.
                The initial state of the RNNCell.
            output_layer:An instance of `tf.layers.Layer`, i.e., `tf.layers.Dense`.
                If not provided, use 1 fc layer.

        Raises:
            TypeError: if `cell`, `helper` or `output_layer` have an incorrect type.
        """
        if not rnn_cell_impl._like_rnncell(cell):  # pylint: disable=protected-access
            raise TypeError("cell must be an RNNCell, received: %s" % type(cell))
        if not isinstance(helper, helper_py.Helper):
            raise TypeError("helper must be a Helper, received: %s" % type(helper))
        if (output_layer is not None and not isinstance(output_layer, layers_base.Layer)):
            raise TypeError(
                "output_layer must be a Layer, received: %s" % type(output_layer))
        self._cell = cell
        self._helper = helper
        self._initial_state = initial_state
        if output_layer is None:
            self._output_layer = layer_core.Dense(2, use_bias=True,
                                                  name="stop_predictor")
        else:
            self._output_layer = output_layer

    def _rnn_output_size(self):
        return self._cell.output_size

    @property
    def output_size(self):
        # Return the cell output and the id
        return BasicVectorDecoderOutput(
            rnn_output=self._rnn_output_size(),
            # stop_vector=tensor_shape.TensorShape([self.batch_size, 2]),
            stop_vector=2,
            stop_id=tensor_shape.TensorShape([]))

    @property
    def output_dtype(self):
        # Assume the dtype of the cell is the output_size structure
        # containing the input_state's first component's dtype.
        # Return that structure and int32 (the id)
        dtype = nest.flatten(self._initial_state)[0].dtype
        return BasicVectorDecoderOutput(
            nest.map_structure(lambda _: dtype, self._rnn_output_size()),
            dtype, dtypes.int32)

    def step(self, time, inputs, state, name=None):
        """Perform a decoding step.

        Args:
            time: scalar `int32` tensor.
            inputs: A (structure of) input tensors.
            state: A (structure of) state tensors and TensorArrays.
            name: Name scope for any created operations.

        Returns:
            `(outputs, next_state, next_inputs, finished)`.
        """
        with ops.name_scope(name, "BasicVectorDecoderStep", (time, inputs, state)):
            cell_outputs, cell_state = self._cell(inputs, state)
            stop_vector = self._output_layer(cell_outputs)
            stop_id = self._helper.sample(
                time=time, stop_vector=stop_vector, state=cell_state)
            (finished, next_inputs, next_state) = self._helper.next_inputs(
                time=time,
                outputs=cell_outputs,
                state=cell_state,
                stop_id=stop_id)
        outputs = BasicVectorDecoderOutput(cell_outputs,
                                           stop_vector,
                                           stop_id)
        return (outputs, next_state, next_inputs, finished)

class VectorTrainingHelper(seq2seq.TrainingHelper):
    """A helper for use during training. Only reads inputs.

    Contrary to TrainingHelper, this reads vector inputs.
    stop_id is sampled in a greedy fashion.
    """
    def sample(self, time, stop_vector, state, name=None):
        """sample stop signal."""
        del time, state  # unused by sample_fn
        stop_id = math_ops.cast(
            math_ops.argmax(stop_vector, axis=-1), dtypes.int32)
        return stop_id

class VectorGreedyEmbeddingHelper(seq2seq.GreedyEmbeddingHelper):
    """A helper for use during inference.

    Pass output vectors as inputs to following sequences.
    Stop signal is predicted by additional layer.
    """

    def __init__(self, start_inputs):
        """Initializer.

        Args:
            start_inputs: inputs for the initial state.
        """
        self._batch_size = start_inputs.get_shape().as_list()[0]
        self._start_inputs = start_inputs

    @property
    def batch_size(self):
        return self._batch_size

    def initialize(self, name=None):
        finished = array_ops.tile([False], [self._batch_size])
        return (finished, self._start_inputs)

    def sample(self, time, stop_vector, state, name=None):
        """sample stop signal."""
        del time, state  # unused by sample_fn
        stop_id = math_ops.cast(
            math_ops.argmax(stop_vector, axis=-1), dtypes.int32)
        return stop_id

    def next_inputs(self, time, outputs, state, stop_id, name=None):
        """next_inputs_fn for GreedyEmbeddingHelper."""
        del time  # unused by next_inputs_fn
        finished = math_ops.equal(stop_id, 1)  # 1 is stop signal
        all_finished = math_ops.reduce_all(finished)
        next_inputs = control_flow_ops.cond(
            all_finished,
            # If we're finished, the next_inputs value doesn't matter
            lambda: self._start_inputs,
            lambda: outputs)
        return (finished, next_inputs, state)

class SyntacticGreedyEmbeddingHelper(seq2seq.Helper):
    """A helper for use during inference.

    Perform the argmax over the grammatically correct output candidates.
    Outputs are passes through an embedding layer to get the next input.
    """

    def __init__(self, dsl_syntax, max_program_len, embedding, start_tokens, end_token):
        """Initializer.

        Args:
            dsl_syntax: Syntax checker for generating next possible tokens.
            max_program_len: maximum program length
            embedding: A callable that takes a vector tensor of `ids` (argmax ids),
                or the `params` argument for `embedding_lookup`. The returned tensor
                will be passed to the decoder input.
            start_tokens: `int32` vector shaped `[batch_size]`, the start tokens
            end_token: `int32` scalar, the token that marks end of decoding.

        Raises:
            ValueError: if `start_tokens` is not a 1D tensor or `end_token` is not a
                scalar.
        """
        self.dsl_syntax = dsl_syntax
        self.max_program_len = max_program_len
        self.previous_tokens = []
        self.previous_probs = []
        self.previous_masks = []

        if callable(embedding):
            self._embedding_fn = embedding
        else:
            self._embedding_fn = (
                lambda ids: embedding_ops.embedding_lookup(embedding, ids))

        self._start_tokens = ops.convert_to_tensor(
            start_tokens, dtype=dtypes.int32, name="start_tokens")
        self._end_token = ops.convert_to_tensor(
            end_token, dtype=dtypes.int32, name="end_token")
        if self._start_tokens.get_shape().ndims != 1:
            raise ValueError("start_tokens must be a vector")
        self._batch_size = array_ops.size(start_tokens)
        if self._end_token.get_shape().ndims != 0:
            raise ValueError("end_token must be a scalar")
        self._start_inputs = self._embedding_fn(self._start_tokens)

    @property
    def batch_size(self):
        return self._batch_size

    def initialize(self, name=None):
        finished = array_ops.tile([False], [self._batch_size])

        def init_prev_tokens(finished):
            self.previous_tokens = []
            self.previous_probs = []
            self.previous_masks = []
            return finished
        new_finished = tf.py_func(init_prev_tokens, [finished], tf.bool)
        new_finished.set_shape(finished.get_shape())
        return (new_finished, self._start_inputs)

    def sample(self, time, outputs, state, name=None):
        """sample for SyntacticGreedyEmbeddingHelper."""
        del time, state  # unused by sample_fn
        # Outputs are logits, use argmax to get the most probable id
        if not isinstance(outputs, ops.Tensor):
            raise TypeError("Expected outputs to be a single Tensor, got: %s" %
                            type(outputs))

        # Mask outputs to reduce candidates to syntatically correct ones.
        def mask_output(outputs, end_token):
            if len(self.previous_tokens) == 0:  # when there is no previous token, skip masking.
                mask = np.zeros(outputs.shape, dtype=outputs.dtype)
                mask[:, self.dsl_syntax.token2int['DEF']] = 1
                return mask
            tokens = np.stack(self.previous_tokens, axis=1)
            masks = []
            for i in range(outputs.shape[0]):
                if tokens[i][-1] == end_token:
                    next_tokens = [end_token]
                else:
                    try:
                        p_str = self.dsl_syntax.intseq2str(tokens[i])
                        next_tokens_with_counts = self.dsl_syntax.get_next_candidates(p_str)
                        next_tokens = [t[0] for t in next_tokens_with_counts
                                       if t[1] <= self.max_program_len - len(tokens[i])]
                    except:
                        # TODO: this code rarely cause syntax error, which
                        # should not happen. We should fix this in the future.
                        next_tokens = [t for t in range(len(self.dsl_syntax.int2token))]
                    else:
                        next_tokens = [self.dsl_syntax.token2int[t] for t in next_tokens]
                mask = np.zeros([outputs.shape[1]], dtype=outputs.dtype)
                for t in next_tokens:
                    mask[t] = 1
                masks.append(mask)
            return np.stack(masks, axis=0)
        masks = tf.py_func(mask_output, [outputs, self._end_token], tf.float32)
        masks.set_shape(outputs.get_shape())
        masked_outputs = tf.exp(outputs) * masks
        sample_ids = math_ops.cast(
            math_ops.argmax(masked_outputs, axis=-1), dtypes.int32)

        def add_sample_ids(sample_ids):
            self.previous_tokens.append(sample_ids)
            return sample_ids
        new_sample_ids = tf.py_func(add_sample_ids, [sample_ids], tf.int32)
        new_sample_ids.set_shape(sample_ids.get_shape())
        return new_sample_ids

    def next_inputs(self, time, outputs, state, sample_ids, name=None):
        """next_inputs_fn for GreedyEmbeddingHelper."""
        del time, outputs  # unused by next_inputs_fn
        finished = math_ops.equal(sample_ids, self._end_token)
        all_finished = math_ops.reduce_all(finished)
        next_inputs = control_flow_ops.cond(
            all_finished,
            # If we're finished, the next_inputs value doesn't matter
            lambda: self._start_inputs,
            lambda: self._embedding_fn(sample_ids))
        return (finished, next_inputs, state)

class SyntacticSampleEmbeddingHelper(SyntacticGreedyEmbeddingHelper):
    """A helper for use during inference.

    Uses sampling (from a distribution) instead of argmax and passes the
    result through an embedding layer to get the next input.
    """

    def __init__(self, dsl_syntax, max_program_len, embedding, start_tokens, end_token, seed=None):
        """Initializer.

        Args:
            dsl_syntax: Syntax checker for generating next possible tokens.
            max_program_len: maximum program length
            embedding: A callable that takes a vector tensor of `ids` (argmax ids),
                or the `params` argument for `embedding_lookup`. The returned tensor
                will be passed to the decoder input.
            start_tokens: `int32` vector shaped `[batch_size]`, the start tokens.
            end_token: `int32` scalar, the token that marks end of decoding.
            seed: The sampling seed.

        Raises:
            ValueError: if `start_tokens` is not a 1D tensor or `end_token` is not a
                scalar.
        """
        super(SyntacticSampleEmbeddingHelper, self).__init__(
            dsl_syntax, max_program_len, embedding, start_tokens, end_token)
        self._seed = seed

    def sample(self, time, outputs, state, name=None):
        """sample for SyntacticGreedyEmbeddingHelper."""
        del time, state  # unused by sample_fn
        # Outputs are logits, use argmax to get the most probable id
        if not isinstance(outputs, ops.Tensor):
            raise TypeError("Expected outputs to be a single Tensor, got: %s" %
                            type(outputs))

        # Mask outputs to reduce candidates to syntatically correct ones.
        def mask_output(outputs, end_token):
            if len(self.previous_tokens) == 0:  # when there is no previous token, skip masking.
                mask = np.zeros(outputs.shape, dtype=outputs.dtype)
                mask[:, self.dsl_syntax.token2int['DEF']] = 1
                return mask
            tokens = np.stack(self.previous_tokens, axis=1)
            masks = []
            for i in range(outputs.shape[0]):
                if tokens[i][-1] == end_token:
                    next_tokens = [end_token]
                else:
                    try:
                        p_str = self.dsl_syntax.intseq2str(tokens[i])
                        next_tokens_with_counts = self.dsl_syntax.get_next_candidates(
                            '{}'.format(p_str))
                        next_tokens = [t[0] for t in next_tokens_with_counts
                                       if t[1] <= self.max_program_len - len(tokens[i])]
                    except:
                        # TODO: this code rarely cause syntax error, which
                        # should not happen. We should fix this in the future.
                        next_tokens = [t for t in range(len(self.dsl_syntax.int2token))]
                    else:
                        next_tokens = [self.dsl_syntax.token2int[t] for t in next_tokens]
                mask = np.zeros([outputs.shape[1]], dtype=outputs.dtype)
                for t in next_tokens:
                    mask[t] = 1
                masks.append(mask)
            return np.stack(masks, axis=0)
        masks = tf.py_func(mask_output, [outputs, self._end_token], tf.float32)
        masks.set_shape(outputs.get_shape())
        masked_outputs = tf.exp(outputs) * masks
        masked_probs = masked_outputs / \
            tf.reduce_sum(masked_outputs, axis=1, keep_dims=True)
        sample_id_sampler = categorical.Categorical(probs=masked_probs)
        sample_ids = sample_id_sampler.sample(seed=self._seed)

        def add_sample_ids(sample_ids, masked_probs, masks):
            self.previous_tokens.append(sample_ids)
            self.previous_probs.append(masked_probs)
            self.previous_masks.append(masks)

            return sample_ids
        new_sample_ids = tf.py_func(add_sample_ids, [sample_ids, masked_probs, masks], tf.int32)
        new_sample_ids.set_shape(sample_ids.get_shape())
        return new_sample_ids