#!/usr/bin/python3

import asyncio
import aiohttp
import logging
import sys
import time
from typing import Optional

from headset import Headset
from mockloop import mock_loop

# initialise logging to file
import logger

PSI_URL = 'http://observer:8888/sync'
# people do not often perceive difference in delays shorter than 40ms
DELAY_THRESHOLD = 0.035

async def fetch_sync() -> Optional[float]:
    try:
        async with aiohttp.ClientSession() as sess:
            async with sess.get(PSI_URL) as resp:
                r = await resp.json()
                return r
    except:
        logging.info("Exception in fetching psi")
        return None

async def loop(leds: Headset, period: float, rand: float) -> None:
    """
    This function uses a generator defined below in the tick() function to call
    the Headset function that makes the lights blink with a period given by the
    period parameter

    It uses a global parameter rand to mock an external parameter that reduces
    the randomness incurred in the blinking
    """
    def tick():
        t = time.time()
        while True:
            t += period
            yield max(t - time.time(), 0)

    gen = tick()

    while rand > 0:
        sync = await fetch_sync()
        logging.info(f"Sync: {sync}")

        if sync is None:
            logging.info("Sync param was not fetched: entering mock synchronous loop")
            return 0

        rand = sync * leds.OFF_DELAY

        if rand > leds.OFF_DELAY:
            rand = leds.OFF_DELAY
        if rand < DELAY_THRESHOLD:
            rand = 0
        logging.info(f'Rand: {rand}')

        time.sleep(next(gen))
        logging.info(f'Tick')

        leds.crown_blink_wait(rand)

    if rand == 0:
        logging.info("Emergence suceeded! Entering rainbow loop")
        return 1

if __name__ == "__main__":

    server_type='player'
    if len(sys.argv) > 1:
        server_type = sys.argv[1]

    if server_type == 'player':
        from ws2801_headset import WS2801Headset

        leds = WS2801Headset((0, 0, 100), (0, 255, 0), 0.5, 1.5)
    elif server_type == 'local':
        leds = Headset((0, 0, 100), (0, 255, 0), 0.5, 1.5)

    print(f"Starting {server_type} experiment")

    rand   = leds.OFF_DELAY
    period = leds.OFF_DELAY + leds.ON_DELAY

    print(f"Rand {rand} and period {period}")

    ret = asyncio.run(loop(leds, period, rand))
    if ret:
        leds.crown_rainbow()
        leds.crown_rainbow()
        leds.all_off()
    else:
        mock_loop(leds, period, 0.5, 2)
        leds.crown_rainbow()
        leds.crown_rainbow()
        leds.all_off()


"""
[1,3,-1,-3,5,3,6,7]

Window position                Max
---------------               -----
[1  3  -1] -3  5  3  6  7       3
 1 [3  -1  -3] 5  3  6  7       3
 1  3 [-1  -3  5] 3  6  7       5
 1  3  -1 [-3  5  3] 6  7       5
 1  3  -1  -3 [5  3  6] 7       6
 1  3  -1  -3  5 [3  6  7]      7

"""
import collections
from typing import List

class Solution:
    def maxSlidingWindow(self, nums, k):
        d = collections.deque()
        out = []
        for i, n in enumerate(nums):
            while d and nums[d[-1]] < n:
                d.pop()
            d += i,
            if i >= k - 1:
                out += nums[d[0]],
        return out

print(Solution().maxSlidingWindow([1, 3, -1, -3, 5, 3, 6, 7], 3))

# public int[] maxSlidingWindow(int[] a, int k) {
# 		if (a == null || k <= 0) {
# 			return new int[0];
# 		}
# 		int n = a.length;
# 		int[] r = new int[n-k+1];
# 		int ri = 0;
# 		// store index
# 		Deque<Integer> q = new ArrayDeque<>();
# 		for (int i = 0; i < a.length; i++) {
# 			// remove numbers out of range k
# 			while (!q.isEmpty() && q.peek() < i - k + 1) {
# 				q.poll();
# 			}
# 			// remove smaller numbers in k range as they are useless
# 			while (!q.isEmpty() && a[q.peekLast()] < a[i]) {
# 				q.pollLast();
# 			}
# 			// q contains index... r contains content
# 			q.offer(i);
# 			if (i >= k - 1) {
# 				r[ri++] = a[q.peek()];
# 			}
# 		}
# 		return r;
# 	}


import sys
import os
import json
import shutil

import logging
logger = logging.getLogger()
logger.setLevel(logging.INFO)

def main():
    try:
        # コマンドライン引数の取得
        # - faq.jsonファイル相対パス
        # - contentフォルダ相対パス
        json_file = sys.argv[1]
        content_dir_path = sys.argv[2]

        # 入力ファイル(faq.json と テンプレート)読み込み
        json_data = json.load(open(json_file, 'r'))
        category_template = read_category_template()
        faq_template = read_faq_template()

        # 事前に対象コンテンツフォルダ(categories, faq)を全削除しておく
        remove_target_contents(content_dir_path)

        # faq.jsonの内容をもとに、category用mdやqa用mdを生成する
        dict_contents = {}
        for item in json_data:
            create_faq_md(content_dir_path, faq_template, item)
            if not item['カテゴリNo'] in dict_contents:
                dict_contents[item['カテゴリNo']] = item['カテゴリ名']
                create_category_md(content_dir_path, category_template, item)

    except Exception as e:
        logger.exception(e)
        raise e

# カテゴリ用mdの生成
def create_category_md(content_dir_path, category_template, item):
    target_category_md_path = os.path.join(content_dir_path, category_md_file_name(item))
    text = category_template
    text = text.replace('{{category}}', item['カテゴリ名'])
    text = text.replace('{{category-no}}', item['カテゴリNo'])
    text = text.replace('{{category-no-num}}', str(int(item['カテゴリNo'])))
    write_file(target_category_md_path, text)

# FAQ用mdの生成
def create_faq_md(content_dir_path, faq_template, item):
    target_faq_md_path = os.path.join(content_dir_path, faq_md_file_name(item))
    text = faq_template
    text = text.replace('{{question}}', item['質問'])
    text = text.replace('{{no}}', item['No'])
    text = text.replace('{{category-no}}', item['カテゴリNo'])
    text = text.replace('{{category-sub-no}}', item['カテゴリ内No'])
    write_file(target_faq_md_path, text)

# ファイル生成
def write_file(path, text):
    if not os.path.exists(os.path.dirname(path)):
        os.makedirs(os.path.dirname(path))

    with open(path, 'w+') as f:
        f.write(text)

# faq用mdファイル名取得
def faq_md_file_name(item):
    return 'faq/{0}/{1}-{2}.ja.md'.format(item['カテゴリNo'], item['カテゴリ内No'], item['No'])

# category用mdファイル名取得
def category_md_file_name(item):
    return 'categories/{0}.ja.md'.format(item['カテゴリNo'])

# 生成対象のコンテンツを削除する
# - content/categories/*
# - content/faq/*
def remove_target_contents(content_dir_path):
    remove_dir_all(os.path.join(content_dir_path, 'categories/'))
    remove_dir_all(os.path.join(content_dir_path, 'faq/'))

# フォルダ内全削除
def remove_dir_all(path):
    if os.path.exists(path):
        shutil.rmtree(path)

# faqテンプレートの読み込み
def read_faq_template():
    return read_template('template/faq.md')

# categoryテンプレートの読み込み
def read_category_template():
    return read_template('template/category.md')

# テンプレートの読み込み
def read_template(filename):
    with open(os.path.join(os.path.dirname(__file__), filename), 'r') as f:
        return f.read()

if __name__ == '__main__':
    main()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Created on Mon Mar 28 03:32:26 2022

The class of hand detector based on mediapipe

@author: Xinyang Chen
"""

import mediapipe as mp
import cv2 as cv

class HandDetector():
    def __init__(self, wid=640, hei=360, mode=False, maxHands=1, model_complexity=1, detectionCon=0.3, trackCon=0.3):
        self.mode = mode# hand tracking mode
        self.maxHands = maxHands# number of hands
        self.model_complexity = model_complexity# complexity of model
        self.detectionCon = detectionCon# detection confidence(palm detector)
        self.trackCon = trackCon# tracking confidence(landmark model)

        self.mpHands = mp.solutions.hands# hand tracking solution
        self.hands = self.mpHands.Hands(self.mode, self.maxHands, self.model_complexity, self.detectionCon, self.trackCon)
        self.mpDraw = mp.solutions.drawing_utils# draw
        self.wid = wid # width of the input image
        self.hei = hei # height of the input image
        self.results = None

    def findHands(self, img, drawOn=None, draw=False):
        """
        @brief: get the hand tracing based on mediapipe
        param img: the input image
        param drawOn: draw tracing result on drawOn
        param draw: enable drawing

        return success: the status of tracing
        return drawOn: output after drawing
        """
        imgRGB = cv.cvtColor(img, cv.COLOR_BGR2RGB)
        self.results = self.hands.process(imgRGB)
        success = False

        if self.results.multi_hand_landmarks:
            success = True
            for handLms in self.results.multi_hand_landmarks:
                if draw:
                    self.mpDraw.draw_landmarks(drawOn, handLms, self.mpHands.HAND_CONNECTIONS)

        return success

    def findPosition(self):
        """
        @brief: get the hand tracing based on mediapipe
        param wid: the width of the input image
        param hei: the height of the input image

        return lmList: the list of hand landmarks, elements are id, x position and y position
        """

        lmList = [] # list of the landmarks

        if self.results.multi_hand_landmarks:
            for handlms in self.results.multi_hand_landmarks:
                for id, lm in enumerate(handlms.landmark):
                    # x and y are normalized to [0.0, 1.0] by the image width and height respectively.
                    lmList.append([id, lm.x * self.wid, lm.y * self.hei])

        return lmList


import numpy as np
import logging
import json

from utils.run_net import evaluate

class Logger():
    def __init__(self, test_loaders, train_loaders,
                 num_tasks, n_cls, args):
        self.te_loaders = test_loaders
        self.tr_loaders = train_loaders
        self.gpu = args.gpu
        self.tasks = num_tasks
        self.n_cls = n_cls

    def evaluate_train(self, net):
        self.train_accs = []
        for task in range(self.tasks):
            task_met = evaluate(net, self.tr_loaders[task], self.gpu)
            self.train_accs.append((task_met[0].item(), task_met[1].item()))

    def log_metrics(self, net, train_met, ep, stdout=False):
        """
        Compute and loss train/test metrics. Slightly expensive
        so must be done infrequently during training (once every 10 epochs)
        """
        # Compute train/test metrics of learner.
        test_met = np.array([0.0, 0.0])
        self.evaluate_train(net)

        task_accs = []
        for task in range(self.tasks):
            task_met = evaluate(net, self.te_loaders[task], self.gpu)
            task_accs.append((task_met[0].item(), task_met[1].item()))
            test_met += task_met

        # Take average accuracy of all tasks
        # Doesn't give extra wt. to a task with more samples
        test_met = test_met / len(task_accs)

        def rnd(x):
            return [tuple(np.round(y, 3)) for y in x]

        info = {
            "Epoch": ep+1, "TrainAcc": train_met[0],
            "TestAcc": test_met[0], "TrainLoss": train_met[1],
            "TestLoss": test_met[1],
            "AllTestMetrics (acc,loss)": rnd(task_accs),
            "AllTrainMetrics (acc,loss)": rnd(self.train_accs)
        }
        logging.info(str(info))
        if stdout:
            taskacc = str(list(np.round(np.array(task_accs)[:, 0], 2)))
            print("Added learner with test accuracies: %s:" % taskacc)

        return test_met[0], test_met[1]

    def log_train(self, net, train_met, ep):
        """
        Log metrics obtained from training single epoch
        """
        info = {
            "Epoch": ep+1, "TrainAcc": train_met[0],
            "TrainLoss": train_met[1]
        }
        logging.info(json.dumps(info))


import random
import re

from flask import Blueprint, abort, request
from flask_restful import Api

from app.models.interest import MajorInterestModel, MinorInterestModel
from app.views import BaseResource

api = Api(Blueprint(__name__, __name__))

@api.resource('/interest-list')
class InterestList(BaseResource):
    def get(self):
        """
        관심사(스포츠 관련 카테고리) 리스트 반환
        """
        size = request.args['size']

        if not re.match('\d+', size):
            abort(400)

        size = int(size)

        major_interest_count = size // 3
        minor_interest_count = size - major_interest_count

        return self.unicode_safe_json_dumps([{
            'id': str(interest.id),
            'name': interest.name
        } for interest in random.shuffle(MajorInterestModel.objects[:major_interest_count] + MinorInterestModel.objects[:minor_interest_count])])

@api.resource('/interest-list/<keyword>')
class InterestListSearch(BaseResource):
    def get(self, keyword):
        """
        관심사 리스트 검색
        """
        if not keyword:
            abort(400)

        regex = re.compile('.*{}.*'.format(keyword))

        return self.unicode_safe_json_dumps([{
            'id': str(interest.id),
            'name': interest.name
        } for interest in random.shuffle(MajorInterestModel.objects(name=regex) + MinorInterestModel.objects(name=regex))])


from django.shortcuts import render_to_response, redirect
from django.views.decorators.csrf import ensure_csrf_cookie
from django.template import RequestContext
from django.contrib.auth.decorators import login_required
from django.utils.translation import ugettext_lazy as _
from django.core import serializers
from django.contrib.admin.views.decorators import staff_member_required
from django.db import IntegrityError

import apps.organizations.tz as tz
import apps.kanban.tasks as kanban_tasks
import apps.projects.tasks as projects_tasks
from apps.kanban.models import KanbanStat
from util import organizationOrNone, generateProjectSlug, generateProjectPrefix, duplicatePrefix
from apps.subscription.decorators import *
from apps.account.models import Account
import apps.github_integration.utils as github_utils

from apps.emailconfirmation.models import EmailAddress

import managers as managers
import apps.kanban.managers as kanban_managers

import signals as signals
from util import reduce_burndown_data
from util import getAllocationsForIteration

import portfolio_managers

import json
import requests
import mixpanel
import math
import logging
import time

from apps.projects.calculation import onDemandCalculateVelocity

logger = logging.getLogger(__name__)

from django.conf import settings

if "notification" in settings.INSTALLED_APPS:
    from notification import models as notification
else:
    notification = None

from apps.projects.access import *
from apps.projects.models import TimeEntry, FileJob, OfflineJob
from apps.projects.forms import *

from apps.organizations.models import Organization

from apps.classic import models as classic_models

import datetime

from django.contrib import messages
from story_views import handleAddStory

botNames=['googlebot',
            'slurp',
            'twiceler',
            'msnbot',
            'kaloogabot',
            'yodaobot',
            'baiduspider',
            'bingbot',
            'speedy spider',
            'dotbot']

@login_required
@ensure_csrf_cookie
def milestones(request, project_slug):
    project = get_object_or_404( Project, slug=project_slug )
    read_access_or_403(project, request.user)
    return render_to_response("projects/milestones.html", {
       "project": project,
       "organization": project.organization
    }, context_instance=RequestContext(request))

@login_required
def wait_job(request, job_id, template="projects/job_wait.html"):
    job = OfflineJob.objects.get(id=job_id)
    if job.owner != request.user:
        raise PermissionDenied()
    return render_to_response(template, {"job": job,
                                         "url": request.GET.get("redirect_url")},
                              context_instance=RequestContext(request))

@login_required
@ensure_csrf_cookie
def story_queue(request, project_slug):
    project = get_object_or_404( Project, slug=project_slug )
    write_access_or_403(project, request.user)
    return render_to_response("projects/story_queue.html", {
       "project": project,
       "organization": project.organization
    }, context_instance=RequestContext(request))

@login_required
def project_tips(request, organization_slug, project_slug=None):
    organization = get_object_or_404(Organization, slug=organization_slug)
    if not organization.hasReadAccess(request.user):
        raise PermissionDenied('You do not have access to that organization')

    if project_slug is None:
        project = None
    else:
        project = get_object_or_404( Project, slug=project_slug )

    return render_to_response("projects/tips.html", {
       "project": project,
       "organization": organization
    }, context_instance=RequestContext(request))

@login_required
@ensure_csrf_cookie
def classic_picker(request):
    return HttpResponseRedirect(reverse('home'))
    # # If a user.account.classic_mode == CLASSIC_FORCE_CLASSIC - go to www
    # # If a user.account.classic_mode == CLASSIC_FORCE_NEW - go to org picker
    # # If a user has no new projects and does have classic projects, redirect to classic WWW immiediately.
    # # If a user has no active classic projects and does have new projects, redirect to normal homepage
    # # If a user has both, display a page prompting which they would prefer to see
    # user = request.user
    # account = Account.objects.get(user=user)
    #
    # # if account.classic_mode == Account.CLASSIC_FORCE_CLASSIC:
    # #     return HttpResponseRedirect(settings.CLASSIC_SITE_URL)
    #
    # if account.classic_mode == Account.CLASSIC_FORCE_NEW:
    #     return HttpResponseRedirect(reverse('home'))
    #
    # newProjects = 0
    # classicProjects = 0
    # for team in Team.objects.filter(members=request.user, access_type='staff'):
    #     newProjects += Project.objects.filter(organization=team.organization, active=True).count()
    #     classicProjects += classic_models.Project.objects.filter(organization=team.organization, active=True).count()
    #
    # newProjects += Project.objects.filter(teams__members=request.user, active=True).count()
    # classicProjects += classic_models.Project.objects.filter(classic_teams__team__members=request.user, active=True).count()
    #
    # # if newProjects == 0 and classicProjects > 0:
    # #     return HttpResponseRedirect(settings.CLASSIC_SITE_URL)
    #
    # if newProjects > 0 and classicProjects == 0:
    #     return HttpResponseRedirect(reverse('home'))
    #
    # return render_to_response('classic_picker.html', context_instance=RequestContext(request) )

@login_required
@ensure_csrf_cookie
def test_page(request):
    organizations = Organization.getOrganizationsForUser(request.user)
    projects = getProjectsForUser(request.user, organizations[0])
    return render_to_response('test_page.html',{
       "my_organizations": organizations[0],
       "project": projects[1].slug
      }, context_instance=RequestContext(request) )

@login_required
@ensure_csrf_cookie
def home(request, template_name="unauthenticated_homepage_2014_09.html", force_org_view=False):
    fov = request.GET.get('force_org_view', False)

    organizations = Organization.getOrganizationsForUser(request.user)
    if not force_org_view and not fov and len(organizations) == 1:
        return HttpResponseRedirect(organizations[0].redirect_to_url())

    return render_to_response("homepage.html", {
       "my_organizations": organizations,
       "now": datetime.datetime.now()
      }, context_instance=RequestContext(request))

@staff_member_required
@ensure_csrf_cookie
def user_details_by_email(request, email):
    emails = EmailAddress.objects.filter(email__contains=email)

    users = [e.user for e in emails]
    return render_to_response("projects/user_list_details.html",
                              {
                                "users":users
                              },
                              context_instance=RequestContext(request))

@staff_member_required
@ensure_csrf_cookie
def user_details_staff(request, username):
    user = User.objects.get(username=username)
    teams = user.teams.all().order_by("organization__name")
    return render_to_response("projects/user_details.html",
                              {
                                "target_user":user,
                                "teams":teams
                              },
                              context_instance=RequestContext(request))

@login_required
@ensure_csrf_cookie
def planning(request, group_slug, sub_tool="iteration"):
    planningUrl = '{}#/planning/planningcolumn'.format(
                    reverse('project_app', kwargs={'project_slug':group_slug}))
    return HttpResponseRedirect(planningUrl)

@login_required
@ensure_csrf_cookie
def activate( request, group_slug, remove_from_portfolio = None):
    project = get_object_or_404( Project, slug=group_slug )
    admin_access_or_403(project, request.user, ignore_active=True)
    if project.portfolio_level is None and project.project_type != Project.PROJECT_TYPE_PORTFOLIO:
        # non-portfolio project
        project.active = True
        project.save()
    else:
        # protfolio project
        managers.activate_project(project, remove_from_portfolio)

    managers.rebuild_project_teams_cache(project)
    managers.rebuild_project_owner_cache(project.organization, project.creator)

    #clear newsitem cache
    managers.clearNewsItemsCache(project.organization, project.slug)

    return HttpResponseRedirect( reverse("project_detail", kwargs={"group_slug":project.slug} ) )

@login_required
@ensure_csrf_cookie
def activate_portfolio( request, group_slug ):
    project = get_object_or_404( Project, slug=group_slug )
    if project.project_type != 2:
        #got a portfolio mid level project
        project = project.portfolio_level.portfolio.root

    admin_access_or_403(project, request.user, ignore_active=True)
    managers.activate_project(project)
    managers.rebuild_project_teams_cache(project)
    managers.rebuild_project_owner_cache(project.organization, project.creator)

    #clear newsitem cache
    managers.clearNewsItemsCache(project.organization, project.slug)

    return HttpResponseRedirect( reverse("project_detail", kwargs={"group_slug":project.slug} ) )

# The project admin page, this is where you can change the title, description, etc. of a project.
@login_required
@ensure_csrf_cookie
def project_admin( request, group_slug ):
    project = get_object_or_404( Project, slug=group_slug )

    admin_access_or_403(project, request.user )

    form = ProjectOptionsForm(instance=project)
    logger.debug(request.POST)
    if request.method == 'POST': # If one of the three forms on the page has been submitted...
        if request.POST.get("action") == "rebuildReports":
            kanban_tasks.scheduleRebuildStepMovements(project, request.user)

        if request.POST.get("action") == "updateIndexes":
            projects_tasks.updateProjectIndexes.delay(project.slug)

        if request.POST.get("action") == "resetBurnup":
            points = project.points_log.all().order_by("-date")
            if len(points) > 0:
                project.burnup_reset = points[0].points_status10
                project.burnup_reset_date = tz.today(project.organization)
                project.save()
            #

        if request.POST.get("action") == "updateProject":
            form = ProjectOptionsForm( request.POST, instance=project)
            logger.debug("updateProject")
            if form.is_valid(): # All validation rules pass
                form.save()
                onDemandCalculateVelocity(project)
                # logger.debug("if form.is_valid() == True")
                #request.user.message_set.create(message="Project options Saved.")
                messages.add_message(request, messages.INFO, "Project options Saved.")
                return HttpResponseRedirect(reverse("project_detail",kwargs={'group_slug':project.slug}))
            else:
                logger.debug("if form.is_valid() == False"); logger.debug(form.errors);
        if request.POST.get("action") == "moveToOrganization":
            organization = get_object_or_404( Organization, id=request.POST.get("organization_id",""))
            user = project.creator
            org = project.organization
            if project.project_type == 2:
                portfolio_managers.move_portfolio_to_org(project, organization)
            else:
                if project.organization:
                    managers.remove_project_from_teams(project)
                project.organization = organization
                project.prefix = generateProjectPrefix(project)
                if duplicatePrefix(project, organization):
                    project.prefix = generateProjectPrefix(project)
                project.save()
                portfolio_managers.remove_project_from_portfolio(project)
                portfolio_managers.clear_story_releases(project)
            owners = organization.getOwnersGroup()
            if owners:
                owners.projects.add(project)
                # update owners project list cache
                managers.rebuild_project_teams_cache(project)
            #request.user.message_set.create(message="Project moved to organization")
            # update user project list cache
            managers.rebuild_staff_teams_cache(organization)
            managers.rebuild_project_owner_cache(org, user)

            #clear newsitem cache
            managers.clearNewsItemsCache(org, project.slug)
            managers.clearNewsItemsCache(organization, project.slug)

            messages.add_message(request, messages.INFO, "Project moved to organization")
            return HttpResponseRedirect(reverse("organization_detail",kwargs={'organization_slug':organization.slug}))

        if request.POST.get("action") == "archiveProject":
            managers.archive_project(project)
            # update user project teams cache
            managers.rebuild_project_teams_cache(project)
            # update user project list cache
            managers.rebuild_project_owner_cache(project.organization, project.creator)
            #clear newsitem cache
            managers.clearNewsItemsCache(project.organization, project.slug)
            return HttpResponseRedirect( reverse("project_detail", kwargs={"group_slug":project.slug} ) )

    return HttpResponseRedirect(reverse("project_detail",kwargs={'group_slug':project.slug}))

# Returns a JSON feed for a given project/iteration that can be transformed with some javascript
# into a burn up, burn down, or stacked chart.
@login_required
@ensure_csrf_cookie
def iteration_burndown(request, group_slug, iteration_id):
    project = get_object_or_404( Project, slug=group_slug )
    read_access_or_403(project, request.user )
    iteration = get_object_or_404( Iteration, id=iteration_id )
    try:
        if iteration.start_date:
            start_date_timestamp = int((time.mktime(iteration.start_date.timetuple()) - time.timezone)*1000)
        if iteration.end_date:
            end_date_timestamp = int((time.mktime(iteration.end_date.timetuple()) - time.timezone)*1000)
    except:
        # An invalid date, nothing more we can do.
        return HttpResponse("");

    has_startdate_data = False
    has_enddate_data = False
    highest_total_points = 0
    last_total_points = 0

    points = [[],[],[],[],[],[],[],[],[],[],[],[],[]]

    for log in iteration.points_log.all():

        if iteration.start_date:
            if log.timestamp() > start_date_timestamp and not has_startdate_data and start_date_timestamp != 0:
                # If the first day isn't filled out, give it 0's
                for i in range(13):
                    points[i].append( [start_date_timestamp, 0])

        has_startdate_data = True
        last_total_points = log.points_total
        if iteration.end_date:
            if log.timestamp() == end_date_timestamp :
                has_enddate_data = True
        if highest_total_points < log.points_total:
            highest_total_points = log.points_total
        points[0].append( [log.timestamp(), log.points_total] );
        points[1].append( [log.timestamp(), log.points_status1] );
        points[2].append( [log.timestamp(), log.points_status2] );
        points[3].append( [log.timestamp(), log.points_status3] );
        points[4].append( [log.timestamp(), log.points_status4] );
        points[5].append( [log.timestamp(), log.points_status5] );
        points[6].append( [log.timestamp(), log.points_status6] );
        points[7].append( [log.timestamp(), log.points_status7] );
        points[8].append( [log.timestamp(), log.points_status8] );
        points[9].append( [log.timestamp(), log.points_status9] );
        points[10].append( [log.timestamp(), log.points_status10] );
        points[11].append( [log.timestamp(), log.time_estimated] );
        points[12].append( [log.timestamp(), log.time_estimated_completed] );

    entries = TimeEntry.objects.filter(iteration=iteration).order_by("date")
    entries_series = []
    last_day = None
    last_count = 0
    for entry in entries:
        if entry.timestamp() != last_day:
            if last_day:
                entries_series.append([last_day, last_count]);
            last_day = entry.timestamp()
            # last_count = 0 don't reset so we get cumulative counts for the charts...
        last_count += entry.minutes_spent
    if last_day:
        entries_series.append([last_day, last_count]);

    allocations = getAllocationsForIteration(project, iteration)
    allocated_time = reduce(lambda total,allocation:total+allocation.minutes_allocated, allocations, 0)

    if allocated_time != 0:
        allocated_series = [[e[0],int((allocated_time-e[1])/60)] for e in entries_series]
    else:
        allocated_series = []

    entries_series = [[e[0], int(e[1]/60)] for e in entries_series]

    if not has_enddate_data and iteration.end_date:
        points[0].append( [end_date_timestamp, last_total_points])

    # TODO - filter out time entries not in date ranges.

    # Filter out stuff too early
    if iteration.start_date:
        for i in range(13):
            points[i] = filter(lambda x:x[0]>=start_date_timestamp, points[i])
            entries_series = filter(lambda x:x[0]>=start_date_timestamp, entries_series)

    # Filter out stuff too late
    if iteration.end_date:
        for i in range(13):
            points[i] = filter(lambda x:x[0]<=end_date_timestamp, points[i])
            entries_series = filter(lambda x:x[0]<=end_date_timestamp, entries_series)

    labels = project.statuses()
    labels = ["Total Points"] + list(labels) + ["Estimated Hours", "Completed"]

    # logger.debug( filter(lambda x:x[1]>0,points[11]))
    # logger.debug( filter(lambda x:x[1]>0,points[12]))

    # Convert minutes to hours
    points[11] = [ [d[0],d[1]/60] for d in points[11]]
    points[12] = [ [d[0],d[1]/60] for d in points[12]]

    data = [0] * 15
    for i in range(13):
        data[i] = {"label":labels[i], "data":reduce_burndown_data(points[i])}

    data[11]["hourseries"] = True
    data[12]["hourseries"] = True

    for i in range(1,11):
        data[i]["stack"] = True
        data[i]["statusType"] = i - 1

    # yaxis = 2 = the hourly axis
    data[11]["yaxis"] = 2;
    data[12]["yaxis"] = 2;
    data[13] = {"hourseries":True,"yaxis":2, "label":"Hours Spent", "data":entries_series, "lines":{ "show": True, "fill": False, "opacity":0.5 }, "points":{"radius":2}}
    data[14] = {"hourseries":True,"yaxis":2, "label":"Hours Left", "data":allocated_series, "lines":{ "show": True, "fill": False, "opacity":0.5 }, "points":{"radius":2}}

    json_serializer = serializers.get_serializer("json")()
    result = json.dumps(data)

    if highest_total_points == 0:
        return HttpResponse("[]")

    return HttpResponse(result) #, mimetype='application/json'

# Returns a JSON feed for a given project that can be transformed with some javascript
# into a burn up chart.
@login_required
@ensure_csrf_cookie
def project_burndown(request, group_slug):
    project = get_object_or_404( Project, slug=group_slug )
    read_access_or_403(project, request.user )
    total_points = [];
    claimed_points = [];
    if project.burnup_reset_date == None:
        points_log = project.points_log.all()
    else:
        points_log = project.points_log.filter(date__gte=project.burnup_reset_date)

    for log in points_log:
        total_points.append( [log.timestamp(), max(0,log.points_total - project.burnup_reset)] );
        claimed_points.append( [log.timestamp(), max(0,log.points_status10 - project.burnup_reset)] );

    total_stats = { "label":"Total Points", "data":reduce_burndown_data(total_points)}
    claimed_stats = { "label":"Claimed Points", "data":reduce_burndown_data(claimed_points)}

    json_serializer = serializers.get_serializer("json")()
    result = json.dumps([ total_stats , claimed_stats ])
    return HttpResponse(result) #, mimetype='application/json'

# Form and handler for creating a new project.
@login_required
@ensure_csrf_cookie
def create(request, group_slug, form_class=ProjectForm, template_name="projects/create.html", project_type="standard"):
    organization = get_object_or_404(Organization, slug=group_slug)

    personal = (project_type == "personal")

    if not (organization.hasStaffAccess(request.user) or personal):
        raise PermissionDenied()

    team_options = [(team.id, team.name) for team in organization.teams.all()]
    project_options = [(project.id, project.name) for project in organization.projects.all().filter(personal=False, project_type=1)]
    workflow_options = [('continuous', 'Continuous'), ('iterative', 'Iterative')]
    if project_options:
        project_options.insert(0,(0, 'None'))
    project_form = form_class(request.POST or None,
                                team_options=team_options,
                                project_options=project_options,
                                personal=personal,
                                workflow_options=workflow_options)

    admin_organizations = Organization.getOrganizationsForUser( request.user ) # The user can create projects in organizations the user is an admin in.
    if project_form.is_valid():
        project = managers.create_project(project_form.cleaned_data['name'],
                                   organization,
                                   request.user,
                                   personal,
                                   project_form.cleaned_data.get('Teams',None),
                                   Project.PROJECT_TYPE_KANBAN,
                                   project_form.cleaned_data.get('Workspace', None)
                                   )
        project_flow = project_form.cleaned_data.get('Workspace Flow', None)
        if project is not None and project_flow is not None and project_flow == 'continuous':
            kanban_managers.initContinousProject(project)

        if project is not None:
            mp = mixpanel.Mixpanel(settings.MIXPANEL_TOKEN)
            mp.track(organization.slug, 'Create Project')

            managers.clearNewsItemsCache(organization, project.slug)
            return HttpResponseRedirect(project.get_absolute_url())
        else:
            #request.user.message_set.create(message="This organization is over it's project limit.")
            messages.add_message(request, messages.INFO, "This organization is over it's project limit.")
            return HttpResponseRedirect( reverse("usage") )

    return render_to_response(template_name, {
        "project_form": project_form,
        "admin_organizations":admin_organizations,
        "organization":organization,
        "personal": personal
    }, context_instance=RequestContext(request))

@login_required
@ensure_csrf_cookie
def delete(request, group_slug=None, redirect_url=None):
    project = get_object_or_404(Project, slug=group_slug)
    project_slug = project.slug
    org = project.organization
    admin_access_or_403(project, request.user )
    if not redirect_url:
        redirect_url = reverse('home')

    managers.delete_project(project)

    #clear newsitem cache
    managers.clearNewsItemsCache(org, project_slug)
    #request.user.message_set.create(message=_("Project %(project_name)s deleted.") % {"project_name": project.name})
    messages.add_message(request, messages.INFO, _("Project %(project_name)s deleted.") % {"project_name": project.name})

    return HttpResponseRedirect(redirect_url)

@login_required
@expired_subscription_check
@ensure_csrf_cookie
def project_detail_picker(request, group_slug):
    """ The old project detail request, now just redirects to the board. """
    project = get_object_or_404(Project, slug=group_slug)
    read_access_or_403(project, request.user )
    return redirect("kanban_board",project_slug=group_slug)

@login_required
@expired_subscription_check
@ensure_csrf_cookie
def project(request, group_slug=None, form_class=ProjectUpdateForm,
        template_name="projects/project.html"):
    project = get_object_or_404(Project, slug=group_slug)
    read_access_or_403(project, request.user )
    if not request.user.is_authenticated():
        is_member = False
    else:
        is_member = project.user_is_member(request.user)

    action = request.POST.get("action")
    if action == "update":
        write_access_or_403(project, request.user )
        project_form = form_class(request.POST, instance=project)
        if project_form.is_valid():
            project = project_form.save()
    else:
        project_form = form_class(instance=project)

    add_story_form = handleAddStory(request, project)

    now = datetime.datetime.now()
    showIntro = (now - request.user.date_joined) < datetime.timedelta(days=3)
    kanban_stat = None

    if project.project_type == Project.PROJECT_TYPE_SCRUM:
        template_name = "projects/scrum_project.html"
    else:
        template_name = "projects/kanban_project.html"
        stats = KanbanStat.objects.filter(project=project).order_by("-created")[:1]
        logger.debug(stats)
        if len(stats) > 0:
            kanban_stat = stats[0]

    return render_to_response(template_name, {
        "project_form": project_form,
        "add_story_form": add_story_form,
        "project": project,
        "showIntro":showIntro,
        "organization": organizationOrNone(project),
        "group": project, # @@@ this should be the only context var for the project
        "is_member": is_member,
        "current_view":"project_page",
        "kanban_stat":kanban_stat
    }, context_instance=RequestContext(request))

# Drives the prediction page
# example: /projects/project/sch-r180-dashboards/project_prediction
@login_required
@ensure_csrf_cookie
def project_prediction(request, group_slug):
    project = get_object_or_404(Project, slug=group_slug)
    read_access_or_403(project, request.user )
    stories = project.get_default_iteration().stories.exclude(status=Story.STATUS_DONE).order_by("rank")

    # there is a form on the page with velocity, iteration length, and carry over.
    # IF they're filled out, we should override the default values.
    requested_velocity = request.GET.get("velocity","x");
    if requested_velocity.isdigit():
        velocity = int( requested_velocity )
    else:
        velocity = project.velocity
    if velocity == 0 or velocity == None:
        # A default velocity just in case this project doesn't have one yet.
        velocity = 25
    requested_length = request.GET.get("iteration_length","x")
    if requested_length.isdigit():
        iteration_length = int( requested_length )
    else:
        iteration_length = 14
    carry_over = (request.GET.get("carry_over","x") == "on")

    points_left = velocity
    temp_stories = []
    predictions = []
    iteration_number = project.iterations.count()
    start_date = datetime.date.today()
    current_iterations = project.get_current_iterations()
    unsized_stories = []
    total_points = 0

    # Find when the last scheduled iteration ends.
    for iteration in current_iterations:
        if iteration.end_date is not None and iteration.end_date > start_date:
            start_date = iteration.end_date

    total = 0
    for story in stories:
        total += story.points_value()

    if stories.count() > 0:
        average = int( request.GET.get('average',total / stories.count()) )
    else:
        average = 0

    # loop through the stories, and try to put them into a predicted iteration.
    for story in stories:
        if story.points == "?":
            points = average
        else:
            points = story.points_value()

        if points > points_left:
            record_prediction(predictions,temp_stories,iteration_number,start_date,iteration_length,points_left,average)
            iteration_number += 1
            if carry_over:
                points_left += velocity
            else:
                points_left = velocity
            temp_stories = []
            start_date += datetime.timedelta(days=iteration_length)
        if story.points == "?":
            unsized_stories.append(story)
        points_left -= int(points)
        total_points += points
        temp_stories.append(story)

    record_prediction(predictions,temp_stories,iteration_number,start_date,iteration_length,points_left,average)

    return render_to_response("projects/project_prediction.html", {
        "project": project,
        "predictions": predictions,
        "average": average,
        "velocity":velocity,
        "total_points":total_points,
        "ideal_iterations":int(math.ceil(total_points/velocity)),
        "predicted_iterations":len(predictions),
        "unsized_stories":unsized_stories,
        "carry_over": carry_over,
        "iteration_length":iteration_length,
        "iterations": project.iterations.all().order_by("-end_date")
    }, context_instance=RequestContext(request))

def record_prediction(predictions, stories,iteration_number,start_date,iteration_length,points_left, average):
    points = 0
    avgStories = 0
    for story in stories:
        if story.points == "?":
            p = average
            avgStories += 1
        else:
            p = story.points_value()
        points += p
    predictions.append( { "avgStories":avgStories, "carried":points_left, "stories":stories , "points":points, "num":iteration_number, "start":start_date, "end":(start_date +  datetime.timedelta(days=(iteration_length-1))) } )

@login_required
@ensure_csrf_cookie
def file_job_download(request, job_id, template="projects/job_download.html"):
    job = FileJob.objects.get(id=job_id)
    if job.owner != request.user:
        raise PermissionDenied()
    return render_to_response(template, { "job":job }, context_instance=RequestContext(request))

@login_required
def file_job_download_file(request, job_id):
    job = FileJob.objects.get(id=job_id)
    if job.owner != request.user:
        raise PermissionDenied()
    return HttpResponseRedirect( job.attachment_file.url )

@login_required
@ensure_csrf_cookie
def project_app(request, project_slug):
    project = get_object_or_404(Project, slug=project_slug)
    read_access_or_403(project, request.user)
    return render_to_response("projects/project.html",
                              {
                                  'project': project,
                                  'organization': project.organization,
                                  'can_write': has_write_access(project, request.user)
                              },
                              context_instance=RequestContext(request))


import os
from django.test import SimpleTestCase
from api.services import cra

class TestCraWrite(SimpleTestCase):
    # Test against cra in file 00007
    def test_write_in_00007(self):
        data = [
            {
                "sin": "270300379",
                "years": [2020],
                "given_name": "Alice",
                "family_name": "Solange",
                "birth_date": "19710122",
                "application_id": "1234",
            },
            {
                "sin": "302435839",
                "years": [2020],
                "given_name": "Wendy",
                "family_name": "Turner",
                "birth_date": "19780521",
                "application_id": "1234",
            },
            {
                "sin": "129922258",
                "years": [2020],
                "given_name": "Lily",
                "family_name": "Redding",
                "birth_date": "19830707",
                "application_id": "1234",
            },
        ]

        file_contents = cra.write(
            data,
            today="20220516",
            program_code="BCVR",
            cra_env="A",
            cra_sequence="00007",
        )

        cra_in_file_00007_filename = os.path.join(
            os.path.dirname(__file__),
            "cra_in_out",
            "in",
            "TO.ATO#@@00.R7005.IN.BCVR.A00007",
        )

        with open(cra_in_file_00007_filename, "r") as cra_in_file_00007:
            file = cra_in_file_00007.read()

            # Check file contents are exactly equal.
            self.assertEqual(file_contents, file)

    # Test against cra in file 00008
    def test_write_in_00008(self):
        data = [
            {
                "sin": "280671421",
                "years": [2020],
                "given_name": "Nancy",
                "family_name": "Durham",
                "birth_date": "19650217",
                "application_id": "ctW8gU57YX4xfQ9o",
            },
            {
                "sin": "270300379",
                "years": [2020],
                "given_name": "Alice",
                "family_name": "Solange",
                "birth_date": "19710122",
                "application_id": "9uXLvNQS5vkKnscD",
            },
            {
                "sin": "549713485",
                "years": [2020],
                "given_name": "Patty",
                "family_name": "Sullivan",
                "birth_date": "19940529",
                "application_id": "B5t92XeH7NnFUwxc",
            },
            {
                "sin": "302435839",
                "years": [2020],
                "given_name": "Wendy",
                "family_name": "Turner",
                "birth_date": "19780521",
                "application_id": "B5t92XeH7NnFUwxc",
            },
        ]

        file_contents = cra.write(
            data,
            today="20220517",
            program_code="BCVR",
            cra_env="A",
            cra_sequence="00008",
        )

        cra_in_file_00008_filename = os.path.join(
            os.path.dirname(__file__),
            "cra_in_out",
            "in",
            "TO.ATO#@@00.R7005.IN.BCVR.A00008",
        )

        with open(cra_in_file_00008_filename, "r") as cra_in_file_00008:
            file = cra_in_file_00008.read()

            # Check file contents are exactly equal.
            self.assertEqual(file_contents, file)

    # Test against cra in file 00009
    def test_write_in_00009(self):
        data = [
            {
                "sin": "270300379",
                "years": [2020],
                "given_name": "AliceAliceAliceAliceAliceAliceTruncateMe",
                "family_name": "SolangeSolangeSolangeSolangeSolange",
                "birth_date": "19710122",
                "application_id": "1234",
            },
            {
                "sin": "302435839",
                "years": [2020],
                "given_name": "Wendy",
                "family_name": "Turner",
                "birth_date": "19780521",
                "application_id": "1234",
            },
            {
                "sin": "129922258",
                "years": [2020],
                "given_name": "Lily",
                "family_name": "Redding",
                "birth_date": "19830707",
                "application_id": "1234",
            },
        ]

        file_contents = cra.write(
            data,
            today="20220516",
            program_code="BCVR",
            cra_env="A",
            cra_sequence="00009",
        )

        cra_in_file_00009_filename = os.path.join(
            os.path.dirname(__file__),
            "cra_in_out",
            "in",
            "TO.ATO#@@00.R7005.IN.BCVR.A00009",
        )

        with open(cra_in_file_00009_filename, "r") as cra_in_file_00009:
            file = cra_in_file_00009.read()

            # Check file contents are exactly equal.
            self.assertEqual(file_contents, file)


# from gevent import monkey
# monkey.patch_all()

from py2neo import neo4j, cypher, node, rel
from bs4 import BeautifulSoup
from celery import Celery
import requests
#import Queue
import urlparse
import logging

logger = logging.getLogger(__name__)
celery = Celery()
celery.config_from_object('celeryconfig')
db = neo4j.GraphDatabaseService("http://localhost:7474/db/data/")

SCHEME = 'http://'
SITE = 'www.test.com'

PAGE_INDEX_NAME = 'page'
PAGE_INDEX_KEY = 'url'
LINK_RELATIONSHIP_TYPE = 'LINKS_TO'

page_index = db.get_or_create_index(neo4j.Node, PAGE_INDEX_NAME)

@celery.task(name='scan.process_page')
def process_page(url):
    #url = node.get_properties()[PAGE_INDEX_KEY]
    node, = page_index.get(PAGE_INDEX_KEY, url)

    if been_processed(node):
        return

    logger.info('Retrieving Link: %s' % url)

    try:
        response = requests.head(SCHEME + url, allow_redirects=False)
        # TODO: allow_redirects and use response.history to create path in db
    except requests.RequestException as e:
        node.update_properties({
                'status': 'EXCEPTION',
                'content': repr(e)
                })
        return

    content_type = response.headers.get('content-type')

    node.update_properties({
            'status': response.status_code,
            'content-type': content_type
            })

    if not 'text/html' in content_type:
        return

    if not is_internal_path(url):
        logger.info('External Page: %s' % url)
        return

    # Get the full content
    try:
        response = requests.get(SCHEME + url, allow_redirects=False)
        # TODO: allow_redirects and use response.history to create path in db
    except requests.RequestException as e:
        return

    # We've got something, try to parse the response for any links
    soup = BeautifulSoup(response.content)
    a_tags = soup.find_all('a')

    for tag in a_tags:
        if tag.attrs.has_key('href'):
            process_link.delay(url, tag.attrs['href'])

@celery.task(name='scan.process_link')
def process_link(origin_node_url, link_path):
    '''Create a new node in the db for this path if we don't already
    know about it, and add it to the queue to be processed
    '''

    logger.info('Processing Link: %s' % link_path)
    origin_node, = page_index.get(PAGE_INDEX_KEY, origin_node_url)

    url = make_canonical_url(
        origin_node_url,#origin_node.get_properties()[PAGE_INDEX_KEY],
        link_path
        )

    # If we don't already have the target for this link in the db,
    # let's add it
    target_node = get_or_create_indexed_page_node(url)

    # Create the relationship if it doesn't exist
    path, = origin_node.get_or_create_path(LINK_RELATIONSHIP_TYPE, target_node)

    # If this node has previously been processed don't need to do
    # anything more)
    if been_processed(target_node):
        return

    # Add the new node to queue to be processed
    # queue.put(url)
    process_page.delay(url)

def been_processed(node):
    return node.get_properties().has_key('status')

def is_internal_path(url):
    full_url = SCHEME + url
    parsed_url = urlparse.urlparse(full_url)
    return parsed_url.netloc.lower() == SITE.lower()

def make_canonical_url(origin_url, link_path):
    joined = urlparse.urljoin(SCHEME + origin_url, link_path)
    parsed = urlparse.urlparse(joined)
    return parsed.netloc + parsed.path + parsed.query

def get_or_create_indexed_page_node(url):
    return db.get_or_create_indexed_node(
        PAGE_INDEX_NAME,
        PAGE_INDEX_KEY,
        url,
        properties={
            PAGE_INDEX_KEY : url
            })

# def push_initial_node():
#     root = get_or_create_indexed_page_node(SITE + '/')
    # queue.put(root.properties()[PAGE_INDEX_KEY])

# def process_queue():
#     while not queue.empty():
#         process_page(queue.get())

def run():
    base_url = SITE + '/'
    get_or_create_indexed_page_node(base_url)
    process_page.delay(base_url)
    #process_queue()

def logging_test():
    logger.debug('debug')
    logger.info('info')
    logger.warn('warn')
    logger.error('error')


# -*- encoding: utf-8 -*-

import numpy as np
import six
import warnings

import autosklearn.automl
from autosklearn.constants import *
from autosklearn.util.backend import create
from sklearn.base import BaseEstimator
import sklearn.utils
import scipy.sparse

class AutoMLDecorator(object):

    def __init__(self, automl):
        self._automl = automl

    def fit(self, *args, **kwargs):
        self._automl.fit(*args, **kwargs)

    def refit(self, X, y):
        """Refit all models found with fit to new data.

        Necessary when using cross-validation. During training, auto-sklearn
        fits each model k times on the dataset, but does not keep any trained
        model and can therefore not be used to predict for new data points.
        This methods fits all models found during a call to fit on the data
        given. This method may also be used together with holdout to avoid
        only using 66% of the training data to fit the final model.

        Parameters
        ----------

        X : array-like or sparse matrix of shape = [n_samples, n_features]
            The training input samples.

        y : array-like, shape = [n_samples] or [n_samples, n_outputs]
            The targets.

        Returns
        -------

        self

        """
        return self._automl.refit(X, y)

    def fit_ensemble(self, y, task=None, metric=None, precision='32',
                     dataset_name=None, ensemble_nbest=None,
                     ensemble_size=None):
        """Build the ensemble.

        This method only needs to be called in the parallel mode.

        Returns
        -------
        self
        """
        return self._automl.fit_ensemble(y, task, metric, precision,
                                         dataset_name, ensemble_nbest,
                                         ensemble_size)

    def predict(self, X):
        return self._automl.predict(X)

    def score(self, X, y):
        return self._automl.score(X, y)

    def show_models(self):
        """Return a representation of the final ensemble found by auto-sklearn

        Returns
        -------
        str
        """
        return self._automl.show_models()

    @property
    def grid_scores_(self):
        return self._automl.grid_scores_

    @property
    def cv_results_(self):
        return self._automl.cv_results_

    @property
    def trajectory_(self):
        return self._automl.trajectory_

    def sprint_statistics(self):
        return self._automl.sprint_statistics()

class AutoSklearnEstimator(AutoMLDecorator, BaseEstimator):

    def __init__(self,
                 time_left_for_this_task=3600,
                 per_run_time_limit=360,
                 initial_configurations_via_metalearning=25,
                 ensemble_size=50,
                 ensemble_nbest=50,
                 seed=1,
                 ml_memory_limit=3072,
                 include_estimators=None,
                 exclude_estimators=None,
                 include_preprocessors=None,
                 exclude_preprocessors=None,
                 resampling_strategy='holdout',
                 resampling_strategy_arguments=None,
                 tmp_folder=None,
                 output_folder=None,
                 delete_tmp_folder_after_terminate=True,
                 delete_output_folder_after_terminate=True,
                 shared_mode=False,
                 disable_evaluator_output=False,
                 configuration_mode='SMAC'):
        """
        Parameters
        ----------
        time_left_for_this_task : int, optional (default=3600)
            Time limit in seconds for the search of appropriate
            models. By increasing this value, *auto-sklearn* has a higher
            chance of finding better models.

        per_run_time_limit : int, optional (default=360)
            Time limit for a single call to the machine learning model.
            Model fitting will be terminated if the machine learning
            algorithm runs over the time limit. Set this value high enough so
            that typical machine learning algorithms can be fit on the
            training data.

        initial_configurations_via_metalearning : int, optional (default=25)
            Initialize the hyperparameter optimization algorithm with this
            many configurations which worked well on previously seen
            datasets. Disable if the hyperparameter optimization algorithm
            should start from scratch.

        ensemble_size : int, optional (default=50)
            Number of models added to the ensemble built by `Ensemble
            selection from libraries of models. Models are drawn with
            replacement.

        ensemble_nbest : int, optional (default=50)
            Only consider the ``ensemble_nbest`` models when building an
            ensemble. Implements `Model Library Pruning` from `Getting the
            most out of ensemble selection`.

        seed : int, optional (default=1)

        ml_memory_limit : int, optional (3072)
            Memory limit in MB for the machine learning algorithm.
            `auto-sklearn` will stop fitting the machine learning algorithm if
            it tries to allocate more than `ml_memory_limit` MB.

        include_estimators : list, optional (None)
            If None, all possible estimators are used. Otherwise specifies
            set of estimators to use.

        exclude_estimators : list, optional (None)
            If None, all possible estimators are used. Otherwise specifies
            set of estimators not to use. Incompatible with include_estimators.

        include_preprocessors : list, optional (None)
            If None all possible preprocessors are used. Otherwise specifies set
            of preprocessors to use.

        exclude_preprocessors : list, optional (None)
            If None all possible preprocessors are used. Otherwise specifies set
            of preprocessors not to use. Incompatible with
            include_preprocessors.

        resampling_strategy : string, optional ('holdout')
            how to to handle overfitting, might need 'resampling_strategy_arguments'

            * 'holdout': 66:33 (train:test) split
            * 'holdout-iterative-fit':  66:33 (train:test) split, calls iterative
              fit where possible
            * 'cv': crossvalidation, requires 'folds'

        resampling_strategy_arguments : dict, optional if 'holdout' (None)
            Additional arguments for resampling_strategy
            * 'holdout': None
            * 'holdout-iterative-fit':  None
            * 'cv': {'folds': int}

        tmp_folder : string, optional (None)
            folder to store configuration output and log files, if ``None``
            automatically use ``/tmp/autosklearn_tmp_$pid_$random_number``

        output_folder : string, optional (None)
            folder to store predictions for optional test set, if ``None``
            automatically use ``/tmp/autosklearn_output_$pid_$random_number``

        delete_tmp_folder_after_terminate: string, optional (True)
            remove tmp_folder, when finished. If tmp_folder is None
            tmp_dir will always be deleted

        delete_output_folder_after_terminate: bool, optional (True)
            remove output_folder, when finished. If output_folder is None
            output_dir will always be deleted

        shared_mode: bool, optional (False)
            Run smac in shared-model-node. This only works if arguments
            ``tmp_folder`` and ``output_folder`` are given and both
            ``delete_tmp_folder_after_terminate`` and
            ``delete_output_folder_after_terminate`` are set to False.

        disable_evaluator_output: bool, optional (False)
            Disable model and prediction output. Cannot be used together with
            ensemble building. predict() cannot be used when setting this
            flag to True.

        Attributes
        ----------
        grid_scores\_ : list of named tuples
            Contains scores for all parameter combinations in param_grid.
            Each entry corresponds to one parameter setting.
            Each named tuple has the attributes:

            * ``parameters``, a dict of parameter settings
            * ``mean_validation_score``, the mean score over the
              cross-validation folds
            * ``cv_validation_scores``, the list of scores for each fold

        cv_results\_ : dict of numpy (masked) ndarrays
            A dict with keys as column headers and values as columns, that can be
            imported into a pandas ``DataFrame``.

            This attribute is a backward port to already support the advanced
            output of scikit-learn 0.18. Not all keys returned by scikit-learn
            are supported yet.

        """
        self.time_left_for_this_task = time_left_for_this_task
        self.per_run_time_limit = per_run_time_limit
        self.initial_configurations_via_metalearning = initial_configurations_via_metalearning
        self.ensemble_size = ensemble_size
        self.ensemble_nbest = ensemble_nbest
        self.seed = seed
        self.ml_memory_limit = ml_memory_limit
        self.include_estimators = include_estimators
        self.exclude_estimators = exclude_estimators
        self.include_preprocessors = include_preprocessors
        self.exclude_preprocessors = exclude_preprocessors
        self.resampling_strategy = resampling_strategy
        self.resampling_strategy_arguments = resampling_strategy_arguments
        self.tmp_folder = tmp_folder
        self.output_folder = output_folder
        self.delete_tmp_folder_after_terminate = delete_tmp_folder_after_terminate
        self.delete_output_folder_after_terminate = delete_output_folder_after_terminate
        self.shared_mode = shared_mode
        self.disable_evaluator_output = disable_evaluator_output
        self.configuration_mode = configuration_mode
        super(AutoSklearnEstimator, self).__init__(None)

    def build_automl(self):
        if self.shared_mode:
            self.delete_output_folder_after_terminate = False
            self.delete_tmp_folder_after_terminate = False
            if self.tmp_folder is None:
                raise ValueError("If shared_mode == True tmp_folder must not "
                                 "be None.")
            if self.output_folder is None:
                raise ValueError("If shared_mode == True output_folder must "
                                 "not be None.")

        backend = create(temporary_directory=self.tmp_folder,
                         output_directory=self.output_folder,
                         delete_tmp_folder_after_terminate=self.delete_tmp_folder_after_terminate,
                         delete_output_folder_after_terminate=self.delete_output_folder_after_terminate)
        automl = autosklearn.automl.AutoML(
            backend=backend,
            time_left_for_this_task=self.time_left_for_this_task,
            per_run_time_limit=self.per_run_time_limit,
            log_dir=backend.temporary_directory,
            initial_configurations_via_metalearning=
            self.initial_configurations_via_metalearning,
            ensemble_size=self.ensemble_size,
            ensemble_nbest=self.ensemble_nbest,
            seed=self.seed,
            ml_memory_limit=self.ml_memory_limit,
            include_estimators=self.include_estimators,
            exclude_estimators=self.exclude_estimators,
            include_preprocessors=self.include_preprocessors,
            exclude_preprocessors=self.exclude_preprocessors,
            resampling_strategy=self.resampling_strategy,
            resampling_strategy_arguments=self.resampling_strategy_arguments,
            delete_tmp_folder_after_terminate=self.delete_tmp_folder_after_terminate,
            delete_output_folder_after_terminate=
            self.delete_output_folder_after_terminate,
            shared_mode=self.shared_mode,
            configuration_mode=self.configuration_mode,
            disable_evaluator_output=self.disable_evaluator_output)

        return automl

    def fit(self, *args, **kwargs):
        self._automl = self.build_automl()
        super(AutoSklearnEstimator, self).fit(*args, **kwargs)

    def fit_ensemble(self, y, task=None, metric=None, precision='32',
                     dataset_name=None, ensemble_nbest=None,
                     ensemble_size=None):
        if self._automl is None:
            self._automl = self.build_automl()
        return self._automl.fit_ensemble(y, task, metric, precision,
                                         dataset_name, ensemble_nbest,
                                         ensemble_size)

class AutoSklearnClassifier(AutoSklearnEstimator):
    """
    This class implements the classification task.
    """

    def build_automl(self):
        automl = super(AutoSklearnClassifier, self).build_automl()
        return AutoMLClassifier(automl)

    def fit(self, X, y,
            metric='acc_metric',
            feat_type=None,
            dataset_name=None):
        """Fit *auto-sklearn* to given training set (X, y).

        Parameters
        ----------

        X : array-like or sparse matrix of shape = [n_samples, n_features]
            The training input samples.

        y : array-like, shape = [n_samples] or [n_samples, n_outputs]
            The target classes.

        metric : str, optional (default='acc_metric')
            The metric to optimize for. Can be one of: ['acc_metric',
            'auc_metric', 'bac_metric', 'f1_metric', 'pac_metric']. A
            description of the metrics can be found in `the paper describing
            the AutoML Challenge.
        feat_type : list, optional (default=None)
            List of str of `len(X.shape[1])` describing the attribute type.
            Possible types are `Categorical` and `Numerical`. `Categorical`
            attributes will be automatically One-Hot encoded.

        dataset_name : str, optional (default=None)
            Create nicer output. If None, a string will be determined by the
            md5 hash of the dataset.

        Returns
        -------
        self

        """
        return super(AutoSklearnClassifier, self).fit(X=X, y=y, metric=metric,
                                                      feat_type=feat_type,
                                                      dataset_name=dataset_name)

    def predict(self, X):
        """Predict classes for X.

        Parameters
        ----------
        X : array-like or sparse matrix of shape = [n_samples, n_features]

        Returns
        -------
        y : array of shape = [n_samples] or [n_samples, n_labels]
            The predicted classes.

        """
        return super(AutoSklearnClassifier, self).predict(X)

    def predict_proba(self, X):

        """Predict probabilities of classes for all samples X.

        Parameters
        ----------
        X : array-like or sparse matrix of shape = [n_samples, n_features]

        Returns
        -------
        y : array of shape = [n_samples, n_classes] or [n_samples, n_labels]
            The predicted class probabilities.
        """
        return self._automl.predict_proba(X)

class AutoSklearnRegressor(AutoSklearnEstimator):
    """
    This class implements the regression task.
    """

    def build_automl(self):
        automl = super(AutoSklearnRegressor, self).build_automl()
        return AutoMLRegressor(automl)

    def fit(self, X, y,
            metric='r2_metric',
            feat_type=None,
            dataset_name=None):
        """Fit *autosklearn* to given training set (X, y).

        Parameters
        ----------

        X : array-like or sparse matrix of shape = [n_samples, n_features]
            The training input samples.

        y : array-like, shape = [n_samples] or [n_samples, n_outputs]
            The regression target.

        metric : str, optional (default='r2_metric')
            The metric to optimize for. Can be one of: ['r2_metric',
            'a_metric']. A description of the metrics can be found in
            `the paper describing the AutoML Challenge
            <http://www.causality.inf.ethz.ch/AutoML/automl_ijcnn15.pdf>`_.

        feat_type : list, optional (default=None)
            List of str of `len(X.shape[1])` describing the attribute type.
            Possible types are `Categorical` and `Numerical`. `Categorical`
            attributes will be automatically One-Hot encoded.

        dataset_name : str, optional (default=None)
            Create nicer output. If None, a string will be determined by the
            md5 hash of the dataset.

        Returns
        -------
        self

        """
        # Fit is supposed to be idempotent!
        # But not if we use share_mode.
        return super(AutoSklearnRegressor, self).fit(X=X, y=y, metric=metric,
                                                     feat_type=feat_type,
                                                     dataset_name=dataset_name)

    def predict(self, X):
        """Predict regression target for X.

        Parameters
        ----------
        X : array-like or sparse matrix of shape = [n_samples, n_features]

        Returns
        -------
        y : array of shape = [n_samples] or [n_samples, n_outputs]
            The predicted values.

        """
        return super(AutoSklearnRegressor, self).predict(X)

class AutoMLClassifier(AutoMLDecorator):

    def __init__(self, automl):
        self._classes = []
        self._n_classes = []
        self._n_outputs = 0

        super(AutoMLClassifier, self).__init__(automl)

    def fit(self, X, y,
            metric='acc_metric',
            loss=None,
            feat_type=None,
            dataset_name=None,
            ):
        # From sklearn.tree.DecisionTreeClassifier
        X = sklearn.utils.check_array(X, accept_sparse="csr",
                                      force_all_finite=False)
        if scipy.sparse.issparse(X):
            X.sort_indices()

        y = self._process_target_classes(y)

        if self._n_outputs > 1:
            task = MULTILABEL_CLASSIFICATION
        else:
            if len(self._classes[0]) == 2:
                task = BINARY_CLASSIFICATION
            else:
                task = MULTICLASS_CLASSIFICATION

        return self._automl.fit(X, y, task, metric, feat_type, dataset_name)

    def fit_ensemble(self, y, task=None, metric=None, precision='32',
                     dataset_name=None, ensemble_nbest=None,
                     ensemble_size=None):
        self._process_target_classes(y)
        return self._automl.fit_ensemble(y, task, metric, precision, dataset_name,
                                         ensemble_nbest, ensemble_size)

    def _process_target_classes(self, y):
        y = np.atleast_1d(y)
        if y.ndim == 2 and y.shape[1] == 1:
            warnings.warn("A column-vector y was passed when a 1d array was"
                          " expected. Please change the shape of y to "
                          "(n_samples,), for example using ravel().",
                          sklearn.utils.DataConversionWarning, stacklevel=2)

        if y.ndim == 1:
            # reshape is necessary to preserve the data contiguity against vs
            # [:, np.newaxis] that does not.
            y = np.reshape(y, (-1, 1))

        self._n_outputs = y.shape[1]

        y = np.copy(y)

        self._classes = []
        self._n_classes = []

        for k in six.moves.range(self._n_outputs):
            classes_k, y[:, k] = np.unique(y[:, k], return_inverse=True)
            self._classes.append(classes_k)
            self._n_classes.append(classes_k.shape[0])

        self._n_classes = np.array(self._n_classes, dtype=np.int)

        # TODO: fix metafeatures calculation to allow this!
        if y.shape[1] == 1:
            y = y.flatten()

        return y

    def predict(self, X):
        predicted_probabilities = self._automl.predict(X)
        if self._n_outputs == 1:
            predicted_indexes = np.argmax(predicted_probabilities, axis=1)
            predicted_classes = self._classes[0].take(predicted_indexes)

            return predicted_classes
        else:
            argmax_v = np.vectorize(np.argmax, otypes=[int])
            predicted_indexes = argmax_v(predicted_probabilities)
            #predicted_indexes = np.argmax(predicted_probabilities, axis=1)
            n_samples = predicted_probabilities.shape[0]
            predicted_classes = np.zeros((n_samples, self._n_outputs), dtype=object)

            for k in six.moves.range(self._n_outputs):
                output_predicted_indexes = predicted_indexes[:, k].reshape(-1)
                predicted_classes[:, k] = self._classes[k].take(output_predicted_indexes)

            return predicted_classes

    def predict_proba(self, X):
        return self._automl.predict(X)

class AutoMLRegressor(AutoMLDecorator):

    def fit(self, X, y,
            metric='r2_metric',
            loss=None,
            feat_type=None,
            dataset_name=None,
            ):
        return self._automl.fit(X=X, y=y, task=REGRESSION, metric=metric,
                                feat_type=feat_type, dataset_name=dataset_name)


import os
import os.path as osp
import xml.etree.ElementTree as ET
from tqdm import tqdm

'''
    来源
    分辨率
    x1, y1, x2, y2, x3, y3, x4, y4, 类别, 难度
'''

gsd_dict = {'GF2/GF3': 0.8}

labels = []

def fair1m2dota(data_folder):
    print('-- start fair1m2dota --')
    xml_folder = osp.join(data_folder, 'labelXml')
    txt_folder = osp.join(data_folder, 'labelTxt')  # 保存
    if not osp.exists(txt_folder):
        os.mkdir(txt_folder)
    xml_names = os.listdir(xml_folder)
    for xml_name in tqdm(xml_names):
        xml_path = osp.join(xml_folder, xml_name)
        txt_path = osp.join(txt_folder, xml_name.replace('.xml', '.txt'))
        # 写txt
        with open(txt_path, 'w') as f:
            tree = ET.parse(xml_path)
            root = tree.getroot()
            imagesource = root.find('source').find('origin').text
            f.write('imagesource:' + imagesource + '\n')  # 写来源
            f.write('gsd:' + str(gsd_dict[imagesource]) + '\n')  # 写分辨率
            for obj in root.find('objects').findall('object'):
                clas = obj.find('possibleresult').find('name').text.replace(' ', '-')
                if clas not in labels:
                    labels.append(clas)
                points = obj.find('points')
                x_list = []
                y_list = []
                for point in points.findall('point'):
                    xy = point.text.split(',')
                    x = float(xy[0])
                    y = float(xy[1])
                    x_list.append(x)
                    y_list.append(y)
                point_str = points_str(x_list, y_list)
                f.write(point_str)  # 写坐标
                f.write(clas + ' 0\n')  # 写类别
    return labels

def points_str(x_list, y_list):
    tmp = 0
    for i in range(5):
        for j in range(i, 5, 1):
            if x_list[i] == x_list[j] and y_list[i] == y_list[j]:
               tmp = j
               break
    del x_list[tmp]
    del y_list[tmp]
    point_str = ''
    for x, y in zip(x_list, y_list):
        point_str += str(x)
        point_str += ' '
        point_str += str(y)
        point_str += ' '

    return point_str

if __name__ == '__main__':
    data_path = r"D:\WorkSpace\NerualNet\dataset\FAIR1M2.0\validation"
    fair1m2dota(data_path)
    print(labels)


import argparse
import math
import torch

import marching_cubes
import numpy as np
import open3d as o3d
import trimesh

from network import ImplicitNet

# os.environ["DISPLAY"] = ":0"
# from postprocess import remove_backface

class MeshExtractor(object):
    def __init__(self,  model, voxels, latents, voxel_size,
                 rotations=None, centroids=None, bit=8, interp=False):
        self.model = model
        self.voxels = voxels
        self.latents = latents
        self.voxel_size = voxel_size
        self.rotations = None
        self.centroids = None

        self.resolution = bit
        self.interp = interp
        if interp:
            self.start = -1.
            self.end = 1. - 2./bit
            self.resolution *= 2
        else:
            self.start = -.5
            self.end = .5 - 1./bit
            self.resolution -= 1

        if rotations is not None:
            self.rotations = rotations.to(latents.device).float()
            self.centroids = centroids.to(latents.device).float()

    @ torch.no_grad()
    def get_sdf(self):
        x = y = z = torch.linspace(self.start, self.end, self.resolution)
        xx, yy, zz = torch.meshgrid(x, y, z)
        samples = torch.stack([xx, yy, zz], dim=-1).float()
        samples = samples.reshape(
            1, -1, 3).expand(self.voxels.shape[0], -1, -1)

        if self.centroids is not None:
            samples = samples - self.centroids.unsqueeze(1)
        if self.rotations is not None:
            samples = samples @ self.rotations.transpose(-1, -2)

        latent = self.latents.unsqueeze(1).expand(samples.shape[:-1]+(-1,))
        inputs = torch.cat([samples, latent], dim=-1)
        inputs = inputs.reshape(inputs.shape[0]*inputs.shape[1], -1)

        chunk_size = 10000
        sdf = torch.cat([self.model(inputs[i: i + chunk_size, :].cuda()).squeeze().detach().cpu()
                        for i in range(0, inputs.size(0), chunk_size)], 0)
        sdf_grid = sdf.reshape(-1, self.resolution,
                               self.resolution, self.resolution)
        return sdf_grid

    @ torch.no_grad()
    def linearize_id(self, xyz, n_xyz):
        return xyz[:, 2] + n_xyz[-1] * xyz[:, 1] + (n_xyz[-1] * n_xyz[-2]) * xyz[:, 0]

    @torch.no_grad()
    def create_mesh_overlay(self):
        sdf = self.get_sdf()

        from skimage.measure import marching_cubes
        verts_all = []
        faces_all = []
        num_verts = 0
        for i in range(len(voxels)):
            isdf = sdf[i].detach().cpu().numpy()
            if np.max(isdf) < 0 or np.min(isdf) > 0:
                continue
            results = marching_cubes(isdf, 0)
            if results is not None:
                verts, faces, _, _ = results
                verts /= (self.resolution-1)
                verts -= 0.5
                verts *= self.voxel_size
                verts += voxels[i].numpy()
                faces_all += [faces+num_verts]
                verts_all += [verts]
                num_verts += len(verts)
        verts = np.concatenate(verts_all)
        faces = np.concatenate(faces_all)

        if len(verts) != 0:
            mesh = trimesh.Trimesh(verts, faces).as_open3d
            mesh.compute_vertex_normals()
            o3d.visualization.draw_geometries([mesh])
        else:
            print("failed extracting surface!")

    @ torch.no_grad()
    def create_mesh(self, output=None):
        sdf = self.get_sdf()
        voxels = self.voxels / self.voxel_size
        min_bound = torch.floor(torch.amin(voxels, dim=0)).int()
        max_bound = torch.ceil(torch.amax(voxels, dim=0)).int()
        n_xyz = (max_bound - min_bound + 1).numpy().tolist()
        indexer = torch.ones((math.prod(n_xyz),),
                             device=sdf.device, dtype=torch.long) * -1
        vec_id_batch_mapping = torch.arange(voxels.shape[0]).int()
        voxels = torch.floor(voxels) - min_bound

        latent_vecs_pos = self.linearize_id(voxels, n_xyz).long()
        indexer[latent_vecs_pos] = vec_id_batch_mapping.long()
        std = torch.ones_like(sdf)
        indexer = indexer.view(n_xyz)
        sdf = -sdf

        if self.interp:
            verts, _, _ = marching_cubes.marching_cubes_sparse_interp(
                indexer.cuda(), latent_vecs_pos.cuda(), vec_id_batch_mapping.cuda(),
                sdf.cuda(), std.cuda(), n_xyz, 2000.0, int(2e7)
            )
        else:
            verts, _, _ = marching_cubes.marching_cubes(
                indexer.cuda(), latent_vecs_pos.cuda(), vec_id_batch_mapping.cuda(),
                sdf.cuda(), std.cuda(), n_xyz, 2000.0, int(2e7)
            )

        if len(verts) != 0:
            print(verts.shape)
            verts = verts.cpu().numpy()
            bound_min = min_bound.numpy()
            verts = verts + bound_min
            verts *= voxel_size
            verts = verts.reshape(-1, 3)
            faces = np.arange(verts.shape[0]).reshape(-1, 3)
            mesh = trimesh.Trimesh(verts, faces).as_open3d
            mesh.compute_vertex_normals()

            if output is not None:
                o3d.io.write_triangle_mesh(output, mesh)
            else:
                o3d.visualization.draw_geometries([mesh])
        else:
            print("failed extracting surface!")

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('data')
    parser.add_argument('--output', type=str, default=None)
    parser.add_argument('--naive', action='store_true')
    parser.add_argument('--interp', action='store_true')
    parser.add_argument('--res', type=int, default=8)
    args = parser.parse_args()

    data = torch.load(args.data)
    latents = data['latents']
    voxels = data['voxels']
    ckpt = data['model']
    cfg = data["config"]
    rotations = data['rotations']
    centroids = data['centroids']
    voxel_size = data['voxel_size']

    model = ImplicitNet(**cfg)
    model.load_state_dict(ckpt)
    model.cuda()
    model.eval()

    extractor = MeshExtractor(
        model, voxels, latents,
        voxel_size, rotations,
        centroids, bit=args.res,
        interp=args.interp)
    if args.naive:
        extractor.create_mesh_overlay()
    else:
        extractor.create_mesh(args.output)


import time

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By

ser_obj = Service("/home/prashant/Documents/Browser_Driver/chrome_102/chromedriver")
driver = webdriver.Chrome(service=ser_obj)

url = "https://demo.nopcommerce.com/register"
driver.get(url)
driver.maximize_window()

# Conditional Commands
# is_displayed() and is_enabled()
search_box = driver.find_element(By.XPATH, "//input[@id='small-searchterms']")
print("Display Status: ", search_box.is_displayed())  # True
print("Enable Status: ", search_box.is_enabled())  # True

# is_selected() - used for radio_buttons and check_boxes
rd_male = driver.find_element(By.XPATH, "//input[@id='gender-male']")
rd_female = driver.find_element(By.XPATH, "//input[@id='gender-female']")

print(rd_male.is_selected())  # False
print(rd_female.is_selected())  # False

rd_male.click()  # selected radio button
time.sleep(3)
print("After Selecting male radio button....")
print(rd_male.is_selected())  # True
print(rd_female.is_selected())  # False

rd_female.click()  # selected radio button
time.sleep(3)
print("After Selecting female radio button....")
print(rd_male.is_selected())  # False
print(rd_female.is_selected())  # True

driver.quit()


#!/usr/bin/env python
# coding: utf-8

# ### 11 Cats_and_dogs_transfer_learning

# In[1]:

# cats_and_dogs classification model with CNN
# train : 2000 images [cat(1000) + dog(1000)]  , size는 다름
# validation : 1000 images [cat(500) + dog(500)] , size는 다름

# In[2]:

import tensorflow as tf
import urllib
import zipfile
from tensorflow.keras.optimizers import RMSprop
from tensorflow.keras.preprocessing.image import ImageDataGenerator

from tensorflow.keras import layers
from tensorflow.keras import Model

# In[3]:

# Google InceptionV3 pretrained model 가중치 다운로드
_INCEPTION_URL = "https://storage.googleapis.com/mledu-datasets/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5"
urllib.request.urlretrieve(_INCEPTION_URL, 'tmp/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5')

# In[4]:

# Colab용 : Linux/GPU사용
# !wget --no-check-certificate \
#     https://storage.googleapis.com/mledu-datasets/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5 \
#     -O /tmp/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5

# In[5]:

# Inception model(pre-trained model) 불러오기

from tensorflow.keras.applications.inception_v3 import InceptionV3

locals_weights_file = 'tmp/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5'
pre_trained_model = InceptionV3(input_shape=(150,150,3),
                               include_top=False,
                               weights=None)

pre_trained_model.load_weights(locals_weights_file)

print(len(pre_trained_model.layers)) #311

# 사전 훈련 계층의 가중치의 훈련(학습) 가능 여부를 설정 : False(학습 안함) , 고정
for layer in pre_trained_model.layers:
    layer.trainable = False

pre_trained_model.summary()

# In[6]:

# 마지막 층 출력 확인하기(마지막 출력 층을 선택)
#  mixed7 (Concatenate)  :    (None, 7, 7, 768)
last_layer = pre_trained_model.get_layer('mixed7')
print('last layer output shape:', last_layer.output_shape)
last_output = last_layer.output

# In[7]:

# layer의 층 번호 구하기
# for i,layer in enumerate(pre_trained_model.layers): # 311회 반복
#     if(layer.name == 'mixed7'):
#         print(i+1)

# layer의 층 번호 구하기
pre_trained_model.layers.index(pre_trained_model.get_layer('mixed7')) + 1

# In[8]:

# 모델 구성  : pre_trained model layer(229)  + 4  --> 233 layers

# (1) Flatten layer
x = tf.keras.layers.Flatten()(last_output)

# (2) FC(=Dense) layer
x = layers.Dense(1024,activation='relu')(x)

# (3) Dropout layer
x = layers.Dropout(0.2)(x)

# (4) Output layer : 2진분류, sigmoid
x = layers.Dense(1,activation='sigmoid')(x)

model = Model(pre_trained_model.input,x)
model.compile(optimizer=RMSprop(learning_rate=0.0001), # 1e-4
             loss='binary_crossentropy',
             metrics=['accuracy'])
model.summary()

# In[9]:

# cats_and_dogs 데이터셋 다운로드 , Windows용
_TRAIN_URL = "https://storage.googleapis.com/mledu-datasets/cats_and_dogs_filtered.zip"
urllib.request.urlretrieve(_TRAIN_URL, 'tmp/cats_and_dogs_filtered.zip')

# In[10]:

#압축해제
local_zip = 'tmp/cats_and_dogs_filtered.zip'
zip_ref = zipfile.ZipFile(local_zip,'r')
zip_ref.extractall('tmp/')
zip_ref.close()

# In[11]:

# 이미지 데이터 경로 설정
import os

base_dir = 'tmp/cats_and_dogs_filtered'

train_dir = os.path.join(base_dir, 'train')
validation_dir = os.path.join(base_dir, 'validation')

# Directory with our training cat/dog pictures
train_cats_dir = os.path.join(train_dir, 'cats')
train_dogs_dir = os.path.join(train_dir, 'dogs')

# Directory with our validation cat/dog pictures
validation_cats_dir = os.path.join(validation_dir, 'cats')
validation_dogs_dir = os.path.join(validation_dir, 'dogs')
train_cat_fnames = os.listdir( train_cats_dir )
train_dog_fnames = os.listdir( train_dogs_dir )

print(train_cat_fnames[:10])
print(train_dog_fnames[:10])

print('total training cat images :', len(os.listdir(train_cats_dir ) ))
print('total training dog images :', len(os.listdir(train_dogs_dir ) ))

print('total validation cat images :', len(os.listdir( validation_cats_dir ) ))
print('total validation dog images :', len(os.listdir( validation_dogs_dir ) ))

# In[12]:

# 이미지 증강

train_datagen = ImageDataGenerator(rescale=1./255,
                                  rotation_range=40,
                                  width_shift_range=0.2,
                                  height_shift_range=0.2,
                                  shear_range=0.2,
                                  zoom_range=0.2,
                                  horizontal_flip=True)
#print(type(train_datagen))

validation_datagen = ImageDataGenerator(rescale=1./255,
                                  rotation_range=40,
                                  width_shift_range=0.2,
                                  height_shift_range=0.2,
                                  shear_range=0.2,
                                  zoom_range=0.2,
                                  horizontal_flip=True)

#train data의 generator
train_generator = train_datagen.flow_from_directory(
                  train_dir,
                  target_size=(150,150), # resize될 크기
                  batch_size=20,
                  class_mode='binary', # 2진 분류

)

#validation data의 generator
validation_generator = validation_datagen.flow_from_directory(
                  validation_dir,
                  target_size=(150,150), # resize될 크기
                  batch_size=20,
                  class_mode='binary', # 2진 분류

)

print(train_generator)

# In[13]:

# 학습
history = model.fit(
                train_generator, # x
                steps_per_epoch=100, # 2000/20 = 8.xxx, train image number/train batch_size
                epochs=20,
                validation_data=validation_generator,
                validation_steps=50 # 1000/20
)

# In[16]:

# 학습 결과 시각화
#-----------------------------------------------------------
# Retrieve a list of list results on training and test data
# sets for each training epoch
#-----------------------------------------------------------
acc      = history.history['accuracy' ]
val_acc  = history.history['val_accuracy' ]
loss     = history.history['loss' ]
val_loss = history.history['val_loss' ]

epochs   = range(len(acc)) # Get number of epochs

#------------------------------------------------
# Plot training and validation accuracy per epoch
#------------------------------------------------
plt.plot(epochs, acc, 'g', label='Training accuracy')
plt.plot(epochs, val_acc, 'r', label='Validation accuracy')
plt.title('Training and validation accuracy')
plt.legend()
plt.figure()

plt.plot(epochs, loss, 'g', label='Training Loss')
plt.plot(epochs, val_loss, 'r', label='Validation Loss')
plt.title('Training and validation loss')
plt.legend()

plt.show()

#  결과 : InceptionV3 pretrained model 사용 시 validation의 정확도가 크게 향상됨
#  val_accuracy: 0.7360 ---> val_accuracy: 0.9660

# In[ ]:


import torch
import time
import argparse
from model import fusion_refine,Discriminator
from train_dataset import dehaze_train_dataset
from test_dataset import dehaze_test_dataset
from val_dataset import dehaze_val_dataset
from torch.utils.data import DataLoader
import os
from torchvision.models import vgg16
from utils_test import to_psnr,to_ssim_skimage
from tensorboardX import SummaryWriter
import torch.nn.functional as F
from perceptual import LossNetwork
from torchvision.utils import save_image as imwrite
from pytorch_msssim import msssim
# --- Parse hyper-parameters train --- #
parser = argparse.ArgumentParser(description='RCAN-Dehaze-teacher')
parser.add_argument('-learning_rate', help='Set the learning rate', default=1e-4, type=float)
parser.add_argument('-train_batch_size', help='Set the training batch size', default=20, type=int)
parser.add_argument('-train_epoch', help='Set the training epoch', default=10000, type=int)
parser.add_argument('--train_dataset', type=str, default='')
parser.add_argument('--data_dir', type=str, default='')
parser.add_argument('--model_save_dir', type=str, default='./output_result')
parser.add_argument('--log_dir', type=str, default=None)
# --- Parse hyper-parameters test --- #
parser.add_argument('--test_dataset', type=str, default='')
parser.add_argument('--predict_result', type=str, default='./output_result/picture/')
parser.add_argument('-test_batch_size', help='Set the testing batch size', default=1,  type=int)
parser.add_argument('--vgg_model', default='', type=str, help='load trained model or not')
parser.add_argument('--imagenet_model', default='', type=str, help='load trained model or not')
parser.add_argument('--rcan_model', default='', type=str, help='load trained model or not')
args = parser.parse_args()
# --- train --- #
learning_rate = args.learning_rate
train_batch_size = args.train_batch_size
train_epoch= args.train_epoch
train_dataset=os.path.join(args.data_dir, 'train')

# --- test --- #
test_dataset = os.path.join(args.data_dir, 'test')
val_dataset = os.path.join(args.data_dir, 'val')
predict_result= args.predict_result
test_batch_size=args.test_batch_size

# --- output picture and check point --- #
if not os.path.exists(args.model_save_dir):
    os.makedirs(args.model_save_dir)
output_dir=os.path.join(args.model_save_dir,'output_result')

# --- Gpu device --- #
device_ids = [Id for Id in range(torch.cuda.device_count())]
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

# --- Define the network --- #
MyEnsembleNet = fusion_refine(args.imagenet_model, args.rcan_model)
print('MyEnsembleNet parameters:', sum(param.numel() for param in MyEnsembleNet.parameters()))
DNet = Discriminator()
print('# Discriminator parameters:', sum(param.numel() for param in DNet.parameters()))

# --- Build optimizer --- #
G_optimizer = torch.optim.Adam(MyEnsembleNet.parameters(), lr=0.0001)
scheduler_G = torch.optim.lr_scheduler.MultiStepLR(G_optimizer, milestones=[5000,7000,8000], gamma=0.5)
D_optim = torch.optim.Adam(DNet.parameters(), lr=0.0001)
scheduler_D = torch.optim.lr_scheduler.MultiStepLR(D_optim, milestones=[5000,7000,8000], gamma=0.5)
# --- Load training data --- #
dataset = dehaze_train_dataset(train_dataset)
train_loader = DataLoader(dataset=dataset, batch_size=train_batch_size, shuffle=True)
# --- Load testing data --- #
test_dataset = dehaze_test_dataset(test_dataset)
test_loader = DataLoader(dataset=test_dataset, batch_size=test_batch_size, shuffle=False, num_workers=0)

val_dataset = dehaze_val_dataset(val_dataset)
val_loader = DataLoader(dataset=val_dataset, batch_size=1, shuffle=False, num_workers=0)

# --- Multi-GPU --- #
MyEnsembleNet = MyEnsembleNet.to(device)
MyEnsembleNet= torch.nn.DataParallel(MyEnsembleNet, device_ids=device_ids)
DNet = DNet.to(device)
DNet= torch.nn.DataParallel(DNet, device_ids=device_ids)
writer = SummaryWriter(os.path.join(args.model_save_dir, 'tensorboard'))

# --- Define the perceptual loss network --- #
vgg_model = vgg16(pretrained=True)
# vgg_model.load_state_dict(torch.load(os.path.join(args.vgg_model , 'vgg16.pth')))
vgg_model = vgg_model.features[:16].to(device)
for param in vgg_model.parameters():
    param.requires_grad = False

loss_network = LossNetwork(vgg_model)
loss_network.eval()

msssim_loss = msssim

# '''server vgg'''
# vgg_model = vgg16(pretrained=True).features[:16]
# vgg_model = vgg_model.to(device)
# for param in vgg_model.parameters():
#     param.requires_grad = False
# loss_network = LossNetwork(vgg_model)
# loss_network.eval()

# --- Load the network weight --- #
try:
    MyEnsembleNet.load_state_dict(torch.load(os.path.join(args.teacher_model , 'epoch100000.pkl')))
    print('--- weight loaded ---')
except:
    print('--- no weight loaded ---')

# --- Strat training --- #
iteration = 0
for epoch in range(train_epoch):
    start_time = time.time()
    scheduler_G.step()
    scheduler_D.step()
    MyEnsembleNet.train()
    DNet.train()
    print(epoch)
    for batch_idx, (hazy, clean) in enumerate(train_loader):
        # print(batch_idx)
        iteration +=1
        hazy = hazy.to(device)
        clean = clean.to(device)
        output= MyEnsembleNet(hazy)

        DNet.zero_grad()
        real_out = DNet(clean).mean()
        fake_out = DNet(output).mean()
        D_loss = 1 - real_out + fake_out
        # no more forward
        D_loss.backward(retain_graph=True)
        MyEnsembleNet.zero_grad()
        adversarial_loss = torch.mean(1 - fake_out)
        smooth_loss_l1 = F.smooth_l1_loss(output, clean)
        perceptual_loss = loss_network(output, clean)
        msssim_loss_ = -msssim_loss(output, clean, normalize=True)
        total_loss = smooth_loss_l1+0.01 * perceptual_loss+ 0.0005 * adversarial_loss+ 0.5*msssim_loss_

        total_loss.backward()
        D_optim.step()
        G_optimizer.step()

#         if iteration % 2 == 0:
#             frame_debug = torch.cat(
#                 (hazy, output, clean), dim=0)
#             writer.add_images('train_debug_img', frame_debug, iteration)
        writer.add_scalars('training', {'training total loss': total_loss.item()
                                        }, iteration)
        writer.add_scalars('training_img', {'img loss_l1': smooth_loss_l1.item(),
                                                'perceptual': perceptual_loss.item(),
                                            'msssim':msssim_loss_.item()

                                               }, iteration)
        writer.add_scalars('GAN_training', {
            'd_loss':D_loss.item(),
            'd_score':real_out.item(),
            'g_score':fake_out.item()
            }, iteration
            )

    if epoch % 5 == 0:
        print('we are testing on epoch: ' + str(epoch))
        with torch.no_grad():
            psnr_list = []
            ssim_list = []
            recon_psnr_list = []
            recon_ssim_list = []
            MyEnsembleNet.eval()
            for batch_idx, (hazy_up,hazy_down,hazy, clean) in enumerate(test_loader):
                # hazy_up = hazy_up.to(device)
                # hazy_down=hazy_down.to(device)
                clean = clean.to(device)
                hazy = hazy.to(device)
                frame_out = MyEnsembleNet(hazy)
                # frame_out_up = MyEnsembleNet(hazy_up)
                # frame_out_down = MyEnsembleNet(hazy_down)
                # frame_out=(torch.cat([frame_out_up.permute(0,2,3,1), frame_out_down[:,:,80:640,:].permute(0,2,3,1)], 1)).permute(0,3,1,2)
                if not os.path.exists(output_dir):
                    os.makedirs(output_dir)
#                 imwrite(frame_out, output_dir +'/' +str(batch_idx) + '.png', range=(0, 1))

                psnr_list.extend(to_psnr(frame_out, clean))
                ssim_list.extend(to_ssim_skimage(frame_out, clean))

            avr_psnr = sum(psnr_list) / len(psnr_list)
            avr_ssim = sum(ssim_list) / len(ssim_list)
            print(epoch,'dehazed', avr_psnr, avr_ssim)
            frame_debug = torch.cat((frame_out,clean), dim =0)
            writer.add_images('my_image_batch', frame_debug, epoch)
            writer.add_scalars('testing', {'testing psnr':avr_psnr,
                'testing ssim': avr_ssim
                                    }, epoch)
            torch.save(MyEnsembleNet.state_dict(), os.path.join(args.model_save_dir,'epoch'+ str(epoch) + '.pkl'))

writer.close()