import time
from random import choice
from itertools import combinations

def zero_optimal(num_list):
    """
    Quadratic algorithm from

    https://en.wikipedia.org/wiki/3SUM
    """
    output = []
    num_list.sort()
    length = len(num_list)
    for i in range(length-2):
        a = num_list[i]
        start = i + 1
        end = length - 1
        while start < end:
            b = num_list[start]
            c = num_list[end]
            if a + b + c == 0:
                output.append((a, b, c))
                end -= 1
            elif a + b + c > 0:
                end -= 1
            else:
                start += 1

def zero_sum(num_list):
    """
    My initial solution
    """
    num_list.sort()
    solution = set()
    for i, val_i in enumerate(num_list[:-2]):
        for j in range(i+1, len(num_list) - 1):
            val_j = num_list[j]
            for k in range(j+1, len(num_list)):
                val_k = num_list[k]
                if val_i + val_j + val_k == 0:
                    solution.add((val_i, val_j, val_k))
    return solution

def zero_comb(num_list):
    """
    Another solution in Thread
    """
    return {tuple(sorted(n)) for n in combinations(num_list, 3) if sum(n) == 0}

methods = [zero_comb, zero_sum, zero_optimal]

with open('run_time.txt', 'w') as f:
    f.write('n comb loop quad\n')

cnt_limit = 5
for n in range(10, 300):
    print('Evaluating {}'.format(n))
    for cnt in range(5):
        vals = [choice([i for i in range(-100, 100)]) for r in range(n)]
        run_data = [n, 0, 0, 0]
        for i, method in enumerate(methods):
            num_list = [x for x in vals]
            start = time.time()
            solution = method(num_list)
            dur = time.time() - start
            run_data[i+1] += dur
    # calc average
    for i in range(1, 4):
        run_data[i] /= cnt_limit
    print(run_data)
    with open('run_time.txt', 'a') as f:
        f.write(' '.join([str(x) for x in run_data]))
        f.write('\n')


import requests

session = requests.Session()

def get_jobs():
    url = 'https://api.github.com/search/repositories?q=machine+learning+language:python'
    first_page = session.get(url).json()
    yield first_page
    num_pages = first_page['last_page']

    for page in range(2, num_pages + 1):
        next_page = session.get(url, params={'page': page}).json()
        yield next_page['page']

for page in get_jobs():

    np = page
    x = 0

x = 0


import pandas as pd
import pickle
import xgboost as xgb
import lightgbm as lgb
from catboost import CatBoostClassifier
import datetime
import glob
import numpy as np

def ensemble_models(df):
    """
        This Function is supposed to read all files in local dir with patter '*MODEL.pkl',
        load these models, append them in a vector, then avegare them.
    """

    # os.chdir("/mydir")
    y_pred = np.zeros(test.shape[0])  # Vector to be filled
    for file in glob.glob("*MODEL.pkl"):
        print(file)
        model = pickle.load(open(file, 'rb'))
        if 'LGB' in file: # Load LGB Models
            y_pred += np.array(model.predict(df))
        elif 'XGB' in file: # Load XGBoost Models
            y_pred += np.array(model.predict(xgb.DMatrix(df.values)))
        elif 'CatBoost' in file:
            y_pred += np.array(model.predict_proba(df)[:, 1])
    lenn = len(glob.glob("*MODEL.pkl"))
    print(f'len: {lenn}')
    y_pred /= len(glob.glob("*MODEL.pkl"))
    return y_pred

test = pd.read_pickle("test.pkl")
test_id = test['ID_code'].values
test.drop(['ID_code'], axis=1, inplace=True)
y_pred = ensemble_models(df=test)

submission = pd.DataFrame({"ID_code": test_id})
submission['target'] = y_pred
filename = "ENSEMBLE_" + datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + ".csv"
filename = filename.replace(':', '_')
submission.to_csv(filename, index=False)


import asyncio

class AsyncQueue(asyncio.Queue):
    """
    asyncio.Queue with a peek method added.
    """

    async def peek(self):
        while self.empty():
            getter = self._loop.create_future()
            self._getters.append(getter)
            try:
                await getter
            except BaseException:
                getter.cancel()  # Just in case getter is not done yet.
                try:
                    # Clean self._getters from canceled getters.
                    self._getters.remove(getter)
                except ValueError:
                    # The getter could be removed from self._getters by a
                    # previous put_nowait call.
                    pass
                if not self.empty() and not getter.cancelled():
                    # We were woken up by put_nowait(), but can't take
                    # the call.  Wake up the next in line.
                    self._wakeup_next(self._getters)
                raise
        return self.peek_nowait()

    def peek_nowait(self):
        if self.empty():
            raise asyncio.QueueEmpty
        item = self._peek()
        self._wakeup_next(self._putters)
        return item

    def _peek(self):
        return self._queue[0]


"""This module defines all functionality specific to Kvazaar."""

from __future__ import annotations

import os
import subprocess
from pathlib import Path
from typing import Iterable

import tester
import tester.core.git as git
import tester.core.cfg as cfg
import tester.core.test as test
from tester.core import vs, cmake
from tester.core.log import console_log
from . import EncoderBase

class Vvenc(EncoderBase):
    file_suffix = "vvc"

    class ParamSet(EncoderBase.ParamSet):
        def __init__(self,
                     quality_param_type: tester.QualityParam,
                     quality_param_value: int,
                     seek: int,
                     frames: int,
                     cl_args: str):

            super().__init__(
                quality_param_type,
                quality_param_value,
                seek,
                frames,
                cl_args
            )

            self._quality_formats[tester.QualityParam.QP] = "-qp "
            for t in range(tester.QualityParam.BITRATE.value, len(tester.QualityParam) + 1):
                self._quality_formats[tester.QualityParam(t)] = "--bitrate"

        def _to_unordered_args_list(self,
                                    include_quality_param: bool = True,
                                    include_seek: bool = True,
                                    include_frames: bool = True,
                                    include_directory_data: bool = False) -> list:
            args = self._cl_args

            if include_quality_param:
                args += " " + " ".join(self.get_quality_value(self.get_quality_param_value()))

            if self._seek:
                raise AssertionError("vvenc does not support seeking")
            if self._frames:
                raise AssertionError("vvenc does not support setting frame_count")

            if include_directory_data:
                if tester.Cfg().frame_step_size != 1:
                    args += f" --TemporalSubsampleRatio={tester.Cfg().frame_step_size}"
                args = args.replace("/", "-").replace("\\", "-").replace(":", "-")

            return args.split()

        @staticmethod
        def _get_arg_order() -> list:
            return []

    def __init__(self,
                 user_given_revision: str,
                 defines: Iterable,
                 use_prebuilt: bool):
        super().__init__(
            name="vvenc",
            user_given_revision=user_given_revision,
            defines=defines,
            git_local_path=tester.Cfg().tester_sources_dir_path / "vvenc",
            git_remote_url=tester.Cfg().vvenc_remote_url,
            use_prebuilt=use_prebuilt,
        )
        if tester.Cfg().system_os_name == "Windows":
            self._exe_src_path = self._git_local_path / "bin" / "release-static" / "vvencapp.exe"
        elif tester.Cfg().system_os_name == "Linux":
            self._exe_src_path = self._git_local_path / "bin" / "release-static" / "vvencapp"

        self._decoder_exe_path: Path = cfg.Cfg().vvc_reference_decoder

    def build(self) -> bool:
        if not self.build_start():
            return False

        env = None
        if not (self._git_local_path / "build").exists():
            (self._git_local_path / "build").mkdir()

        if tester.Cfg().system_os_name == "Windows":
            if self.get_defines():
                env = os.environ
                temp = " ".join([f"/D{x}".replace("=", "#") for x in self.get_defines()])
                env["CL"] = temp
            msbuild_args = vs.get_msbuild_args()

            build_cmd = (
                            "cd", str(self._git_local_path),
                            "&&", "cd", "build",
                            "&&", "cmake", "..",
                            "-G", cmake.get_cmake_build_system_generator(),
                            "-A", cmake.get_cmake_architecture(),
                            "&&", "call", str(vs.get_vsdevcmd_bat_path()),
                            "&&", "msbuild", "vvenc.sln",
                        ) + tuple(msbuild_args)

        elif tester.Cfg().system_os_name == "Linux":
            cflags_str = f"CFLAGS={''.join([f'-D{define} ' for define in self._defines])}".strip()

            build_cmd = (
                "cd", str(self._git_local_path),
                "&&", "make", cflags_str
            )

        else:
            raise RuntimeError("Invalid operating system")

        return self.build_finish(build_cmd, env)

    def clean(self) -> None:

        self.clean_start()

        clean_cmd = ()

        if tester.Cfg().system_os_name == "Linux":
            clean_cmd = (
                "cd", str(self._git_local_path),
                "&&", "make", "clean"
            )

        elif tester.Cfg().system_os_name == "Windows":
            msbuild_args = vs.get_msbuild_args(target="Clean")
            clean_cmd = (
                            "call", str(vs.get_vsdevcmd_bat_path()),
                            "&&", "msbuild", str(self._git_local_path / "build" / "vvenc.sln")
                        ) + tuple(msbuild_args)

        self.clean_finish(clean_cmd)

    def dummy_run(self,
                  param_set: EncoderBase.ParamSet, env) -> bool:
        self.dummy_run_start(param_set)

        RESOLUTION_PLACEHOLDER = "128x128"

        dummy_cmd = \
            (
                str(self._exe_path),
                "-i", os.devnull,
                "-s", RESOLUTION_PLACEHOLDER,
                "--FrameRate=30",
                "-o", os.devnull,
            ) + param_set.to_cmdline_tuple()

        return self.dummy_run_finish(dummy_cmd, param_set, env)

    @staticmethod
    def validate_config(test_config: test.Test):
        # Using the public property raises an exception, so access the private attribute instead.
        if not test_config.use_prebuilt and not git.git_remote_exists(tester.Cfg().vvenc_remote_url):
            console_log.error(f"VVenC: Remote '{tester.Cfg().vvenc_remote_url}' is not available")
            raise RuntimeError

        try:
            subprocess.call((cfg.Cfg().vvc_reference_decoder, ), )
        except FileNotFoundError:
            raise RuntimeError("VVenC: VVC reference decoder is needed for decoding VVC currently")

    def encode(self,
               encoding_run: test.EncodingRun) -> None:
        if not self.encode_start(encoding_run):
            return

        quality = encoding_run.param_set.get_quality_value(encoding_run.qp_value)

        encode_cmd = \
            (
                str(self._exe_path),
                "-i", str(encoding_run.input_sequence.get_filepath()),
                "-s", f"{encoding_run.input_sequence.get_width()}x{encoding_run.input_sequence.get_height()}",
                f"--FrameRate={encoding_run.input_sequence.get_framerate()}",
                f"--InputBitDepth={encoding_run.input_sequence.get_bit_depth()}",
                f"--InputChromaFormat={encoding_run.input_sequence.get_chroma()}",
                "-o", str(encoding_run.output_file.get_filepath()),
                "--frames", str(encoding_run.frames * tester.Cfg().frame_step_size),
            ) + encoding_run.param_set.to_cmdline_tuple(include_quality_param=False,
                                                        include_frames=False) + quality
        self.encode_finish(encode_cmd, encoding_run)

    def _decode(self,
                encoding_run: test.EncodingRun):

        decode_cmd = (
            str(self._decoder_exe_path),
            "-b", str(encoding_run.output_file.get_filepath()),
            "-o", str(encoding_run.decoded_output_file_path),
            "-d", str(encoding_run.input_sequence.get_bit_depth())
        )

        try:
            subprocess.check_output(
                decode_cmd,
                stderr=subprocess.STDOUT
            )
        except:
            console_log.error(f"VVenc: Failed to decode file "
                              f"'{encoding_run.output_file.get_filepath()}'")
            raise

class Vvencff(Vvenc):
    def __init__(self,
                 user_given_revision: str,
                 defines: Iterable,
                 use_prebuilt: bool):
        super(Vvenc, self).__init__(
            name="vvencff",
            user_given_revision=user_given_revision,
            defines=defines,
            git_local_path=tester.Cfg().tester_sources_dir_path / "vvenc",
            git_remote_url=tester.Cfg().vvenc_remote_url,
            use_prebuilt=use_prebuilt,
        )
        if tester.Cfg().system_os_name == "Windows":
            self._exe_src_path = self._git_local_path / "bin" / "release-static" / "vvencFFapp.exe"
        elif tester.Cfg().system_os_name == "Linux":
            self._exe_src_path = self._git_local_path / "bin" / "release-static" / "vvencFFapp"

        self._decoder_exe_path: Path = cfg.Cfg().vvc_reference_decoder

    def encode(self,
               encoding_run: test.EncodingRun) -> None:
        if not self.encode_start(encoding_run):
            return

        quality = encoding_run.param_set.get_quality_value(encoding_run.qp_value)

        encode_cmd = \
            (
                str(self._exe_path),
            ) + encoding_run.param_set.to_cmdline_tuple(include_quality_param=False,
                                                        include_frames=False) + (
                "-i", str(encoding_run.input_sequence.get_encode_path()),
                "-s", f"{encoding_run.input_sequence.get_width()}x{encoding_run.input_sequence.get_height()}",
                f"--FrameRate={encoding_run.input_sequence.get_framerate()}",
                f"--InputBitDepth={encoding_run.input_sequence.get_bit_depth()}",
                f"--InputChromaFormat={encoding_run.input_sequence.get_chroma()}",
                "-b", str(encoding_run.output_file.get_filepath()),
                "-f", str(encoding_run.frames * tester.Cfg().frame_step_size),
                "-o", os.devnull,
            ) + quality
        self.encode_finish(encode_cmd, encoding_run)

    class ParamSet(EncoderBase.ParamSet):
        def __init__(self,
                     quality_param_type: tester.QualityParam,
                     quality_param_value: int,
                     seek: int,
                     frames: int,
                     cl_args: str):

            super().__init__(
                quality_param_type,
                quality_param_value,
                seek,
                frames,
                cl_args
            )

            self._quality_formats[tester.QualityParam.QP] = "-q "
            for t in range(tester.QualityParam.BITRATE.value, len(tester.QualityParam) + 1):
                self._quality_formats[tester.QualityParam(t)] = "--RateControl=2 --TargetBitrate="

        def _to_unordered_args_list(self,
                                    include_quality_param: bool = True,
                                    include_seek: bool = True,
                                    include_frames: bool = True,
                                    include_directory_data: bool = False) -> list:
            args = self._cl_args

            if include_quality_param:
                args += " " + " ".join(self.get_quality_value(self.get_quality_param_value()))

            if include_seek and self._seek:
                args += f" -fs {self._seek}"
            if include_frames and self._frames:
                args += f" -f {self._frames}"

            if include_directory_data:
                if tester.Cfg().frame_step_size != 1 and "TemporalSubsampleRatio" not in args:
                    args += f" --TemporalSubsampleRatio={tester.Cfg().frame_step_size}"
                args = args.replace("/", "-").replace("\\", "-").replace(":", "-")

            return args.split()

        @staticmethod
        def _get_arg_order() -> list:
            return ["-c"]


from fastapi import FastAPI, BackgroundTasks
from fastapi.middleware.cors import CORSMiddleware

from .server.routers import gpus, price
from .server.tasks import crawler

app = FastAPI()

app.add_middleware(
    CORSMiddleware,
    allow_origins="*",
    allow_credentials=True,
    allow_methods=["GET"],
    allow_headers=["*"],
)

app.include_router(gpus.router)
app.include_router(price.router)

def start_crawling():
    crawler.start()

@app.get("/", tags=["root"])
def get_root(background_tasks: BackgroundTasks):
    background_tasks.add_task(start_crawling)
    return {"message": "This is a root"}


import gensim
from nltk.tokenize import RegexpTokenizer
from stop_words import get_stop_words
from nltk.stem.porter import PorterStemmer
from gensim import corpora, models
from nltk.corpus import stopwords
from nltk.stem.wordnet import WordNetLemmatizer
import string
import pyLDAvis.gensim
import pandas as pd

tokenizer = RegexpTokenizer(r'\w+')

# create English stop words list
en_stop = get_stop_words('en')

# Create p_stemmer of class PorterStemmer
p_stemmer = PorterStemmer()

# create sample documents
doc_a = "Brocolli is good to eat. My brother likes to eat good brocolli, but not my mother."
doc_b = "My mother spends a lot of time driving my brother around to baseball practice."
doc_c = "Some health experts suggest that driving may cause increased tension and blood pressure."
doc_d = "I often feel pressure to perform well at school, but my mother never seems to drive my brother to do better."
doc_e = "Health professionals say that brocolli is good for your health."

# compile sample documents into a list
doc_set = [doc_a, doc_b, doc_c, doc_d, doc_e]

with open("rt-polarity.txt")as fo:
    for rec in fo:
        doc_set.append(rec)

# list for tokenized documents in loop
texts = []

# loop through document list
for i in doc_set:
    # clean and tokenize document string

    raw = i.lower()
    tokens = tokenizer.tokenize(raw)

    # remove stop words from tokens
    stopped_tokens = [i for i in tokens if not i in en_stop]

    # stem tokens
    stemmed_tokens = [p_stemmer.stem(i) for i in stopped_tokens]

    # add tokens to list
    texts.append(stemmed_tokens)

# turn our tokenized documents into a id <-> term dictionary
dictionary = corpora.Dictionary(texts)

# convert tokenized documents into a document-term matrix
corpus = [dictionary.doc2bow(text) for text in texts]

# generate LDA model
ldamodel = gensim.models.ldamodel.LdaModel(corpus, num_topics=2, id2word=dictionary, passes=20)

stopwords = set(stopwords.words('english'))
punctuation = set(string.punctuation)
lemmatize = WordNetLemmatizer()

def cleaning(article):
    one = " ".join([i for i in article.lower().split() if i not in stopwords])
    two = "".join(i for i in one if i not in punctuation)
    three = " ".join(lemmatize.lemmatize(i) for i in two.split())
    return three

def pre_new(doc):
    one = cleaning(doc).split()
    two = dictionary.doc2bow(one)
    return two

print(ldamodel.print_topics(num_topics=2, num_words=4))
print(len(doc_set))

for i in ldamodel.print_topics():
    for j in i:
        print (j)

ldamodel.save('topic.model')
from gensim.models import LdaModel
loading = LdaModel.load('topic.model')
print(loading.print_topics(num_topics=2, num_words=4))

belong = loading[corpus[0]]
new = pd.DataFrame(belong,columns=['id','prob']).sort_values('prob',ascending=False)
new['topic'] = new['id'].apply(loading.print_topics)

#pyLDAvis.enable_notebook()
text_list = [k.split() for k in doc_set]
print(len(text_list))
print(text_list[0])

dictionary = corpora.Dictionary(text_list)
dictionary.save('dictionary.dict')
print(dictionary)

doc_term_matrix = [dictionary.doc2bow(doc) for doc in text_list]
corpora.MmCorpus.serialize('corpus.mm', doc_term_matrix)

print(len(doc_term_matrix))
print(doc_term_matrix[0])

d = gensim.corpora.Dictionary.load('dictionary.dict')
c = gensim.corpora.MmCorpus('corpus.mm')
lda = gensim.models.LdaModel.load('topic.model')

#data = pyLDAvis.gensim.prepare(lda, c, d)


import os
from dataclasses import FrozenInstanceError
from typing import Generator

import pytest

from pokeapi.search import config
from pokeapi.search.config import Singleton

def dynamic_connection_url() -> Generator[str, None, None]:
    if os.getenv("STAGE") == "development":
        yield "http://elasticsearch:9200"
    else:
        yield "http://localhost:9200"

class TestSingleton:
    class SingletonTestClass(metaclass=Singleton):
        pass

    def test_singleton(self) -> None:
        assert self.SingletonTestClass() is self.SingletonTestClass()

class TestConfig:
    @pytest.mark.usefixtures("_setup_get_config")
    @pytest.mark.parametrize("expected", dynamic_connection_url())
    def test_get_config(self, expected: str) -> None:
        actual = config.get_config()

        assert actual.ES_INDEX == "pokemon"
        assert actual.ES_CONNECTION_URL == expected

        actual_clone = config.get_config()
        assert actual is actual_clone

        with pytest.raises(FrozenInstanceError):
            actual.ES_INDEX = "hoge"  # type: ignore

        with pytest.raises(FrozenInstanceError):
            actual.ES_CONNECTION_URL = "hoge"  # type: ignore


from cProfile import label
from operator import index
from re import I
from tkinter import Label
from typing import Counter
import cv2
from sys import argv
import numpy as np
import matplotlib.pyplot as plt

mode = argv[1]

if mode == 'news':
	dir = 'news_out_modified/'
	amount = 1380
elif mode == 'ngc':
	dir = 'ngc_out_modified/'
	amount = 1060
elif mode == 'ftfm':
	dir = 'ftfm_out_modified/'
	amount = 770
else:
	print("Error input!")
	quit()

####################### Algorithm #######################

def evaluate(similarity1,similarity2,similarity3,similarity4,similarity5,weightedsim):
    # Evaluating the performance
    plt.figure("similarity - frame")
    plt.plot(similarity1, color='purple', label='Gray Hist')
    plt.plot(similarity2, color='yellow', label='RGB Hist')
    plt.plot(similarity3, color='red',  label='aHash')
    plt.plot(similarity4, color='green',  label='dHash')
    plt.plot(similarity5, color='blue',  label='pHash')
    plt.plot(weightedsim, color='aqua',  label='Weighted algorithm')
    plt.plot(thres,color='black', label='Threshold')
    plt.xlabel("frame")
    plt.ylabel("similarity")
    plt.legend(loc='best')
    plt.show()
    return

def create_gray_hist(image):
    #img = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    hist = cv2.calcHist([image], [0], None, [256],  [0, 256])
    return hist

def hist_compare(image1, image2):

    hist1 = create_gray_hist(image1)
    hist2 = create_gray_hist(image2)
    # 進行三種方式的直方圖比較
    match1 = cv2.compareHist(hist1, hist2, cv2.HISTCMP_BHATTACHARYYA)
    match2 = cv2.compareHist(hist1, hist2, cv2.HISTCMP_CORREL)
    similarity = ((1-match1) + match2)/2
    return similarity

def classify_hist_with_split(image1, image2, size=(256, 256)):
    # RGB每個通道的直方圖相似度
    # 將圖像resize後，分離爲RGB三個通道，再計算每個通道的相似值
    image1 = cv2.resize(image1, size)
    image2 = cv2.resize(image2, size)
    sub_image1 = cv2.split(image1)
    sub_image2 = cv2.split(image2)
    sub_data = 0
    for im1, im2 in zip(sub_image1, sub_image2):
        sub_data += hist_compare(im1, im2)
    sub_data = sub_data / 3
    return sub_data

def aHash(img):
    # 均值哈希算法
    # 縮放爲8*8
    img = cv2.resize(img, (8, 8))
    # 轉換爲灰度圖
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    # s爲像素和初值爲0，hash_str爲hash值初值爲''
    s = 0
    hash_str = ''
    # 遍歷累加求像素和
    for i in range(8):
        for j in range(8):
            s = s+gray[i, j]
    # 求平均灰度
    avg = s/64
    # 灰度大於平均值爲1相反爲0生成圖片的hash值
    for i in range(8):
        for j in range(8):
            if gray[i, j] > avg:
                hash_str = hash_str+'1'
            else:
                hash_str = hash_str+'0'
    return hash_str

def dHash(img):
    # 差值哈希算法
    # 縮放8*8
    img = cv2.resize(img, (9, 8))
    # 轉換灰度圖
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    hash_str = ''
    # 每行前一個像素大於後一個像素爲1，相反爲0，生成哈希
    for i in range(8):
        for j in range(8):
            if gray[i, j] > gray[i, j+1]:
                hash_str = hash_str+'1'
            else:
                hash_str = hash_str+'0'
    return hash_str

def pHash(img):
    # 感知哈希算法
    # 縮放32*32
    img = cv2.resize(img, (32, 32))   # , interpolation=cv2.INTER_CUBIC

    # 轉換爲灰度圖
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    # 將灰度圖轉爲浮點型，再進行dct變換
    dct = cv2.dct(np.float32(gray))
    # opencv實現的掩碼操作
    dct_roi = dct[0:8, 0:8]

    hash = []
    average = np.mean(dct_roi)
    for i in range(dct_roi.shape[0]):
        for j in range(dct_roi.shape[1]):
            if dct_roi[i, j] > average:
                hash.append(1)
            else:
                hash.append(0)
    return hash

def cmpHash(hash1, hash2):
    # Hash值對比
    # 算法中1和0順序組合起來的即是圖片的指紋hash。順序不固定，但是比較的時候必須是相同的順序。
    # 對比兩幅圖的指紋，計算漢明距離，即兩個64位的hash值有多少是不一樣的，不同的位數越小，圖片越相似
    # 漢明距離：一組二進制數據變成另一組數據所需要的步驟，可以衡量兩圖的差異，漢明距離越小，則相似度越高。漢明距離爲0，即兩張圖片完全一樣
    n = 0
    # hash長度不同則返回-1代表傳參出錯
    if len(hash1) != len(hash2):
        return -1
    # 遍歷判斷
    for i in range(len(hash1)):
        # 不相等則n計數+1，n最終爲相似度
        if hash1[i] != hash2[i]:
            n = n + 1
    return n

####################### Performance Analysis #######################

def pfanaly(similarity):
    correcthit = 0
    falsepositive = 0
    hitprebound2 = 0
    hitprebound3 = 0

    for i in range(1,amount):
        hit=0
        if similarity[i,0] < threshold:

            if mode == 'news':
                f1 = open('news_ground.txt')
                for line in f1:
                    if(i == int(line)):
                        hit=1
                        correcthit += 1
                        break

                if (hit == 0):
                    falsepositive += 1

                f1.close()

            elif mode == 'ngc':
                f2 = open('ngc_ground.txt')
                for line in f2:
                    if '~' in line:
                        prebound2=line.split("~")[0]
                        postbound2=line.split("~")[1]

                        if( (i <= int(postbound2)) & (i >= int(prebound2)) ):
                            hit=1
                            if(hitprebound2 != int(prebound2)):
                                hitprebound2 = int(prebound2)
                                correcthit += 1
                                break
                    else:
                        if(i == int(line)):
                            hit=1
                            correcthit += 1
                            break

                if (hit == 0):
                    falsepositive += 1

                f2.close()

            elif mode == 'ftfm':
                f3 = open('ftfm_ground.txt')
                for line in f3:
                    if '~' in line:
                        prebound3=line.split("~")[0]
                        postbound3=line.split("~")[1]
                        if( (i <= int(postbound3)) & (i >= int(prebound3)) ):
                            hit=1
                            if(hitprebound3 != int(prebound3)):
                                hitprebound3 = int(prebound3)
                                correcthit += 1
                                break
                    else:
                        if(i == int(line)):
                            hit=1
                            correcthit += 1
                            break

                if (hit == 0):
                    falsepositive += 1

                f3.close()

    print("Correct hit amount is : ",correcthit)
    print("False positive amount is : ",falsepositive)

    if mode == 'news':
        recall = correcthit/7 #groundhit1 = 7
        if(correcthit + falsepositive) == 0:
            precision = 0
        else:
            precision = correcthit/(correcthit + falsepositive)
        return recall, precision
    elif mode == 'ngc':
        recall = correcthit/36 #groundhit2 = 36
        if(correcthit + falsepositive) == 0:
            precision = 0
        else:
            precision = correcthit/(correcthit + falsepositive)
        return recall, precision
    elif mode == 'ftfm':
        recall = correcthit/27 #groundhit3 = 27
        if(correcthit + falsepositive) == 0:
            precision = 0
        else:
            precision = correcthit/(correcthit + falsepositive)
        return recall, precision

#######################Main#######################

recall1 = np.zeros(25)
precision1 = np.zeros(25)
recall2 = np.zeros(25)
precision2 = np.zeros(25)
recall3 = np.zeros(25)
precision3 = np.zeros(25)
recall4 = np.zeros(25)
precision4 = np.zeros(25)
recall5 = np.zeros(25)
precision5 = np.zeros(25)
recall = np.zeros(25)
precision = np.zeros(25)
index = 0

for m in range (0,25):
    threshold = 0.5 + m*0.02

    similarity1 = np.ones((amount,1))
    similarity2 = np.ones((amount,1))
    similarity3 = np.ones((amount,1))
    similarity4 = np.ones((amount,1))
    similarity5 = np.ones((amount,1))
    weightedsim = np.ones((amount,1))
    thres = np.full((amount,1),threshold)

    for i in range(1,amount):
        j=str(i-1)
        #print(j)

        k=str(i)
        #print(k)

        if mode == 'ngc':
            testpath = dir + j + '.jpeg'
            #print (testpath)
            test_img = cv2.imread(testpath)
            test_img = cv2.resize(test_img, (128,128), interpolation = cv2.INTER_AREA)

            datapath = dir + k + '.jpeg'
            #print (datapath)
            data_img = cv2.imread(datapath)
            data_img = cv2.resize(data_img, (128,128), interpolation = cv2.INTER_AREA)
        else:
            testpath = dir + j + '.jpg'
            #print (testpath)
            test_img = cv2.imread(testpath)
            test_img = cv2.resize(test_img, (128,128), interpolation = cv2.INTER_AREA)

            datapath = dir + k + '.jpg'
            #print (datapath)
            data_img = cv2.imread(datapath)
            data_img = cv2.resize(data_img, (128,128), interpolation = cv2.INTER_AREA)

        score1 = hist_compare(test_img,data_img)
        similarity1[i,0] = score1

        score2 = classify_hist_with_split(test_img,data_img)
        similarity2[i,0] = score2

        hash1 = aHash(data_img)
        hash2 = aHash(test_img)
        score3 = cmpHash(hash1, hash2)
        score3 = (1-float(score3/64))
        similarity3[i,0] = score3

        hash1 = dHash(data_img)
        hash2 = dHash(test_img)
        score4 = cmpHash(hash1, hash2)
        score4 = (1-float(score4/64))
        similarity4[i,0] = score4

        hash1 = pHash(data_img)
        hash2 = pHash(test_img)
        score5 = cmpHash(hash1, hash2)
        score5 = (1-float(score5/64))
        similarity5[i,0] = score5

        weightedsim[i,0] = (score1+score2+score3+score4+score5)/5

    print("Gray hist comparison detects shot change at : ")
    for i in range (1,amount):
        if similarity1[i,0] < threshold:
            print (i)

    print("\nRGB hist comparison detects shot change at : ")
    for i in range (1,amount):
        if similarity2[i,0] < threshold:
            print (i)

    print("\naHash comparison detects shot change at : ")
    for i in range (1,amount):
        if similarity3[i,0] < threshold:
            print (i)

    print("\ndHash comparison detects shot change at : ")
    for i in range (1,amount):
        if similarity4[i,0] < threshold:
            print (i)

    print("\npHash comparison detects shot change at : ")
    for i in range (1,amount):
        if similarity5[i,0] < threshold:
            print (i)

    print("\nWeighted algorithms detects shot change at : ")
    for i in range (1,amount):
        if weightedsim[i,0] < threshold:
            print (i)

    result1=pfanaly(similarity1)
    recall1[index] = result1[0]
    precision1[index] = result1[1]

    result2=pfanaly(similarity2)
    recall2[index] = result2[0]
    precision2[index]= result2[1]

    result3=pfanaly(similarity3)
    recall3[index] = result3[0]
    precision3[index]= result3[1]

    result4=pfanaly(similarity4)
    recall4[index] = result4[0]
    precision4[index] = result4[1]

    result5=pfanaly(similarity5)
    recall5[index] = result5[0]
    precision5[index] = result5[1]

    result=pfanaly(weightedsim)
    recall[index] = result[0]
    #print(recall[index])
    precision[index] = result[1]
    #print(precision[index])

    index += 1

#print("Precision for Gray hist comparison is : ",precision1)
#print("Recall for Gray hist comparison is : ",recall1)

#print("Precision for RGB hist comparison is : ",precision2)
#print("Recall for RGB hist comparison is : ",recall2)

#print("Precision for aHash comparison is : ",precision3)
#print("Recall for aHash comparison is : ",recall3)

#print("Precision for dHash comparison is : ",precision4)
#print("Recall for dHash comparison is : ",recall4)

#print("Precision for pHash comparison is : ",precision5)
#print("Recall for pHash comparison is : ",recall5)

#print("Precision for weighted algorithms is : ",precision)
#print("Recall for weighted algorithms is : ",recall)

plt.figure("PR Curve")
plt.plot(recall1,precision1,label='Gray hist')
plt.plot(recall2,precision2,label='RGB hist')
plt.plot(recall3,precision3,label='aHash')
plt.plot(recall4,precision4,label='dHash')
plt.plot(recall5,precision5,label='pHash')
plt.plot(recall,precision,label='weighted algorithms')
plt.xlabel("Recall")
plt.ylabel("Precision")
plt.legend(loc='best')
plt.show()

#hist = evaluate(similarity1,similarity2,similarity3,similarity4,similarity5,weightedsim)


from models.user import User
from routes import *

import os
import flask

main = Blueprint('uploads', __name__)

Model = User

def file_name(filename):
    count = len(os.listdir('uploads')) + 1
    name = str(count) + '.' + filename.split('.')[-1]
    print(name)
    return name

@main.route('/add', methods=['POST'])
@login_required
def add():
    # 通过 request.files 访问上传的文件
    # uploaded 是上传时候的文件名
    f = request.files.get('uploaded')
    print(request)
    print('upload, ', request.files)
    if f:
        filename = f.filename
        ext = filename.split('.')[-1]
        valid_filetypes = ('png', 'jpg', 'jpeg', 'gif', 'apng')
        if ext in valid_filetypes:
            filename = file_name(filename)
            print('filename, ', filename)
            path = 'uploads/' + filename
            print(path)
            print(os.getcwd())
            f.save(path)
            u = current_user()
            u.img = '/' + path
            u.save()
            return redirect(url_for('user.profile'))
        else:
            return '
文件类型或大小不对
'
    else:
        return '
没有上传
'

@main.route('/<filename>')
def read(filename):
    # 用这个函数直接返回文件的响应
    return flask.send_from_directory('uploads/', filename)


from flask import Blueprint, Flask, request, jsonify
import api.book_routes as br
import api.author_routes as ar
import api.category_routes as cr
from datetime import datetime
import connection as conn
import query_data as db

app = Flask(__name__)

@app.route("/", methods=["GET"])
def welcome():
    return "Welcome to my App!"

@app.route("/hello/<name>", methods=["GET"])
def say_hello(name):
    return f"Hello {name}"

@app.route("/about")
def about():
    print(request.args)
    return "about me"

@app.route("/login", methods=["GET", "POST"])
def login():
    if request.method == "GET":
        return f"Please Log in"
    elif request.method == "POST":
        print(request)
        print(request.json)
        if request.json["username"] == "Hello" and request.json["password"] == "World":
            return "Thank you for logging in"
        else:
            return "Incorrect Username or Password"

@app.route("/date/today")
def send_date():
    today = datetime.now()
    data = {
        "day": today.day,
        "month": today.month,
        "year": today.year
    }
    # return f"<h1>{today}</h1>"
    return jsonify(data)

app.register_blueprint(ar.publisher_routes, url_prefix='/api/publisher')
app.register_blueprint(cr.category_routes, url_prefix='/api/category')
app.register_blueprint(br.book_routes, url_prefix='/api/book')

if __name__ == "__main__":
    app.run(debug=True)

phase 2 general practice/flask exercise/app.py


# _*_ coding:utf-8 _*_

from fastapi import APIRouter, status, HTTPException, Request
from fastapi.responses import JSONResponse

router = APIRouter()

"""响应状态"""

@router.get("/", status_code=200)
async def read_item():
    return {"item": "itemname"}

@router.post("/", status_code=status.HTTP_201_CREATED)
async def create_item(name: str):
    return {"name": name}

"""异常处理"""

# 2.1 抛出异常
@router.get("/exception/{item_id}/")
async def read_item1(item_id: str):
    items = {"foo": "The Foo Wrestlers"}
    if item_id not in items:
        raise HTTPException(status_code=404, detail="Item not found")
    return {"item": items[item_id]}

# 2.2 异常中自定义头部信息
@router.get("/exception/my-header/{item_id}/")
async def read_item2(item_id: str):
    items = {"foo": "The Foo Wrestlers"}
    if item_id not in items:
        raise HTTPException(
            status_code=404,
            detail="My Define Response Header",
            headers={"X-Error": "There goes my error"}
    )
    return {"item": items[item_id]}

# 2.3 自定义异常类 (APIRouter中没有`exception_handler`方法,FastAPI中才有,这种使用方法在app中才能使用)
# class UnicornException(Exception):
#     def __init__(self, description: str):
#         self.description = description

# @router.exception_handler(UnicornException)  # 传入的是框架内部的异常类，则是重写异常处理
# async def unicorn_exception_handler(request: Request, exec: UnicornException):
#     return JSONResponse(
#         status_code=418,
#         content={"message": f"Oops! {exec.name} did something."}
#     )

# @router.get("/exception/unicorn/")
# async def read_unicorn(description: str):
#     if description != "yolo":
#         raise UnicornException(description=description)
#     return {"unicorn_description": description}


from collections import deque

# Считываем входные данные
n, m = map(int, input().split())
maze = [list(map(int, input().split())) for _ in range(n)]
x1, y1, x2, y2 = map(int, input().split())

deq = deque([(x1-1, y1-1, 0)])

distances = [[-1] * m for _ in range(n)]
distances[x1-1][y1-1] = 0

dx = [-1, 0, 1, 0]
dy = [0, 1, 0, -1]

# bfs
while deq:
    x, y, d = deq.popleft()
    for i in range(4):
        if (
            (0 <= (x + dx[i]) < n) and (0 <= (y + dy[i]) < m) and
            maze[x + dx[i]][y + dy[i]] == 0 and distances[x + dx[i]][y + dy[i]] == -1
        ):
            distances[x + dx[i]][y + dy[i]] = d + 1
            deq.append((x + dx[i], y + dy[i], d + 1))

print(distances[x2-1][y2-1])


import os
import random
import argparse

def train_test_split(split_perc, dir_name, img_format, trainf, testf):
    split_perc/=100
    files = os.listdir(dir_name)
    img_files = sorted([file for file in files if file.endswith(img_format)], key=lambda x:int(x.split('.')[0]))
    test_files_idx = random.sample(range(0, len(img_files)),int(split_perc*len(img_files)))
    # print(img_files)
    # print(test_files_idx)
    # print(len(img_files))
    with open(testf, 'w') as test_f:
        with open(trainf, 'w') as train_f:
            for i, f in enumerate(img_files):
                if i in test_files_idx:
                    test_f.write(os.path.join(dir_name,f)+'\n')
                else:
                    train_f.write(os.path.join(dir_name,f)+'\n')
    return True

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--out', type=str, required=False, default='custom_data')
    parser.add_argument('--train_f', type=str, required=False, default='train.txt')
    parser.add_argument('--test_f', type=str, required=False, default='test.txt')
    parser.add_argument('--img_fmt', type=str, required=False, default=".jpg")
    args = parser.parse_args()
    _ = train_test_split(split_perc = int(input('enter test data% [0-100] : ')), dir_name=args.out, img_format=args.img_fmt, trainf=args.train_f, testf=args.test_f)


from pyspark.sql import SparkSession
from pyspark.sql.functions import when

spark = SparkSession.builder.appName("MobileAnalysis").getOrCreate()
df = spark.read.format("csv").option("header","True").option("inferSchema",True).option("delimiter",",").load('F:\SparkCourse\csv\mobileDataset.csv')

# Replace empty strings with null values
df = df.withColumn("Storage", when(df["Internal_memory"] != "", df["Internal_memory"]).otherwise(None))
df = df.withColumn("BatteryCapacity", when(df["Battery"] != "", df["Battery"]).otherwise(None))
df = df.withColumn("Processor", when(df["CPU"] != "", df["CPU"]).otherwise(None))
df = df.withColumn("Brand", when(df["Brand"] != "", df["Brand"]).otherwise(None))
df = df.withColumn("Network", when(df["Network"] != "", df["Network"]).otherwise(None))
df = df.withColumn("Model", when(df["Model"] != "", df["Model"]).otherwise(None))

# Convert relevant columns to numeric types
df = df.withColumn("Storage", df["Storage"].cast("int"))
df = df.withColumn("BatteryCapacity", df["BatteryCapacity"].cast("int"))

# Filter the dataset to exclude rows with null values
df = df.na.drop(subset=["Brand", "Model", "Storage", "BatteryCapacity", "Processor", "Network"])

from pyspark.sql.window import Window
from pyspark.sql.functions import row_number, desc

# Define a window specification
window_spec = Window.partitionBy("Operating_System").orderBy(desc("Storage"), desc("BatteryCapacity"), desc("Processor"))

# Add row numbers to the DataFrame based on the window specification
df = df.withColumn("rank", row_number().over(window_spec))

# Filter the top 5 phones for iOS and Android
ios_best_phones = df.filter((df["Operating_System"] == "iOS") & (df["rank"] <= 5))
android_best_phones = df.filter((df["Operating_System"] == "Android") & (df["rank"] <= 5))

#1.Market captured by mobile phone brands according to the number of phone models:
brand_market_share = df.groupBy("Brand").count().withColumnRenamed("count", "ModelCount")

#2.Phone models with 4G or LTE network, and phones with 5G network:
from pyspark.sql.functions import col

# Filter phone models with 4G/LTE and 5G networks
models_with_4g_lte = df.filter(col("Network").rlike(r'4G|LTE'))
models_with_5g = df.filter(col("Network").rlike(r'5G'))

#3.Brand with the most number of models with (LTE, 4G network) or 5G network:
from pyspark.sql.functions import when, count

# Create a column to flag whether a phone has (LTE, 4G network) or 5G network
df = df.withColumn("HasLTE4G", when(col("Network").rlike(r'4G|LTE'), "Yes").otherwise("No"))
df = df.withColumn("Has5G", when(col("Network").rlike(r'5G'), "Yes").otherwise("No"))

# Count the number of models for each brand with (LTE, 4G network) and 5G network
brand_with_most_models_lte = df.groupBy("Brand").agg(count("Model").alias("LTE4GCount")).orderBy(desc("LTE4GCount")).first()
brand_with_most_models_5g = df.groupBy("Brand").agg(count("Model").alias("5GCount")).orderBy(desc("5GCount")).first()

#4.List of discontinued phones released after the year 2015:
from pyspark.sql.functions import year, when

# Extract the year from the "Announced" column
df = df.withColumn("AnnouncedYear", year(df["Announced"]))

# Filter discontinued phones released after 2015
discontinued_phones_after_2015 = df.filter((df["Status"] == "Discontinued") & (df["AnnouncedYear"] > 2015))

export_path = "F:/project/"

brand_market_share.write.csv(export_path + "brand_market_share.csv", header=True, mode="overwrite")
models_with_4g_lte.write(export_path + "models_with_4g_lte.csv", header=True, mode="overwrite")
models_with_5g.write(export_path + "models_with_5g.csv", header=True, mode="overwrite")
brand_with_most_models_lte.write(export_path + "brand_with_most_models_lte.csv", header=True, mode="overwrite")
brand_with_most_models_5g.write(export_path + "brand_with_most_models_5g.csv", header=True, mode="overwrite")
discontinued_phones_after_2015.write(export_path + "discontinued_phones_after_2015.csv", header=True, mode="overwrite")


#THIS CODE DIDN'T END UP SWTICHING THE LAT AND LON COLUMN ORDER AS DESIRED
#WILL WRITE ANOTHER CODE TO READ THROUGH THE FILES AGAIN TO DO THIS
#GONNA POST THIS ANYWAYS SINCE IT WAS A LITTLE USEFUL

import os
import pandas as pd
from tqdm import tqdm

def process_csv_files(root_directory):
    for root, dirs, files in os.walk(root_directory):
        for file in tqdm(files, desc="Processing files"):
            if file.endswith(".csv"):
                file_path = os.path.join(root, file)
                process_csv_file(file_path)

def process_csv_file(file_path):
    df = pd.read_csv(file_path)

    # Switch LAT and LON columns
    df['LAT'], df['LON'] = df['LON'], df['LAT']

    # Add an empty column titled ELEV before the PRECIP column
    df.insert(df.columns.get_loc('PRECIP'), 'ELEV', '')

    # Add an empty column titled NAME after the STATION_ID column
    df.insert(df.columns.get_loc('STATION_ID') + 1, 'NAME', '')

    # Save the updated data back to CSV
    df.to_csv(file_path, index=False)

# Specify the root directory where your CSV files are located
root_directory = 'C:/Users/vnell/OneDrive - Texas A&M University/Extreme Precip/CONVERTEDCRN_subhour'

# Call the function to process the CSV files
process_csv_files(root_directory)


from xml.sax.handler import ContentHandler
import xml.sax
import sys

class textHandler(ContentHandler):
    def characters(self, ch):
        sys.stdout.write(ch.encode("Latin-1"))

parser = xml.sax.make_parser()
handler = textHandler()
parser.setContentHandler(handler)
parser.parse("test.xml")


#!/usr/bin/python
#-*-coding:utf-8-*-

import requests
import os
import sys
import json

'''
json.loads(json_str) json字符串转换成字典
json.dumps(dict) 字典转换成json字符串
'''

class AutoLearning:

    def __init__(self, url):
        print('autolearing')
        self.session = requests.session()
        self.session.get(url, verify=False)

    def type_link(self):
        url = 'https://api.auto-learning.com/v3/xcx/type-link?sessionId=undefined&pageIndex=0'
        res = self.session.get(url)
        self.write_txt_file('D:\docs\Dev\Python\Code\\tingshu-scrap\\type-link.txt', res.content)
        #
        # json_data = res.content
        # print(json_data)
        # print(type(json_data))
        # dict_data = json.loads(json_data)
        # print(dict_data['meta']['enMessage'])

    def write_txt_file(self, path, content):
        with open(path, 'w') as file:
            file.write(content)

if __name__ == '__main__':
    obj = AutoLearning('https://api.auto-learning.com')
    obj.type_link()


import os
import yaml
from os.path import join
from datetime import datetime
import csv
import pickle
import re
import argparse
import sys
from shutil import copyfile
import matplotlib.pyplot as plt
plt.switch_backend('agg')
import tools

def main(args):
    load_label = bool(args.load_label)

    project_dir = os.path.dirname(os.getcwd())
    with open(join(project_dir, 'config.yaml'), 'r') as f:
        cfg = yaml.load(f)

    middle_data_path = join(project_dir, cfg['base_conf']['middle_data_path'])
    meeting_paths = os.listdir(middle_data_path)
    meeting_paths = list(filter(lambda x: x != 'time' and x != 'tmp' and x != 'problem data', meeting_paths))

    true_label = {}

    if load_label:
        with open(join(project_dir, 'middle_data', 'true_label.pk'), 'rb') as f:
            true_label = pickle.load(f)
    else:
        for meet in meeting_paths:
            name_paths = os.listdir(join(middle_data_path, meet, 'classifier'))
            for name in name_paths:
                if os.path.isfile(join(middle_data_path, meet, 'classifier',name)):
                    continue
                for pic in os.listdir(join(middle_data_path, meet, 'classifier', name)):
                    true_label[pic] = name

        if os.path.isfile(join(project_dir, 'middle_data', 'true_label.pk')):
            time = datetime.strftime(datetime.now(), '%m-%d-%H-%M-%S')
            copyfile(join(project_dir, 'middle_data', 'true_label.pk'),
                     join(project_dir, 'important_file', '%s_true_label.pk'%time))
        with open(join(project_dir, 'middle_data', 'true_label.pk'), 'wb') as f:
            pickle.dump(true_label, f, protocol=pickle.HIGHEST_PROTOCOL)
    peoples = []
    for k, v in true_label.items():
        if v!='other' and v not in cfg['filter_people']:
            peoples.append(v)
    peoples = list(set(peoples))

    result_data_path = join(project_dir, 'final_result')
    paths = os.listdir(result_data_path)
    paths = list(filter(lambda x: re.match(re.escape(args.header) + r'.+\.csv', x), paths))
    # print(true_label)

    tem_label = {}
    for k,v in true_label.items():
        temp = k.split('_')
        tem_label["%s_%s_%s"%(temp[0], temp[1], temp[2])] = v
    true_label = tem_label
    max_person_sta = {}

    for p in peoples:
        max_person_sta[p] = 0

    draw_acc_x = []
    draw_acc_y = []
    with open(join(project_dir, 'report', '%s_report.csv'%args.header), 'w') as final_report:
        final_report.write('file_name,hyper_para,add_person,size,miss_num,accuarcy')
        for p in peoples:
            # print(cfg['mac_name'][p])
            final_report.write(',%s,%s_size'%(p,p))
        final_report.write(',mean_acc')
        final_report.write('\n')
        for path in paths:
            temp_path = os.path.split(path)[-1]
            final_report.write('%s,' % temp_path)
            temp_path = re.split(r'-|_', temp_path)
            final_report.write('%s,%s,'%(temp_path[-2], temp_path[-1][:-4]))
            if temp_path[-1][:-5] != 'voting':
                draw_acc_x.append(int(temp_path[-1][:-4]))
            else:
                draw_acc_x.append(0)
            true = 0
            miss = 0
            all = 0
            person_sta_true = {}
            person_sta_false = {}
            with open(join(project_dir, 'final_result', path)) as f:
                result = csv.reader(f, delimiter=',')
                for row in result:
                    r = row[0].split('/')
                    if row[1] not in person_sta_false:
                        person_sta_false[row[1]] = 0
                    if row[1] not in person_sta_true:
                        person_sta_true[row[1]] = 0
                    temp = r[-1].split('_')
                    r[-1] = '%s_%s_%s' % (temp[0], temp[1], temp[2])
                    if r[-1] in true_label:
                        all += 1
                        if true_label[r[-1]] == row[1]:
                            true += 1
                            person_sta_true[row[1]] += 1
                        else:
                            # print('label %s, predict %s'%(true_label[r[-1]], row[1]))
                            person_sta_false[row[1]] += 1
                    else:
                        miss += 1
                        print('miss pic %s'%r[-1])
                print("true total pic %d, miss pic %d"%(all, miss))
                final_report.write('%d,%d,' % (all, miss))
            if all == 0:
                acc = 0
            else:
                acc = true/all
            final_report.write('%f' % acc)
            print('acc %f'%acc)
            mean_acc = []
            for p in peoples:
                if p in person_sta_true:
                    if person_sta_true[p]+person_sta_false[p] == 0:
                        acc_per = 0
                    else:
                        acc_per = person_sta_true[p]/(person_sta_true[p]+person_sta_false[p])
                    print("%s : %f size: %d" % (p, acc_per, person_sta_true[p] + person_sta_false[p]))
                    mean_acc.append(acc_per)
                else:
                    acc_per = 0
                if acc_per>max_person_sta[p]:
                    max_person_sta[p] = acc_per

            draw_acc_y.append(sum(mean_acc) / float(len(mean_acc)))

            for p in peoples:
                # print(cfg['mac_name'][p])
                if p in person_sta_true:
                    if person_sta_true[p] + person_sta_false[p] == 0:
                        acc_per = 0
                    else:
                        acc_per = person_sta_true[p] / (person_sta_true[p] + person_sta_false[p])
                    final_report.write(',%f,%d' % (acc_per, person_sta_true[p] + person_sta_false[p]))
                else:
                    final_report.write(',0,0')
            print('\n')
            final_report.write(',%f'%draw_acc_y[-1])
            final_report.write('\n')

        for k, v in max_person_sta.items():
            print("%s max acc rate: %f"%(k, v))
        plt.figure(figsize=(12, 4))
        di = list(zip(draw_acc_x, draw_acc_y))
        di.sort(key=lambda x:x[0])
        plt.plot([a[0] for a in di], [a[1] for a in di], color="red", linewidth=2)
        plt.ylabel("Mean Acc(%)")
        plt.xlabel("Add Person")
        plt.title("Accuracy plot")
        plt.savefig(join(project_dir, 'report', '%s_report.jpg'%args.header))

def parse_arguments(argv):
    parser = argparse.ArgumentParser()
    parser.add_argument('-l', '--load_label', type=int,
                        help='Directory where to place model.', default=1)
    parser.add_argument('-d', '--header', type=str,
                        help='The header name of result file.', default='2018-04-16-22-18-30')
    return parser.parse_args(argv)

if __name__ == '__main__':
    main(parse_arguments(sys.argv[1:]))
    tools.email_subject(os.path.basename(__file__))


import numpy as np
import matplotlib.pyplot as plt
import math

data1='/Users/LJ/gas_kinematicsdat.txt'
gas=[]
f=open(data1,'r')
f.seek(0)
for i in range(4194304):
 List=f.readline().strip().split()
 gas.append(List)

x_sun=8.3*np.cos(153./180.*np.pi)
y_sun=8.3*np.sin(153./180.*np.pi)
v_sun=210.
v_sun_x=-v_sun*np.sin(27./180.*np.pi)
v_sun_y=-v_sun*np.cos(27./180.*np.pi)
l=[]

for i in range(4194304):
 if np.sqrt((float(gas[i][0])**2+float(gas[i][1])**2))>=14.5:
  c=0
  l.append(c)
 if np.sqrt((float(gas[i][0])**2+float(gas[i][1])**2))<14.5:
  if float(gas[i][1])>(y_sun/x_sun)*float(gas[i][0]) and abs(float(gas[i][1])-(y_sun/x_sun)*float(gas[i][0]))>10**(-6):
   c=-1*math.acos((x_sun**2+y_sun**2-x_sun*float(gas[i][0])-y_sun*float(gas[i][1]))/(np.sqrt(x_sun**2+y_sun**2)*np.sqrt((float(gas[i][0])-x_sun)**2+(float(gas[i][1])-y_sun)**2)))*180/np.pi
   l.append(c)
  if float(gas[i][1])<(y_sun/x_sun)*float(gas[i][0]) and abs(float(gas[i][1])-(y_sun/x_sun)*float(gas[i][0]))>10**(-6):
   c=1*math.acos((x_sun**2+y_sun**2-x_sun*float(gas[i][0])-y_sun*float(gas[i][1]))/(np.sqrt(x_sun**2+y_sun**2)*np.sqrt((float(gas[i][0])-x_sun)**2+(float(gas[i][1])-y_sun)**2)))*180/np.pi
   l.append(c)
  if abs(float(gas[i][1])-(y_sun/x_sun)*float(gas[i][0]))<=10**(-6):
   c=0
   l.append(c)

v_los=[]
omega_b=33.
tmp_r=np.zeros(4194304)
e_los_x=np.zeros(4194304)
e_los_y=np.zeros(4194304)
velocity_x=np.zeros(4194304)
velocity_y=np.zeros(4194304)
v_los_tmp=np.zeros(4194304)
v_los_sun=np.zeros(4194304)

for k in range(4194304):
 if np.sqrt((float(gas[k][0])**2+float(gas[k][1])**2))<14.5:
  tmp_r[k]=np.sqrt((float(gas[k][0])-x_sun)**2+(float(gas[k][1])-y_sun)**2)
  e_los_x[k]=(float(gas[k][0])-x_sun)/tmp_r[k]
  e_los_y[k]=(float(gas[k][1])-y_sun)/tmp_r[k]
  v_los_tmp[k]=float(gas[k][3])*e_los_x[k]+float(gas[k][4])*e_los_y[k]
  v_los_sun[k]=v_sun_x*e_los_x[k]+v_sun_y*e_los_y[k]
  c=v_los_tmp[k]-v_los_sun[k]
  v_los.append(c)
 if np.sqrt((float(gas[k][0])**2+float(gas[k][1])**2))>=14.5:
  c=0
  v_los.append(c)

l=np.asarray(l)
v_los=np.asarray(v_los)
l1=[]
v_los1=[]
for j in range(900):
 index1=np.where((l>=-90.0+0.1*j) & (l<-90.0+0.1*(j+1)))
 index2=np.where(v_los==min(v_los[index1]))
 l1.append(l[index2])
 v_los1.append(v_los[index2])

l2=[]
v_los2=[]
v2=[]
for n in range(900):
 index3=np.where((l>=0.0+0.1*n) & (l<0.0+0.1*(n+1)))
 index4=np.where(v_los==max(v_los[index3]))
 l2.append(l[index4])
 v_los2.append(v_los[index4])

fig=plt.figure()
plt.plot(l1,v_los1,'.')
plt.plot(l2,v_los2,'.')
plt.xlabel("l(degree)")
plt.ylabel("v_los(km/s)")
plt.title("lv diagram")
plt.xlim(-90.0,90.0)
plt.show()


import argparse, os, sys, datetime, glob, importlib
from omegaconf import OmegaConf
import numpy as np
from PIL import Image
import torch
import torchvision
from torch.utils.data import random_split, DataLoader, Dataset
import pytorch_lightning as pl
from pytorch_lightning import seed_everything
from pytorch_lightning.trainer import Trainer
from pytorch_lightning.callbacks import ModelCheckpoint, Callback, LearningRateMonitor
from pytorch_lightning.utilities import rank_zero_only
from pytorch_lightning.strategies.ddp import DDPStrategy

from dataset import cityscape_ds
from dataset.cityscape_ds import collate_fn
import os
os.environ["CUDA_VISIBLE_DEVICES"] = '1'

def get_obj_from_str(string, reload=False):
    module, cls = string.rsplit(".", 1)
    if reload:
        module_imp = importlib.import_module(module)
        importlib.reload(module_imp)
    return getattr(importlib.import_module(module, package=None), cls)

def get_parser(**parser_kwargs):
    def str2bool(v):
        if isinstance(v, bool):
            return v
        if v.lower() in ("yes", "true", "t", "y", "1"):
            return True
        elif v.lower() in ("no", "false", "f", "n", "0"):
            return False
        else:
            raise argparse.ArgumentTypeError("Boolean value expected.")

    parser = argparse.ArgumentParser(**parser_kwargs)
    parser.add_argument(
        "-n",
        "--name",
        type=str,
        const=True,
        default="",
        nargs="?",
        help="postfix for logdir",
    )
    parser.add_argument(
        "-r",
        "--resume",
        type=str,
        const=True,
        default="",
        nargs="?",
        help="resume from logdir or checkpoint in logdir",
    )
    parser.add_argument(
        "-b",
        "--base",
        nargs="*",
        metavar="base_config.yaml",
        help="paths to base configs. Loaded from left-to-right. "
        "Parameters can be overwritten or added with command-line options of the form `--key value`.",
        default=list(),
    )
    parser.add_argument(
        "-t",
        "--train",
        type=str2bool,
        const=True,
        default=False,
        nargs="?",
        help="train",
    )
    parser.add_argument(
        "--no-test",
        type=str2bool,
        const=True,
        default=False,
        nargs="?",
        help="disable test",
    )
    parser.add_argument("-p", "--project", help="name of new or path to existing project")
    parser.add_argument(
        "-d",
        "--debug",
        type=str2bool,
        nargs="?",
        const=True,
        default=False,
        help="enable post-mortem debugging",
    )
    parser.add_argument(
        "-s",
        "--seed",
        type=int,
        default=23,
        help="seed for seed_everything",
    )
    parser.add_argument(
        "-f",
        "--postfix",
        type=str,
        default="",
        help="post-postfix for default name",
    )

    return parser

def nondefault_trainer_args(opt):
    parser = argparse.ArgumentParser()
    parser = Trainer.add_argparse_args(parser)
    args = parser.parse_args([])
    return sorted(k for k in vars(args) if getattr(opt, k) != getattr(args, k))

def instantiate_from_config(config):
    if not "target" in config:
        raise KeyError("Expected key `target` to instantiate.")
    return get_obj_from_str(config["target"])(**config.get("params", dict()))

class WrappedDataset(Dataset):
    """Wraps an arbitrary object with __len__ and __getitem__ into a pytorch dataset"""
    def __init__(self, dataset):
        self.data = dataset

    def __len__(self):
        return len(self.data)

    def __getitem__(self, idx):
        return self.data[idx]

class DataModuleFromConfig(pl.LightningDataModule):
    def __init__(self, batch_size, train=None, validation=None, test=None,
                 wrap=False, num_workers=None):
        super().__init__()
        self.batch_size = batch_size
        self.dataset_configs = dict()
        self.num_workers = num_workers if num_workers is not None else batch_size*2
        if train is not None:
            self.dataset_configs["train"] = train
            self.train_dataloader = self._train_dataloader
        if validation is not None:
            self.dataset_configs["validation"] = validation
            self.val_dataloader = self._val_dataloader
        if test is not None:
            self.dataset_configs["test"] = test
            self.test_dataloader = self._test_dataloader
        self.wrap = wrap

    def prepare_data(self):
        for data_cfg in self.dataset_configs.values():
            instantiate_from_config(data_cfg)

    def setup(self, stage=None):
        self.datasets = dict(
            (k, instantiate_from_config(self.dataset_configs[k]))
            for k in self.dataset_configs)
        if self.wrap:
            for k in self.datasets:
                self.datasets[k] = WrappedDataset(self.datasets[k])

    def _train_dataloader(self):
        return DataLoader(self.datasets["train"], batch_size=self.batch_size, drop_last=True,
                          num_workers=self.num_workers, shuffle=True, collate_fn=collate_fn)

    def _val_dataloader(self):
        return DataLoader(self.datasets["validation"],
                          batch_size=self.batch_size, drop_last=True,
                          num_workers=self.num_workers, collate_fn=collate_fn)

    def _test_dataloader(self):
        return DataLoader(self.datasets["test"], batch_size=self.batch_size,
                          num_workers=self.num_workers, collate_fn=collate_fn)

class SetupCallback(Callback):
    def __init__(self, resume, now, logdir, ckptdir, cfgdir, config, lightning_config):
        super().__init__()
        self.resume = resume
        self.now = now
        self.logdir = logdir
        self.ckptdir = ckptdir
        self.cfgdir = cfgdir
        self.config = config
        self.lightning_config = lightning_config

    def on_pretrain_routine_start(self, trainer, pl_module):
        if trainer.global_rank == 0:
            # Create logdirs and save configs
            os.makedirs(self.logdir, exist_ok=True)
            os.makedirs(self.ckptdir, exist_ok=True)
            os.makedirs(self.cfgdir, exist_ok=True)

            print("Project config")
            print(self.config.pretty())
            OmegaConf.save(self.config,
                           os.path.join(self.cfgdir, "{}-project.yaml".format(self.now)))

            print("Lightning config")
            print(self.lightning_config.pretty())
            OmegaConf.save(OmegaConf.create({"lightning": self.lightning_config}),
                           os.path.join(self.cfgdir, "{}-lightning.yaml".format(self.now)))

        else:
            # ModelCheckpoint callback created log directory --- remove it
            if not self.resume and os.path.exists(self.logdir):
                dst, name = os.path.split(self.logdir)
                dst = os.path.join(dst, "child_runs", name)
                os.makedirs(os.path.split(dst)[0], exist_ok=True)
                try:
                    os.rename(self.logdir, dst)
                except FileNotFoundError:
                    pass

class ImageLogger(Callback):
    def __init__(self, batch_frequency, max_images, clamp=True, increase_log_steps=True):
        super().__init__()
        self.batch_freq = batch_frequency
        self.max_images = max_images
        self.logger_log_images = {
            pl.loggers.WandbLogger: self._wandb,
            pl.loggers.TestTubeLogger: self._testtube,
        }
        self.log_steps = [2 ** n for n in range(int(np.log2(self.batch_freq)) + 1)]
        if not increase_log_steps:
            self.log_steps = [self.batch_freq]
        self.clamp = clamp

    @rank_zero_only
    def _wandb(self, pl_module, images, batch_idx, split):
        raise ValueError("No way wandb")
        grids = dict()
        for k in images:
            grid = torchvision.utils.make_grid(images[k])
            grids[f"{split}/{k}"] = wandb.Image(grid)
        pl_module.logger.experiment.log(grids)

    @rank_zero_only
    def _testtube(self, pl_module, images, batch_idx, split):
        for k in images:
            grid = torchvision.utils.make_grid(images[k])
            grid = (grid+1.0)/2.0 # -1,1 -> 0,1; c,h,w

            tag = f"{split}/{k}"
            pl_module.logger.experiment.add_image(
                tag, grid,
                global_step=pl_module.global_step)

    @rank_zero_only
    def log_local(self, save_dir, split, images,
                  global_step, current_epoch, batch_idx):
        root = os.path.join(save_dir, "images", split)
        for k in images:
            grid = torchvision.utils.make_grid(images[k], nrow=4)

            grid = (grid+1.0)/2.0 # -1,1 -> 0,1; c,h,w
            grid = grid.transpose(0,1).transpose(1,2).squeeze(-1)
            grid = grid.numpy()
            grid = (grid*255).astype(np.uint8)
            filename = "{}_gs-{:06}_e-{:06}_b-{:06}.png".format(
                k,
                global_step,
                current_epoch,
                batch_idx)
            path = os.path.join(root, filename)
            os.makedirs(os.path.split(path)[0], exist_ok=True)
            Image.fromarray(grid).save(path)

    def log_img(self, pl_module, batch, batch_idx, split="train"):
        if (self.check_frequency(batch_idx) and  # batch_idx % self.batch_freq == 0
                hasattr(pl_module, "log_images") and
                callable(pl_module.log_images) and
                self.max_images > 0):
            logger = type(pl_module.logger)

            is_train = pl_module.training
            if is_train:
                pl_module.eval()

            with torch.no_grad():
                images = pl_module.log_images(batch, split=split, pl_module=pl_module)

            for k in images:
                N = min(images[k].shape[0], self.max_images)
                images[k] = images[k][:N]
                if isinstance(images[k], torch.Tensor):
                    images[k] = images[k].detach().cpu()
                    if self.clamp:
                        images[k] = torch.clamp(images[k], -1., 1.)

            self.log_local(pl_module.logger.save_dir, split, images,
                           pl_module.global_step, pl_module.current_epoch, batch_idx)

            logger_log_images = self.logger_log_images.get(logger, lambda *args, **kwargs: None)
            logger_log_images(pl_module, images, pl_module.global_step, split)

            if is_train:
                pl_module.train()

    def check_frequency(self, batch_idx):
        if (batch_idx % self.batch_freq) == 0 or (batch_idx in self.log_steps):
            try:
                self.log_steps.pop(0)
            except IndexError:
                pass
            return True
        return False

    def on_train_batch_end(self, trainer, pl_module, outputs, batch, batch_idx, dataloader_idx):
        self.log_img(pl_module, batch, batch_idx, split="train")

    def on_validation_batch_end(self, trainer, pl_module, outputs, batch, batch_idx, dataloader_idx):
        self.log_img(pl_module, batch, batch_idx, split="val")

def main():
    parser = get_parser()
    opt, unknown = parser.parse_known_args()
    configs = [OmegaConf.load(cfg) for cfg in opt.base]
    cli = OmegaConf.from_dotlist(unknown)
    config = OmegaConf.merge(*configs, cli)

    from pytorch_lightning.loggers import WandbLogger
    wandb_logger = WandbLogger(name="Fine tune SPADE VQModel official")

    from torch.utils.data import ConcatDataset

    train_dataset = cityscape_ds.load_data(
        data_dir=config.ds_1.data_dir,
        resize_size=config.ds_1.image_size,
        subset_type='train',
        ret_dataset=True,
        fn_qry=r"[a-z]*/*.png"
    )
    # kitti_ds = vq_dataset.load_data(
    #     data_dir=config.ds_2.data_dir,
    #     resize_size=config.ds_2.image_size,
    #     subset_type='train',
    #     ret_dataset=True,
    # )
    # train_dataset = ConcatDataset([kitti_ds, city_ds])
    train_loader = torch.utils.data.DataLoader(
        train_dataset,
        batch_size = config.ds_1.batch_size,
        num_workers = 8,
        collate_fn=collate_fn,
        shuffle=True
    )

    val_loader = cityscape_ds.load_data(
        data_dir=config.data.params.data_dir,
        resize_size=config.data.params.image_size,
        subset_type='val',
        ret_dataset=False,
        data_ld_kwargs={'batch_size': config.ds_1.batch_size, 'num_workers': 8}
    )

    model = instantiate_from_config(config.model)

    trainer = pl.Trainer(devices=1, precision="16-mixed", logger=wandb_logger, max_epochs=20)#strategy=DDPStrategy(find_unused_parameters=True)
    trainer.fit(model, train_loader, val_loader)#, ckpt_path="/data/harry/Data_generation/diffusers-main/VQVAE/lightning_logs/hm2zmxps/checkpoints/epoch=58-step=58528.ckpt")

if __name__ == "__main__":
    main()


#!/usr/bin/env python

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

from sklearn.datasets import make_classification, make_circles, make_moons
from sklearn.model_selection import train_test_split, cross_val_score

from sklearn.preprocessing import StandardScaler
from sklearn.feature_selection import VarianceThreshold

from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier

import lightgbm.sklearn as lgb
from catboost import CatBoostClassifier

from ikaros.pipeline import IPipeline
from ikaros.stages   import SequentialStage, StackedStage
from ikaros.sampling import discrete, continuous

from functools import partial

def sample_loss( estimator, X, y ):
    return cross_val_score(estimator,
                           X,
                           y=y,
                           scoring='roc_auc',
                           cv=2).mean()

def _main():

    X, y = make_classification(n_samples=1000,
                               n_features=10,
                               n_informative=8,
                               n_redundant=2)

    s1 = SequentialStage()
#    s1.add( VarianceThreshold, params={ 'threshold' : continuous( 0, 0.05 ) } )
    s1.add( StandardScaler )

    s2 = StackedStage()
    s2.add( KNeighborsClassifier, params={ 'n_neighbors' : 5,
                                           'weights' : 'uniform' } )
    s2.add( RandomForestClassifier, params={ 'n_estimators' : 350 } )
    s2.add( SVC, params={ 'gamma' : continuous(1,4) } )
    s2.add( LogisticRegression, params={ 'C' : continuous(0.001,10) } )

    ik = IPipeline( stages=[s1,s2], loss=partial(sample_loss,X=X,y=y), verbose=True )

    ik.fit( X, y )
    ik.optimize(n_iterations=30)

    bpipeline = ik.pipelines[0]
    yest = np.array(bpipeline.stage_loss)
    Xvest = np.array([(i['SVC__gamma'],i['LogisticRegression__C']) for i in bpipeline.stage_space])

    bpipeline.bootstrap(bootstrap=100)

    y = np.array(bpipeline.stage_loss)
    X = np.array([(i['SVC__gamma'],i['LogisticRegression__C']) for i in bpipeline.stage_space])

    optimum = y[y.argmax()]
    optimum_X = X[y.argmax()]

    fig = plt.figure()
    ax = fig.add_subplot(111)
    ax.set_title( 'sample loss surface' )
    ax.set_xlabel( 'gamma')
    ax.set_ylabel( 'C' )
    ax.set_xlim(1,4)
    ax.set_ylim(0,10)
    cp = ax.tricontourf(X[:,0], X[:,1], y, 100, cmap='viridis')
    plt.colorbar(cp)
    ax.scatter(X[:,0], X[:,1], color='blue', s=20)
    ax.scatter(optimum_X[0], optimum_X[1], marker='*', c='gold', s=150)
    ax.annotate('{}'.format(optimum), xy=(optimum_X[0], optimum_X[1]), xytext=(optimum_X[0]+0.2, optimum_X[1]+0.2),
                arrowprops=dict(facecolor='black', shrink=0.05),
                )

    plt.savefig( '../images/kernelsurface.png' )
    plt.show()

    return 0

if __name__ == '__main__':
    import sys
    sys.exit( _main() )


# -*- coding: utf-8 -*-
import scrapy

class LjTopSpider(scrapy.Spider):
    custom_settings = {
        'FOLLOW_CANONICAL_LINKS': False,
        'ROBOTSTXT_OBEY': False,
    }
    name = 'lj_top'
    allowed_domains = ['www.livejournal.com']
    start_urls = [
        # 'https://www.livejournal.com/'
    ]

    csv_dump_name = 'livejournal_top'

    def __init__(self, *args, **kwargs):
        super(LjTopSpider, self).__init__(*args, **kwargs)

        users = [
            'https://www.livejournal.com/ratings/users/authority/?country=cyr&page={}'.format(p + 1)
            for p in range(3800)
        ]
        communities = [
            'https://www.livejournal.com/ratings/community/authority/?country=cyr&page={}'.format(p + 1)
            for p in range(190)
        ]
        self.start_urls = users + communities

    def parse(self, response):
        links = response.xpath('//a[contains(@class, "i-ljuser-username")]/@href').extract()
        for link in links:
            yield {
                '__csv_dump_name': self.csv_dump_name,
                'domain': link
            }


#
# File: lorenzattractor_test.py
#
import torch
from ceem.systems.lorenzattractor import LorenzAttractor

def test_jacobians():
    sigma = torch.tensor([10.])
    rho = torch.tensor([28.])
    beta = torch.tensor([8. / 3.])
    B = 2
    T = 2
    n = 3

    C = torch.randn(2, 3)

    csys = LorenzAttractor(sigma, rho, beta, C, 0.1)
    theta_keys = [k for k, v in csys.named_parameters()]
    x = torch.randn(B, T, n)

    # jac_dyn_theta = csys.jac_dyn_theta(None, x, None)
    # # Test keys length and names
    # assert len(theta_keys) == len(jac_dyn_theta.keys())
    # assert (list(theta_keys) == list(jac_dyn_theta.keys()))

    # for k, v in csys.named_parameters():
    #     assert jac_dyn_theta[k].shape == (B, T, n, 1)

    jac_obs_theta = csys.jac_obs_theta(None, x, None)
    # Test keys length and names
    assert len(theta_keys) == len(jac_obs_theta.keys())
    assert (list(theta_keys) == list(jac_obs_theta.keys()))

    for v in jac_obs_theta.values():
        assert v is None


import socket, errno, sys
from pysnmp.carrier.asyncore.base import AbstractSocketTransport
from pysnmp.carrier import sockfix, sockmsg, error
from pysnmp import debug

# Ignore these socket errors
sockErrors = {errno.ESHUTDOWN: True,
              errno.ENOTCONN: True,
              errno.ECONNRESET: False,
              errno.ECONNREFUSED: False,
              errno.EAGAIN: False,
              errno.EWOULDBLOCK: False}

if hasattr(errno, 'EBADFD'):
    # bad FD may happen upon FD closure on n-1 select() event
    sockErrors[errno.EBADFD] = True

class DgramSocketTransport(AbstractSocketTransport):
    sockType = socket.SOCK_DGRAM
    retryCount = 3
    retryInterval = 1
    addressType = lambda x: x
    def __init__(self, sock=None, sockMap=None):
        self.__outQueue = []
        self._sendto = lambda s, b, a: s.sendto(b, a)
        def __recvfrom(s, sz):
            d, a = s.recvfrom(sz)
            return d, self.addressType(a)
        self._recvfrom = __recvfrom
        AbstractSocketTransport.__init__(self, sock, sockMap)

    def openClientMode(self, iface=None):
        if iface is not None:
            try:
                self.socket.bind(iface)
            except socket.error:
                raise error.CarrierError('bind() for %s failed: %s' % (iface is None and "<all local>" or iface, sys.exc_info()[1]))
        return self

    def openServerMode(self, iface=None):
        try:
            self.socket.bind(iface)
        except socket.error:
            raise error.CarrierError('bind() for %s failed: %s' % (iface, sys.exc_info()[1],))
        return self

    def enableBroadcast(self, flag=1):
        try:
            self.socket.setsockopt(
                socket.SOL_SOCKET, socket.SO_BROADCAST, flag
            )
        except socket.error:
            raise error.CarrierError('setsockopt() for SO_BROADCAST failed: %s' % (sys.exc_info()[1],))
        debug.logger & debug.flagIO and debug.logger('enableBroadcast: %s option SO_BROADCAST on socket %s' % (flag and "enabled" or "disabled", self.socket.fileno()))
        return self

    def enablePktInfo(self, flag=1):
        if not hasattr(self.socket, 'sendmsg') or \
           not hasattr(self.socket, 'recvmsg'):
            raise error.CarrierError('sendmsg()/recvmsg() interface is not supported by this OS and/or Python version')

        try:
            if self.socket.family in (socket.AF_INET, socket.AF_INET6):
                self.socket.setsockopt(socket.SOL_IP, socket.IP_PKTINFO, flag)
            if self.socket.family == socket.AF_INET6:
                self.socket.setsockopt(socket.SOL_IPV6, socket.IPV6_RECVPKTINFO, flag)
        except socket.error:
            raise error.CarrierError('setsockopt() for %s failed: %s' % (self.socket.family == socket.AF_INET6 and "IPV6_RECVPKTINFO" or "IP_PKTINFO", sys.exc_info()[1]))

        self._sendto = sockmsg.getSendTo(self.addressType)
        self._recvfrom = sockmsg.getRecvFrom(self.addressType)

        debug.logger & debug.flagIO and debug.logger('enablePktInfo: %s option %s on socket %s' % (self.socket.family == socket.AF_INET6 and "IPV6_RECVPKTINFO" or "IP_PKTINFO", flag and "enabled" or "disabled", self.socket.fileno()))
        return self

    def enableTransparent(self, flag=1):
        try:
            if self.socket.family == socket.AF_INET:
                self.socket.setsockopt(
                    socket.SOL_IP, socket.IP_TRANSPARENT, flag
                )
            if self.socket.family == socket.AF_INET6:
                self.socket.setsockopt(
                    socket.SOL_IPV6, socket.IP_TRANSPARENT, flag
                )
        except socket.error:
            raise error.CarrierError('setsockopt() for IP_TRANSPARENT failed: %s' % sys.exc_info()[1])
        except OSError:
            raise error.CarrierError('IP_TRANSPARENT socket option requires superusre previleges')

        debug.logger & debug.flagIO and debug.logger('enableTransparent: %s option IP_TRANSPARENT on socket %s' % (flag and "enabled" or "disabled", self.socket.fileno()))
        return self

    def sendMessage(self, outgoingMessage, transportAddress):
        self.__outQueue.append(
            (outgoingMessage, self.normalizeAddress(transportAddress))
            )
        debug.logger & debug.flagIO and debug.logger('sendMessage: outgoingMessage queued (%d octets) %s' % (len(outgoingMessage), debug.hexdump(outgoingMessage)))

    def normalizeAddress(self, transportAddress):
        if not isinstance(transportAddress, self.addressType):
            transportAddress = self.addressType(transportAddress)
        if not transportAddress.getLocalAddress():
            transportAddress.setLocalAddress(self.getLocalAddress())
        return transportAddress

    def getLocalAddress(self):
        # one evil OS does not seem to support getsockname() for DGRAM sockets
        try:
            return self.socket.getsockname()
        except:
            return ('0.0.0.0', 0)

    # asyncore API
    def handle_connect(self):
        pass

    def writable(self):
        return self.__outQueue

    def handle_write(self):
        outgoingMessage, transportAddress = self.__outQueue.pop(0)
        debug.logger & debug.flagIO and debug.logger('handle_write: transportAddress %r -> %r outgoingMessage (%d octets) %s' % (transportAddress.getLocalAddress(), transportAddress, len(outgoingMessage), debug.hexdump(outgoingMessage)))
        if not transportAddress:
            debug.logger & debug.flagIO and debug.logger('handle_write: missing dst address, loosing outgoing msg')
            return
        try:
            self._sendto(
                self.socket, outgoingMessage, transportAddress
            )
        except socket.error:
            if sys.exc_info()[1].args[0] in sockErrors:
                debug.logger & debug.flagIO and debug.logger('handle_write: ignoring socket error %s' % (sys.exc_info()[1],))
            else:
                raise error.CarrierError('sendto() failed for %s: %s' % (transportAddress, sys.exc_info()[1]))

    def readable(self):
        return 1

    def handle_read(self):
        try:
            incomingMessage, transportAddress = self._recvfrom(
                self.socket, 65535
            )
            transportAddress = self.normalizeAddress(transportAddress)
            debug.logger & debug.flagIO and debug.logger('handle_read: transportAddress %r -> %r incomingMessage (%d octets) %s' % (transportAddress, transportAddress.getLocalAddress(), len(incomingMessage), debug.hexdump(incomingMessage)))
            if not incomingMessage:
                self.handle_close()
                return
            else:
                self._cbFun(self, transportAddress, incomingMessage)
                return
        except socket.error:
            if sys.exc_info()[1].args[0] in sockErrors:
                debug.logger & debug.flagIO and debug.logger('handle_read: known socket error %s' % (sys.exc_info()[1],))
                sockErrors[sys.exc_info()[1].args[0]] and self.handle_close()
                return
            else:
                raise error.CarrierError('recvfrom() failed: %s' % (sys.exc_info()[1],))

    def handle_close(self):
        pass # no datagram connection


#! /usr/bin/env python
# -*- coding: utf-8 -*-

import argparse
import subprocess

def main():
    parser = argparse.ArgumentParser(description='Synchronize with the virtual machine.')
    parser.add_argument('--user', type=str, help='user who have the permissions on the destination', default='root')
    parser.add_argument('virual_machine', type=str, help='VM with which to synchronize')
    parser.add_argument('src', type=str, help='source directory')
    parser.add_argument('dest', type=str, help='destination directory')
    args = parser.parse_args()

    cmd = ["rsync", "-avz", args.src, "%s@%s:%s" % (args.user, args.virual_machine, args.dest)]
    subprocess.call(cmd)

if __name__ == '__main__':
    main()


import codecs
import glob
import os
from setuptools import setup, find_packages

# Prevent spurious errors during `python setup.py test`, a la
# http://www.eby-sarna.com/pipermail/peak/2010-May/003357.html:
try:
    import multiprocessing
except ImportError:
    pass

def read(fname):
    fpath = os.path.join(os.path.dirname(__file__), fname)
    with codecs.open(fpath, 'r', 'utf8') as f:
        return f.read().strip()

setup(
    name='collector',
    version='0.3',
    description=('Socorro is a server to accept Breakpad'
                 ' crash reports.'),
    long_description=open('README.md').read(),
    author='Mozilla',
    author_email='socorro-dev@mozilla.com',
    license='MPL',
    url='https://github.com/mozilla/socorro-collector',
    classifiers=[
        'Intended Audience :: Developers',
        'Intended Audience :: System Administrators',
        'License :: OSI Approved :: MPL License',
        'Programming Language :: Python :: 2.6',
        'Programming Language :: Python :: 2.7',
        'Topic :: Internet :: WWW/HTTP :: WSGI :: Application',
        ],
    keywords=['socorro', 'breakpad', 'crash', 'reporting', 'minidump',
              'stacktrace'],
    packages=find_packages(),
    install_requires=[], # use peep -r requirements.txt instead
    entry_points={
        'console_scripts': [
                'socorro = socorro.app.socorro_app:SocorroWelcomeApp.run'
            ],
        },
    test_suite='nose.collector',
    zip_safe=False,
),


import os

from src     import genie_client as gc
from src.lib import util as du
from src.lib import util_stopwatch as sw

import json
import pandas as pd

class AccuracyFineTuning():

    def __init__( self ):

        self.project_root     = du.get_project_root()
        self.prompt           = du.get_file_as_string( self.project_root + "/src/prompts/classification-experiment-template.txt" )
        self.class_dictionary = {}

        print( "Using project root [{}]".format( self.project_root ) )

        pd.set_option( "display.width", 512)
        pd.set_option( "display.max_columns", 6 )

    def load_json( self, path ):

        df = pd.read_json( path )

        return df

    def load_training_data( self ):

        # load data
        paths = [
            self.project_root + "/src/prompts/data/synthetic-data-load-url-current-tab.json",
            self.project_root + "/src/prompts/data/synthetic-data-load-url-new-tab.json"
        ]
        dfs = []
        for path in paths:

            df = self.load_json( path )
            total = df.shape[ 0 ]
            print( "prompts in this df: {}".format( total ) )
            dfs.append( df )

        df = pd.concat( dfs )
        df.reset_index( inplace=True )
        df.drop( "index", axis=1, inplace=True )

        # Create numeric representation of categorical system_command
        df[ "system_command_categorical" ] = pd.factorize( df[ "system_command" ] )[ 0 ]

        return df

    def test_data_baseline( self ):

        df = self.load_training_data()
        print( "Total number of prompts: {}".format( df.shape[ 0 ] ) )

        commands = df[ "synonymous_command" ].tolist()
        system_commands = df[ "system_command" ].tolist()
        correct = [ 0 ] * len( commands )

        genie_client = gc.GenieClient()
        timer = sw.Stopwatch()
        correct_count = 0

        for idx, command in enumerate( commands[ 0:2 ] ):

            prompt = self.prompt.replace( "{synonymous_command}", command )

            timer_item = sw.Stopwatch()
            response = genie_client.ask_chat_gpt_using_raw_prompt_and_content( prompt )
            response = json.loads( response )

            print( idx, command, response )
            timer_item.print( "Done interpreting one command", use_millis=True )
            # print( "-" * 120 )

            if system_commands[ 0 ] == response[ "classification" ]:
                correct_count += 1
                correct[ idx ] = 1

        timer.print( "Done Iterating commands", use_millis=False )
        df[ "correct" ] = correct

        accuracy = 100.0 * correct_count / len( commands )
        accuracy = round( accuracy, 1 )

        du.print_banner(
            "[{}] correct out of [{}] commands. Accuracy: {}%".format( correct_count, len( commands ), accuracy )
        )

        return df

    def get_training_prompts( self ):

        df = self.load_training_data()
        prompts_df = df[ [ "synonymous_command", "system_command_categorical" ] ].copy()
        prompts_df.rename( columns={ "synonymous_command": "prompt", "system_command_categorical": "completion" }, inplace=True )

        # Insert hash marks and space according to these instructions below
        # https://community.openai.com/t/gpt3-finetuning-for-multilabel-classification/19105/5
        prompts_df[ "prompt" ]     = prompts_df[ "prompt" ] + "\n\n###\n\n"
        prompts_df[ "completion" ] = " " + prompts_df[ "completion" ].astype( str )

        return prompts_df

if __name__ == "__main__":

    accuracy = AccuracyFineTuning()

    # df = accuracy.test_data_baseline()
    df = accuracy.get_training_prompts()
    df.to_json( accuracy.project_root + "/src/prompts/data/jsonl/open-new-or-current-tab.jsonl", orient='records', lines=True )
    # df = accuracy.load_training_data()
    # df.
    print( df )


import os.path
import sys

INTO_FILE = False
output = []

def fast_input(file_name):
    global INTO_FILE
    if os.path.isfile(file_name):
        INTO_FILE = True
        with open(file_name, 'r') as file:
            input_lines = file.readlines()
    else:
        INTO_FILE = False
        input_lines = sys.stdin.readlines()

    for line in input_lines:
        yield line

fast_input_reader = fast_input('input.txt')

def input():
    return fast_input_reader.__next__()

def print(line):
    output.append(str(line))

def solve():
    from collections import deque
    # read data
    H, W = map(int, input().strip().split())
    field = []
    for _ in range(H):
        field.append(input().strip())
    start_x, start_y = map(int, input().strip().split())
    end_x, end_y = map(int, input().strip().split())

    # build infrastructure
    directions = [(-1, 0), (1, 0), (0, -1), (0, 1), (-1, -1), (-1, 1), (1, -1), (1, 1)]
    MAX_DISTANCE = H * W + 1
    MAX_W, MAX_N_DIR = 1000 * 10, 10
    distances = [[[MAX_DISTANCE for _ in directions] for _ in range(W)] for _ in range(H)]
    queue = deque([])
    for dir_index, _ in enumerate(directions):
        distances[H - start_y][start_x - 1][dir_index] = 1
        code = ((H - start_y) * MAX_W + (start_x - 1)) * MAX_N_DIR + dir_index
        queue.append(code)

    # loop
    res = -1
    while queue:
        code = queue.popleft()
        i, j, dir_index = code // (MAX_W * MAX_N_DIR), (code // MAX_N_DIR) % MAX_W, code % MAX_N_DIR
        d = distances[i][j][dir_index]

        if (i, j) == (H - end_y, end_x - 1):
            res = d
            break

        # 0-edge
        di, dj = directions[dir_index]
        if 0 <= i + di < H and 0 <= j + dj < W and field[i + di][j + dj] == '.':
            dir_index = directions.index((di, dj))
            if d < distances[i + di][j + dj][dir_index]:
                distances[i + di][j + dj][dir_index] = d
                queue.appendleft(((i + di) * MAX_W + (j + dj)) * MAX_N_DIR + dir_index)

        # 1-edge
        for di_, dj_ in directions:
            if 0 <= i + di_ < H and 0 <= j + dj_ < W and field[i + di_][j + dj_] == '.':
                dir_index = directions.index((di_, dj_))
                if distances[i + di_][j + dj_][dir_index] == MAX_DISTANCE:
                    distances[i + di_][j + dj_][dir_index] = d + 1
                    queue.append(((i + di_) * MAX_W + (j + dj_)) * MAX_N_DIR + dir_index)

    print(res)

def main():
    solve()
    answer = '\n'.join(output) + '\n'

    if INTO_FILE:
        with open('output.txt', 'w') as output_file:
            output_file.write(answer)
    else:
        sys.stdout.write(answer)

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
'''
example script for estimation of adaptation parameters of an adaptive leaky or
exponential I&F neuron (LIF/EIF) subject to fluctuating inputs, using method 1a,
cf. Ladenbauer et al. 2019 (Results section 5)
-- written by Josef Ladenbauer in 2018/2019

run time was <5 min on an Intel i7-8550U Laptop using Python 3.7 (Anaconda distrib.
v. 2019.03; also tested using Python 2.7, Anaconda distrib. v. 5.3.0)
'''

import inference_methods as im
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
import time
import scipy.optimize
import scipy.io
import scipy.stats
#matplotlib.rc('text', usetex=True)
matplotlib.rc('xtick', labelsize=12)
matplotlib.rc('ytick', labelsize=12)

# SET PARAMETERS --------------------------------------------------------------
params = dict()
# neuron model parameters:
params['tau_m'] = 20.0  # ms, membrane time constant
params['V_r'] = 0.0  # mV, reset voltage
params['V_s'] = 30.0  # mV, spike voltage
params['Delta_T'] = 0.0  # mV, threshold slope factor (set 0 for LIF model,
                         #                                >0 for EIF model)
params['V_T'] = 15.0  # mV, effective threshold voltage (only used for EIF)
params['T_ref'] = 0.0  # ms, refractory duration
tau_w_true = 100.0  #ms, adaptation time constant
Delta_w_true = 0.5 #mV/ms, adaptation strength

# input parameters:
mu = 1.75  # mV/ms, input mean
sigma = 2.5 # mV/sqrt(ms), input standard deviation (noise intensity)

estimate_input = False  # including the input parameters in the estimation
                        # takes longer (<40 min)

# parameters for data generation:
params['dt_sim'] = 0.05  # ms, simulation time step
t_start = 0.0  # ms
t_end = 1e5  # ms
t_limits = [10e3, 11e3]  # ms, for plots
V_init = params['V_r']  # initial condition
N_spk = 1000  # number of spikes used for estimation
np.random.seed(10)

# parameters for calculation of likelihood (finite volume numerical scheme)
params['V_lb'] = -80.0  # mV, lower bound
if estimate_input:
    params['pISI_method'] = 'fvm' # this parameter only need to be specified if
                                  # input parameters are also estimated
# the next 3 lines are only required for im.find_mu_init, which uses a rapid
# computation of the mean ISI from the model
d_V = 0.025  # mV, spacing of voltage grid
params['V_vals'] = np.arange(params['V_lb'],params['V_s']+d_V/2,d_V)
params['V_r_idx'] = np.argmin(np.abs(params['V_vals']-params['V_r']))
                    # index of reset voltage on grid, this should be a grid point
params['neuron_model'] = 'LIF'
params['integration_method'] = 'implicit'
params['N_centers_fvm'] = 1000  # number of centers for voltage discretization
params['fvm_v_init'] = 'delta'  # voltage density initialization
params['fvm_delta_peak'] = params['V_r']  # location of initial density peak
params['fvm_dt'] = 0.1  # ms, time step for finite volume method
Delta_w_vals = np.arange(0.1, 1.5, 0.005)  # Delta_w values to scan for each
                                           # tau_w value
mu_pert_vals = np.arange(0.2, 2.51, 0.10)  # mean input perturbation magnitudes
                                           # which relate to adaptation strength
                                           # (see im.spiketrain_likel_adapt)
tau_w_init = 70.0  # initial tau_w value

if __name__ == '__main__':

    # GENERATE SPIKE TRAIN from neuron model ----------------------------------
    tgrid = np.arange(t_start, t_end+params['dt_sim']/2, params['dt_sim'])
            # time points for simulation

    rand_vec = np.random.randn(len(tgrid))

    start = time.time()
    V_trace, w_trace, Sp_times = \
        im.simulate_EIF_adapt_numba(tgrid, V_init, params['tau_m'], params['V_s'],
                                    params['V_r'], params['V_T'], params['Delta_T'],
                                    params['T_ref'], mu, sigma, Delta_w_true,
                                    tau_w_true, rand_vec)
    print('')
    print('simulation took {dur}s'.format(dur=np.round(time.time() - start,2)))

    # for plots (further below)
    inds = (tgrid>=t_limits[0]) & (tgrid<=t_limits[1])
    V_trace = V_trace[inds]
    w_trace = w_trace[inds]

    Spt_obs = Sp_times[:N_spk]
    ISIs = np.diff(Spt_obs)
    ISImin = np.min(ISIs)
    ISImax = 1.1*np.max(ISIs)
    lastSpidx = [len(Spt_obs)-1]

    # ESTIMATE PARAMETERS from spike train ------------------------------------
    start = time.time()

    t_grid = np.arange(0, ISImax, params['fvm_dt'])
    params['t_grid'] = t_grid

    if estimate_input:
        # estimate input parameters w/o adaptation for an initial guess (see
        # baseline_input_inference.py)
        sigma_init = 3.0
        mu_init = im.find_mu_init(np.mean(ISIs), args=(params,sigma_init))
        init_vals = np.array([mu_init, sigma_init])
        sol = scipy.optimize.minimize(im.spiketrain_likel_musig, init_vals,
                                      args=(params, ISIs, np.mean(ISIs), ISImax),
                                      method='nelder-mead',
                                      options={'xatol':0.01, 'fatol':0.01})
        mu_estim, sigma_estim = sol.x
        lastrun = False
        tau_w_init = 50.0
        mu_init = mu_estim + 0.5
        sigma_init = 1.2*sigma_estim
        init_vals = np.array([tau_w_init, mu_init, sigma_init])
        args_fixed = (mu_estim, sigma_estim, Delta_w_vals, mu_pert_vals,
                      Spt_obs, lastSpidx, params, ISImin, ISImax, lastrun)
        # now (re)estimate input and adaptation parameters together
        print('')
        print('likelihood optimization may take some time...')
        sol = scipy.optimize.minimize(im.spiketrain_likel_musig_adapt, init_vals,
                                      args=args_fixed, method='nelder-mead',
                                      options={'xatol':0.1, 'fatol':0.05})
        print(sol)
        lastrun = True  # another (last) run to get also Delta_w_estim
        args_fixed = (mu_estim, sigma_estim, Delta_w_vals, mu_pert_vals,
                      Spt_obs, lastSpidx, params, ISImin, ISImax, lastrun)
        Delta_w_estim, tau_w_estim, mu_estim, sigma_estim, loglike = \
            im.spiketrain_likel_musig_adapt(sol.x, *args_fixed)

    else:
        # baseline parameters are assumed to be known at this point
        lastrun = False
        args_fixed = (mu, sigma, Delta_w_vals, mu_pert_vals, Spt_obs, lastSpidx,
                      params, ISImin, ISImax, lastrun)

        print('')
        print('likelihood optimization may take a couple of minutes...')
        sol = scipy.optimize.minimize(im.spiketrain_likel_adapt, tau_w_init,
                                      args=args_fixed, method='nelder-mead',
                                      options={'xatol':0.5, 'fatol':0.1})
        # spiketrain_likel_adapt finds optimal Delta_w via brute force for given tau_w
        print('')
        print(sol)

        tau_w_estim = sol.x

        lastrun = True  # another (last) run to get also Delta_w_estim
        args_fixed = (mu, sigma, Delta_w_vals, mu_pert_vals, Spt_obs, lastSpidx,
                      params, ISImin, ISImax, lastrun)
        Delta_w_estim, tau_w_estim, loglike = im.spiketrain_likel_adapt(tau_w_estim,
                                                                        *args_fixed)

    print('')
    print('estimation took {dur}s'.format(dur=np.round(time.time() - start,2)))
    print('')

    # PLOT --------------------------------------------------------------------
    plt.figure()
    ax = plt.subplot(211)
    plt.plot(tgrid[inds], V_trace, color=(0.6,0.6,0.6))
    Spt_plot = Sp_times[(Sp_times>=t_limits[0]) & (Sp_times<=t_limits[1])]
    for i in range(len(Spt_plot)):
        plt.plot([Spt_plot[i], Spt_plot[i]], [params['V_s'], params['V_s']+2.5],
                 'k', linewidth=2)
    plt.plot([10e3-50, 10e3-50], [-10, 10], 'k', linewidth=2)
    plt.plot([10e3-50, 10e3+150], [-10, -10], 'k',  linewidth=2)
    plt.ylim([-12, 40])
    plt.axis('off')

    plt.subplot(212, sharex=ax)
    plt.plot(tgrid[inds], w_trace, color=(0.6,0.6,0.6))
    plt.plot([10e3-50, 10e3-50], [0.1, 0.3], 'k', linewidth=2)
    plt.plot([10e3-50, 10e3+150], [0.1, 0.1], 'k', linewidth=2)
    plt.axis('off')

    tgrid_tmp, w_trace_estim = im.get_w_trace_numba(Spt_obs, tau_w_estim,
                                                    params['dt_sim'])
    inds = (tgrid_tmp>=t_limits[0]) & (tgrid_tmp<=t_limits[1])
    plt.plot(tgrid_tmp[inds], Delta_w_estim*w_trace_estim[inds], 'g', linewidth=2)

    plt.show()