import os
import re
import shutil
from stuffs.general import General
from tools.helper import bcolors, get_download_dir, print_color, run

class Houdini(General):
    download_loc = get_download_dir()
    copy_dir = "./houdini"
    init_rc_component = """
on early-init
    mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc

on property:ro.enable.native.bridge.exec=1
    exec -- /system/bin/sh -c "echo ':arm_exe:M::\\\\x7f\\\\x45\\\\x4c\\\\x46\\\\x01\\\\x01\\\\x01\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x02\\\\x00\\\\x28::/system/bin/houdini:P' > /proc/sys/fs/binfmt_misc/register"
    exec -- /system/bin/sh -c "echo ':arm_dyn:M::\\\\x7f\\\\x45\\\\x4c\\\\x46\\\\x01\\\\x01\\\\x01\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x03\\\\x00\\\\x28::/system/bin/houdini:P' >> /proc/sys/fs/binfmt_misc/register"
    exec -- /system/bin/sh -c "echo ':arm64_exe:M::\\\\x7f\\\\x45\\\\x4c\\\\x46\\\\x02\\\\x01\\\\x01\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x02\\\\x00\\\\xb7::/system/bin/houdini64:P' >> /proc/sys/fs/binfmt_misc/register"
    exec -- /system/bin/sh -c "echo ':arm64_dyn:M::\\\\x7f\\\\x45\\\\x4c\\\\x46\\\\x02\\\\x01\\\\x01\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x03\\\\x00\\\\xb7::/system/bin/houdini64:P' >> /proc/sys/fs/binfmt_misc/register"
"""
    dl_links = {
        "11.0.0": [
            "https://github.com/supremegamers/vendor_intel_proprietary_houdini/archive/81f2a51ef539a35aead396ab7fce2adf89f46e88.zip",
            "fbff756612b4144797fbc99eadcb6653"],
        "12.0.0": [
            "https://github.com/supremegamers/vendor_intel_proprietary_houdini/archive/0e0164611d5fe5595229854759c30a9b5c1199a5.zip",
            "9709701b44b6ab7fc311c7dc95945bd0"],
        "13.0.0": [
            "https://github.com/supremegamers/vendor_intel_proprietary_houdini/archive/978d8cba061a08837b7e520cd03b635af643ba08.zip",
            "1e139054c05034648fae58a1810573b4"
        ],
        # "9.0.0":[],
        # "8.1.0":[]
    }
    dl_file_name = os.path.join(download_loc, "libhoudini.zip")
    extract_to = "/tmp/houdiniunpack"

    def __init__(self, version):
        self.version = version
        if version in self.dl_links.keys():
            self.dl_link = self.dl_links[version][0]
            self.act_md5 = self.dl_links[version][1]
        else:
            raise ValueError(
                "No available libhoudini for Android {}".format(version))

    def download(self):
        print_color("Downloading libhoudini now .....", bcolors.GREEN)
        super().download()

    def copy(self):
        if os.path.exists(self.copy_dir):
            shutil.rmtree(self.copy_dir)
        run(["chmod", "+x", self.extract_to, "-R"])

        print_color("Copying libhoudini library files ...", bcolors.GREEN)
        name = re.findall("([a-zA-Z0-9]+)\.zip", self.dl_link)[0]
        shutil.copytree(os.path.join(self.extract_to, "vendor_intel_proprietary_houdini-" + name,
                        "prebuilts"), os.path.join(self.copy_dir, "system"), dirs_exist_ok=True)

        init_path = os.path.join(self.copy_dir, "system", "etc", "init", "houdini.rc")
        if not os.path.isfile(init_path):
            os.makedirs(os.path.dirname(init_path), exist_ok=True)
        with open(init_path, "w") as initfile:
            initfile.write(self.init_rc_component)
        os.chmod(init_path, 0o644)


import os.path
import tempfile
import time
import uuid

from flask import Flask
from flask import Response
from flask import abort
from flask import request
from flask import send_file
from flask import send_from_directory
from flask import url_for
from flask_cors import CORS, cross_origin
import requests

import task_queue

HOST = 'https://deoldify-044f6348.localhost.run'
UPLOAD_FOLDER = 'uploads'
ALLOWED_EXTENSIONS = {'png', 'jpg', 'jpeg', 'gif'}

app = Flask(__name__)
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
app.config['MAX_CONTENT_LENGTH'] = 16 * 1024 * 1024
CORS(app)

@app.route('/ping', methods=['GET'])
def ping():
    return 'pong'

@app.route('/deoldify', methods=['POST'])
@cross_origin()
def deoldify():
    data = request.get_json() or request.form
    source_url = data.get('source_url')
    if not source_url:
        file = request.files.get('image')
        if file is None or file.filename == '':
            return abort(400, 'Missing image')

        if file and allowed_file(file.filename):
            filename = f'{uuid.uuid4()}.{_get_extension(file.filename)}'
            file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
            source_url = HOST + url_for('uploaded_file', filename=filename)
            print('Sharing to worker:', source_url)

    job_id = task_queue.deoldify_job(source_url, int(data.get('render_factor', 10)))
    time.sleep(0.2)
    while True:
        status, result = task_queue.get_output(job_id)
        if status == 'Done':
            with tempfile.NamedTemporaryFile('wb', suffix='.jpg') as f:
                result.save(f.name)
                callback = send_file(f.name, mimetype='image/jpeg')
                return callback, 200
        elif status == 'Failed':
            print('Failed job status')
            abort(500, 'Unable to process request')
        else:
            time.sleep(0.5)

@app.route('/reenact', methods=['POST'])
@cross_origin()
def reenact():
    resp = requests.request(
        method=request.method,
        url='http://104.211.46.237:5000/reenact',
        headers={key: value for (key, value) in request.headers if key != 'Host'},
        data=request.get_data(),
        cookies=request.cookies,
        allow_redirects=False)

    excluded_headers = ['content-encoding', 'content-length', 'transfer-encoding', 'connection']
    headers = [(name, value) for (name, value) in resp.raw.headers.items()
               if name.lower() not in excluded_headers]

    response = Response(resp.content, resp.status_code, headers)
    return response

def _get_extension(filename):
    return '.' in filename and filename.rsplit('.', 1)[1].lower()

def allowed_file(filename):
    return _get_extension(filename) in ALLOWED_EXTENSIONS

@app.route('/uploads/<filename>')
def uploaded_file(filename):
    return send_from_directory(app.config['UPLOAD_FOLDER'],
                               filename)

if __name__ == '__main__':
    port = 8080
    host = '0.0.0.0'

    app.run(host=host, port=port)


import os
import numpy as np
import matplotlib as mpl; mpl.use("Qt5Agg")
import matplotlib.pyplot as plt
import pandas as pd

def collectCoeffs(pressure, temperature):

    # pressure in GPa
    # temperature in degK

    coeffs = {
        'alpha': 0,
          'beta': 0,
          'chi': 0,
          'delta': 0,
          'epsilon': 0
    }

    if 0.0 <= pressure <= 2:
        coeffs['alpha'] = 1.11
        coeffs['beta'] = -1.18
        coeffs['chi'] = -0.85
        coeffs['delta'] = 1680
        coeffs['epsilon'] = 487
    # elif pressure == 2:
    #     if 2300 < temperature < 2600:
    #         coeffs['alpha'] = 0.84
    #         coeffs['beta'] = -1.22
    #         coeffs['chi'] = -0.85
    #         coeffs['delta'] = 3245
    #         coeffs['epsilon'] = 487
    # elif pressure == 6:
    #     if 2300 < temperature < 2700:
    #         coeffs['alpha'] = 1.17
    #         coeffs['beta'] = -1.06
    #         coeffs['chi'] = -0.90
    #         coeffs['delta'] = 3337
    #         coeffs['epsilon'] = 487
    elif 2 < pressure:
        coeffs['alpha'] = 1.05
        coeffs['beta'] = -1.10
        coeffs['chi'] = -0.84
        coeffs['delta'] = 3588
        coeffs['epsilon'] = -102

    # if 0.5 <= pressure <= 2:
    #     if 2100 < temperature < 2600:
    #         coeffs['alpha'] = 1.11
    #         coeffs['beta'] = -1.18
    #         coeffs['chi'] = -0.85
    #         coeffs['delta'] = 1680
    #         coeffs['epsilon'] = 487
    # # elif pressure == 2:
    # #     if 2300 < temperature < 2600:
    # #         coeffs['alpha'] = 0.84
    # #         coeffs['beta'] = -1.22
    # #         coeffs['chi'] = -0.85
    # #         coeffs['delta'] = 3245
    # #         coeffs['epsilon'] = 487
    # # elif pressure == 6:
    # #     if 2300 < temperature < 2700:
    # #         coeffs['alpha'] = 1.17
    # #         coeffs['beta'] = -1.06
    # #         coeffs['chi'] = -0.90
    # #         coeffs['delta'] = 3337
    # #         coeffs['epsilon'] = 487
    # elif 2 < pressure < 18:
    #     if 2300 < temperature < 2700:
    #         coeffs['alpha'] = 1.05
    #         coeffs['beta'] = -1.10
    #         coeffs['chi'] = -0.84
    #         coeffs['delta'] = 3588
    #         coeffs['epsilon'] = -102

    return coeffs

def partition(pressure, temperature, deltaIW):
    nbo_t = 2.6
    coeffs = collectCoeffs(pressure=pressure, temperature=temperature)
    alpha = coeffs['alpha']
    beta = coeffs['beta']
    chi = coeffs['chi']
    delta = coeffs['delta']
    epsilon = coeffs['epsilon']
    logD = alpha + (beta * deltaIW) + (chi * nbo_t) + (delta * (1/temperature)) + (epsilon * (pressure/temperature))
    return logD

def hydrostaticPressure(densityMelt, surfaceGrav, depth, atmosPressure):
    p = (densityMelt * surfaceGrav * depth * (10**(-9))) + atmosPressure # GPa
    return p

def adiabatTemp(Tsurf, surfaceGravity, thermExpansion, heatCapacity):
    dTdh = (thermExpansion * surfaceGravity * Tsurf) / heatCapacity
    return dTdh

magmaOceanDepth = (3500 + 2900) * 1000
magmaOceanDepthRange = list(np.arange(1, magmaOceanDepth, 10000))
magmaOceanDepthRangeKM = [i / 1000 for i in magmaOceanDepthRange]

surfaceGrav = 9.8 # m/s^2
densityMelt = 3750 # kg m3
atmosPressure = (1.01325 * (10**(-4))) # GPa
Tsurf = 2000 # degK
thermExpansion = (6 * 10**(-5)) # K^-1
cp = 10**3 # J kg^-1 K^-1
deltaIW = [0, -1, -2]

pressureList = []
adiabatList = []
logfO2list = []

depthT = 1000
for index, inc in enumerate(magmaOceanDepthRange):
    if index != 0:
        dTdh = adiabatTemp(Tsurf=Tsurf, surfaceGravity=surfaceGrav, thermExpansion=thermExpansion, heatCapacity=cp) * (inc - magmaOceanDepthRange[index - 1])
        depthT += dTdh
    pressure = hydrostaticPressure(densityMelt=densityMelt, surfaceGrav=surfaceGrav, depth=inc, atmosPressure=atmosPressure)

    adiabatList.append(depthT)
    pressureList.append(pressure)

for i in deltaIW:
    logDlist = []
    for index, j in enumerate(magmaOceanDepthRange):
        if index != 0:
            logD = partition(pressure=pressureList[index], temperature=adiabatList[index], deltaIW=i)
            logDlist.append(logD)
    logfO2list.append(logDlist)

fig = plt.figure()
ax = fig.gca()
ax2 = ax.twiny()
for index, i in enumerate(logfO2list):
    ax.plot(pressureList[1:], i, label="IW {}".format(deltaIW[index]))
    ax.plot(adiabatList[1:], i, label="IW {}".format(deltaIW[index]))

    ax2.plot(magmaOceanDepthRangeKM[1:], i)
    ax2.set_ylabel('Depth')

ax.grid()
ax.legend(loc='upper right')
# ax.set_xlabel("Pressure (GPa)")
ax.set_xlabel("Temperature (degK)")
ax.set_ylabel("log(D)")
# ax2.set_xlabel("Depth (km)")

df = pd.read_csv('w.csv')

for row in df.index:
    pressure = df['Pressure'][row]
    temperature = df['Temperature'][row]
    fO2 = df['fO2']
    measuredlogD = df['Corrected_logD'][row]
    capsule = df['Capsule'][row]
    if capsule == 'C':
        # ax.scatter(pressure, measuredlogD, color='blue')
        ax.scatter(temperature, measuredlogD, color='blue')
    else:
        # ax.scatter(pressure, measuredlogD, color='red')
        ax.scatter(temperature, measuredlogD, color='red')

# ax.set_title("Partitioning Model (Cottrell et al. 2009)")

plt.show()


import os
import toml

class Helper:
    def createFile(self, file):
        fileLocation = f"temp/{file.name}"
        with open(fileLocation, "wb+") as fileObject:
            fileObject.write(file.getbuffer())

        return fileLocation

    def deleteFile(self, filePath):
        os.remove(filePath)

    def set_api_key(self,google_api_key):
        with open(".streamlit/secrets.toml", "r+") as f:
            secrets = toml.load(f)
            secrets["palm_api_key"] = google_api_key
            f.seek(0)
            toml.dump(secrets, f)
            f.truncate()
        return True


from rest_framework.generics import ListAPIView,RetrieveAPIView
from freelancers.models import freelancer
from .serializers import freelancerSerializer
from rest_framework import filters
from django.http import HttpResponse
from django.http import request
from django.shortcuts import get_object_or_404, render
from django.http import JsonResponse
from rest_framework.response import Response
from rest_framework import serializers
from django.core import serializers
from rest_framework import generics

import json

class FreelancerListView(ListAPIView):
    queryset=freelancer.objects.all()
    serializer_class=freelancerSerializer

class SearchFreelancerListView(ListAPIView):
    queryset=freelancer.objects.all()
    serializer_class=freelancerSerializer
    filter_backends=(filters.SearchFilter,)
    search_fields=('first_name','last_name')

class FreelancerDetailView(RetrieveAPIView):
    queryset=freelancer.objects.all()
    serializer_class=freelancerSerializer

def AdvanceSearchFreelancerListView(request):
        first_name=request.POST.get('cityName')
        print(request)
        print(first_name)
        queryset=freelancer.objects.filter(first_name=first_name)
        print(queryset)
        test = []
        for val in queryset:
            print(val)
            fl = FreeelanceView()
            posts_serialized = FreeelanceView()
            test.append(fl)
        posts_serialized = serializers.serialize('json', test)
        return Response(posts_serialized, safe=False)

class FreeelanceView(object):
    firstName = ""
    lastName = ""
    id = 0

class MultipleFieldLookupMixin(object):
    """
    Apply this mixin to any view or viewset to get multiple field filtering
    based on a `lookup_fields` attribute, instead of the default single field filtering.
    """
    def get_object(self):
        queryset = self.get_queryset()             # Get the base queryset
        queryset = self.filter_queryset(queryset)  # Apply any filter backends
        filter = {}
        for field in self.lookup_fields:
            if self.kwargs[field]: # Ignore empty fields.
                filter[field] = self.kwargs[field]
        obj = get_object_or_404(queryset, **filter)  # Lookup the object
        self.check_object_permissions(self.request, obj)
        return obj

class RetrieveFreelancerView(MultipleFieldLookupMixin, generics.RetrieveAPIView):
    queryset=freelancer.objects.all()
    serializer_class=freelancerSerializer
    lookup_fields = ('first_name', 'id')


#! /usr/bin/env python

import os
from astropy.io import fits
from termcolor import colored

def rc(fname, ext=1):
    eop = fname.rfind('/') #eop = End Of Path; "rfind" finds last occurence of designated string

    if eop != -1:
        path = fname[:eop+1]
        file_only = fname[eop+1:]
    else:
        path = os.getcwd() + '/'
        file_only = fname

    print ('')
    print ('Reading in extension '+colored(ext, 'white')+' of FITS file '+colored(file_only, 'white')+' and copying table')
    print ('File located at: '+colored(path, 'white'))
    print ('')

    f   = fits.open(fname)
    fd  = f[ext].data
    fdc = fd.copy()

    return fdc


from elasticsearch import Elasticsearch
import os

def get_es_instance(_type: str='local'):
    es_url = 'http://localhost:9200/'
    es_node = Elasticsearch(
        hosts=["localhost"],
        port=9200,
        timeout=20
    )

    if _type == 'remote':
        es_url = os.getenv('ELASTIC_URL', 'Elastic URL not found.')
        es_node = Elasticsearch(
            hosts=[es_url],
            port=9243,
            http_auth=(os.getenv('ELASTIC_USER', 'Elastic user not found.'), os.getenv('ELASTIC_PASS', 'Elastic pass not found.')),
            timeout=20
        )

    return es_url, es_node

class Config:
    SECRET_KEY = os.getenv('SECRET_KEY', 'my_secret_key')
    DEBUG = False
    JSON_SORT_KEYS = True

    ES_SIZE = 2000
    ES_SCROLL_TIMEOUT = '1m'

    URL_ENDPOINTS = {
        'references_base': '/rd-cross-referencing',
        'references_orphacodes': '/rd-cross-referencing/orphacodes',
        'references_by_orphacode': '/rd-cross-referencing/orphacodes/{}',
        'references_by_name': '/rd-cross-referencing/orphacodes/names/{}',
        'references_by_omim': '/rd-cross-referencing/omims/{}',
        'references_icds': '/rd-cross-referencing/icd-10s',
        'references_by_icd': '/rd-cross-referencing/icd-10s/{}',
        'classification_hchids': '/rd-classification/hchids',
        'classification_by_hchid': '/rd-classification/hchids/{}',
        'classification_orphacodes_by_hchid': '/rd-classification/hchids/{}/orphacodes',
        'classification_hchids_by_orphacode': '/rd-classification/orphacodes/{}/hchids',
        'classification_by_orphacode_and_hchid': '/rd-classification/orphacodes/{}/hchids/{}',
        'phenotypes_base': '/rd-phenotypes',
        'phenotypes_orphacodes': '/rd-phenotypes/orphacodes',
        'phenotypes_by_orphacode': '/rd-phenotypes/orphacodes/{}',
        'phenotypes_by_hpoids': '/rd-phenotypes/hpoids/{}',
        'genes_base': '/rd-associated-genes',
        'genes_orphacodes': '/rd-associated-genes/orphacodes',
        'genes_by_orphacode': '/rd-associated-genes/orphacodes/{}',
        'genes_genes': '/rd-associated-genes/genes',
        'genes_by_symbol': '/rd-associated-genes/genes/symbols/{}',
        'genes_by_name': '/rd-associated-genes/genes/names/{}',
        'linearization_by_orphacode': '/rd-medical-specialties/orphacodes/{}',
        'linearization_parents': '/rd-medical-specialties/parents',
        'linearization_by_parent': '/rd-medical-specialties/parents/{}',
        'epidemiology_base': '/rd-epidemiology',
        'epidemiology_orphacodes': '/rd-epidemiology/orphacodes',
        'epidemiology_by_orphacode': '/rd-epidemiology/orphacodes/{}',
        'history_base': '/rd-natural_history',
        'history_orphacodes': '/rd-natural_history/orphacodes',
        'history_by_orphacode': '/rd-natural_history/orphacodes/{}'
    }

    BACKEND_BASE_URL = "https://api.orphadata.com"

class DevelopmentConfig(Config):
    DEBUG = True

    ES_URL, ES_NODE = get_es_instance(_type='remote')

class TestingConfig(Config):
    DEBUG = True
    TESTING = True
    PRESERVE_CONTEXT_ON_EXCEPTION = False

    ES_URL, ES_NODE = get_es_instance(_type='local')

class ProductionConfig(Config):
    DEBUG = False

    ES_URL, ES_NODE = get_es_instance(_type='remote')

config_by_name = dict(
    dev=DevelopmentConfig,
    test=TestingConfig,
    production=ProductionConfig
)

key = Config.SECRET_KEY


''' Predict script for single-input phase retrieval'''

import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras.preprocessing.image import ImageDataGenerator
import os
import matplotlib.pyplot as plt
from PIL import Image
from tensorflow.keras.preprocessing import image
import glob
from tensorflow.keras.layers import BatchNormalization, Conv2D, ReLU, Conv2DTranspose, add, concatenate
from scipy.io import loadmat
import numpy as np
# from mobilev3 import MobileNetV3Large
from vgg_pr import VGG_PR
from tensorflow.keras.callbacks import TensorBoard
import logging
import cv2

# 参数配置
img_size = (299,299)
# batch_size = 4
num_label = 20
total_epoch = 100
exp_thresh = [0.1e4,0.5e4,3e4]
# 编号
# case_num = 10

os.chdir(os.getcwd())

test_img_list = sorted(glob.glob('../dataset/test/intensity/*.mat'))[:100]
test_label_list = sorted(glob.glob('../dataset/test/phase/*.txt'))[:100]
ckpt_path = '../checkpoints/VGG-{epoch}.ckpt'.format(epoch=99)
result_path = '../result/single/'
if not os.path.exists(result_path):
    os.mkdir(result_path)

# read data
####################### 利用tf.data高级API进行数据读取 ########################
def read_img(filename):
    # print(filename)
    image_dict = loadmat(filename.decode('utf-8'))
    # process 3 降采样，可以提升batch_size
    exp_thresh = 1e4
    image_decoded = image_dict['Iz']
    image_decoded = cv2.resize(image_decoded, img_size, interpolation=cv2.INTER_AREA)
    image_decoded[image_decoded>exp_thresh] = exp_thresh
    image_decoded /= exp_thresh
    image_resized = np.float32(np.expand_dims(image_decoded, axis=-1))
    # image_resized = tf.convert_to_tensor(image_resized)
    return image_resized

def read_label(filename):
    label = open(filename).read()
    # print(label)
    label = label.strip().split(' ')
    label = [np.float32(i) for i in label if i!='']
    label = np.reshape(label, [1,1,-1])
    label = np.array(label) + 0.5  # 归一化
    # label = tf.convert_to_tensorf(label)
    return label

def parse_function(image_filename, label_filename):
    # print('--------------{}-------------------'.format(tf.as_string(image_filename))) # filename变成tensor了？
    # img1, img2, img3 = tf.numpy_function(read_multi_imgs, [image_filename1, image_filename2, image_filename3], tf.float32)
    img = tf.numpy_function(read_img, [image_filename], tf.float32)
    label = tf.numpy_function(read_label, [label_filename], tf.float32)
    return img, label

def predict():
    p_rmse = tf.metrics.Mean(name="prediction_rmse")
    logging.basicConfig(level=logging.INFO)
    test_dataset = tf.data.Dataset.from_tensor_slices((test_img_list, test_label_list))
    test_dataset = test_dataset.map(parse_function,3).batch(1)
    model = VGG_PR(num_classes=num_label)
    model.load_weights(ckpt_path)
    logging.info("model loaded")
    predictions = []
    for img, labels in test_dataset:
        prediction = model(img, training=False)
        predictions.append(prediction)
        if len(prediction.shape) == 2:
            pred = tf.reshape(prediction,[-1, 1, 1, num_label])
        print("predicted results: {}".format(prediction))
        rmse = tf.sqrt(tf.reduce_mean(tf.square(pred-labels)))
        p_rmse(rmse)
    print("Average rmse: {}".format(p_rmse.result()))
    np.savetxt(os.path.join(result_path, 'results.txt'), np.squeeze(np.array(predictions)), fmt='%.8f', delimiter=' ')

if __name__ == "__main__":
    predict()


#!/usr/bin/python3
# -*- coding:utf-8 -*-
from pyspark import SparkContext

import numpy as np
import csv
import sys
import time
import random

def swap(a, b) :
    return b, a

def init_cards(card_def, suitdef):
    cards = list()
    cards_str = list()
    a = 0
    b = 1
    sa = 0
    sb = -1
    for i in range(n) :
        if i % 13 == 0 :
            sa = 0
            sb += 1
            a += 100
            b = 1
        cards.append('%d.jpg' % (a + b))
        cards_str.append('%s%s' % (card_def[sa], suit_def[sb]))
        sa += 1
        b += 1
    return cards, cards_str

def gen_cards(cards) :
    n = 52
    for i in range(n) :
        j = random.randint(i, n - 1)
        cards[0][i], cards[0][j] = swap(cards[0][i], cards[0][j])
        cards[1][i], cards[1][j] = swap(cards[1][i], cards[1][j])
    return cards[0], cards[1]

if __name__ == '__main__' :

    logFile = "README.md"  # Should be some file on your system
    sc = SparkContext("local", "texas hold'em calc")

    n = 52
    card_def = ['A', '2', '3', '4', '5', '6', '7', '8', '9', 'T', 'J', 'Q', 'K']
    suit_def = ['s', 'h', 'c', 'd']

    csvfile = open('cards.csv', 'w')
    writer = csv.writer(csvfile)
    count = 100000

    if len(sys.argv) > 1 :
        count = int(sys.argv[1])

    print ('num of card game:', count)
    data = list()
    t = time.time()
    cards, cards_str = init_cards(card_def, suit_def)

    lst = [[cards, cards_str] for n in range(count)]
    distData = sc.parallelize(lst)

    ori_data = distData.map(gen_cards)
    data = ori_data.map(lambda a: a[1]).collect()

    print('time to gen data:', time.time() - t)
    t = time.time()
    print('len of data:', len(data))
    writer.writerows(data)
    print('time to write:', time.time() - t)
    csvfile.close()

    sc.stop()


import subprocess
import threading
import os,signal
import psutil
import random

""" Run system commands with timeout
"""
def kill(proc_pid):
    process = psutil.Process(proc_pid)
    for proc in process.children(recursive=True):
        proc.kill()
    process.kill()

class Command(object):
    def __init__(self, cmd):
        self.cmd = cmd
        self.process = None
        self.out = None

    def run_command(self, capture = False):
        if not capture:
            self.process = subprocess.Popen(self.cmd, shell=True)
            self.process.communicate()
            return
        # capturing the outputs of shell commands
        self.process = subprocess.Popen(self.cmd, stdout=subprocess.PIPE, shell=True)
        out,err = self.process.communicate()
        if len(out) > 0:
            self.out = out.splitlines()
        else:
            self.out = None

    # set default timeout to 2 minutes
    def run(self, capture = False, timeout = 120, filename=""):
        thread = threading.Thread(target=self.run_command, args=(capture,))
        thread.start()
        thread.join(timeout)
        if thread.is_alive():
            print ("Command timeout, kill it: " + self.cmd)
            if os.path.exists(filename):
            	os.remove(filename)
            # self.process.kill()
            kill(self.process.pid)
            thread.join()
        return self.out

'''basic test cases'''

# # run shell command without capture
# Command('pwd').run()
# # capture the output of a command
# date_time = Command('date').run(capture=True)
# print (date_time)

# Command('echo "sleep 10 seconds"; sleep 10; echo "done"').run(timeout=2)

import os

def findProbs(containingDir,pddlFiles):
    probFiles = []
    for other in pddlFiles:
        head, tail = os.path.split(other)
        probName = tail.replace(".pddl","")
        dkel = containingDir+"/"+probName+"-redop.dkel"
        if other.startswith(containingDir) and not os.path.isfile(dkel) and not other.endswith("domain.pddl"):
            probFiles.append(other)
    return probFiles

benchmark = "blocksworld-large-simple"

pddlFiles = []
# r=root, d=directories, f = files
for r, d, f in os.walk(benchmark):
    for file in f:
        if '.pddl' in file:
            pddlFiles.append(os.path.join(r, file))
redop = "../../hsps/pddlcat -redop -a"
prep_domain = "../../hsps/pddlcat -prep -no-problem"
prep_prob = "../../hsps/pddlcat -prep -no-domain"

random.shuffle(pddlFiles)

for f in pddlFiles:
    if f.endswith("domain.pddl"):
        domain = f
        containingDir = domain.replace("domain.pddl","")
        probFiles = findProbs(containingDir,pddlFiles)
        probFiles.sort()
        for prob in probFiles:
            print(prob)
            date_time = Command('date').run(capture=True)
            print (date_time)
            head, tail = os.path.split(prob)
            probName = tail.replace(".pddl","")
            dkel = containingDir+"/"+probName+"-redop.dkel"

            reduced_domain = containingDir+"/"+"reduced-"+probName+"-domain.pddl"
            reduced_prob = containingDir+"/"+"reduced-"+probName+".pddl"
            Command(redop+" "+domain+" "+prob+" > "+dkel).run(timeout=1000, filename=dkel)
            if os.path.isfile(dkel):
            	Command(prep_domain+" "+domain+" "+prob+" "+dkel+" > "+reduced_domain).run(timeout=300)
            	Command(prep_prob+" "+domain+" "+prob+" "+dkel+" > "+reduced_prob).run(timeout=300)
            else:
            	break


import pygame
from pygame.locals import *
import time

SIZE = 40

class Apple:
    def __init__(self, parent_screen):
        self.parent_screen = parent_screen
        self.block = pygame.image.load("resources/apple.jpg").convert()
        self.block_x = SIZE*3
        self.block_y = SIZE*3

    def draw_apple(self):
        self.parent_screen.blit(self.block, (self.block_x, self.block_y))
        pygame.display.flip()

class Snake:
    def __init__(self, parent_screen, length):
        self.parent_screen = parent_screen
        self.length = length
        self.block = pygame.image.load("resources/block.jpg").convert()
        self.block_x = [SIZE] * length
        self.block_y = [SIZE] * length
        self.direction = 'down'

    def draw_block(self):
        self.parent_screen.fill((200, 220, 0))

        # Draw the image to the screen at the given position.
        for i in range(self.length):
            self.parent_screen.blit(self.block, (self.block_x[i], self.block_y[i]))

        # display.flip() will update the contents of the entire display.
        # display.update() allows to update a portion of the screen,
        # instead of the entire area of the screen.
        # Passing no arguments, updates the entire display.
        pygame.display.flip()

    def move_left(self):
        self.direction = 'left'

    def move_right(self):
        self.direction = 'right'

    def move_up(self):
        self.direction = 'up'

    def move_down(self):
        self.direction = 'down'

    def set_direction(self):

        for i in range(self.length - 1, 0, -1):
            self.block_x[i] = self.block_x[i - 1]
            self.block_y[i] = self.block_y[i - 1]

        if self.direction == 'up':
            self.block_y[0] -= SIZE
            self.draw_block()

        if self.direction == 'down':
            self.block_y[0] += SIZE
            self.draw_block()

        if self.direction == 'left':
            self.block_x[0] -= SIZE
            self.draw_block()

        if self.direction == 'right':
            self.block_x[0] += SIZE
            self.draw_block()

class Game:
    def __init__(self):
        pygame.init()
        self.surface = pygame.display.set_mode((1000, 500))
        self.surface.fill((200, 220, 0))

        self.snake = Snake(self.surface, 5)
        self.snake.draw_block()

        self.apple = Apple(self.surface)
        self.apple.draw_apple()

    def play(self):
        self.snake.set_direction()
        self.apple.draw_apple()

    def run(self):
        running = True

        while running:
            for event in pygame.event.get():
                if event.type == KEYDOWN:
                    if event.key == K_ESCAPE:
                        running = False
                    if event.key == K_UP:
                        self.snake.move_up()
                    if event.key == K_DOWN:
                        self.snake.move_down()
                    if event.key == K_LEFT:
                        self.snake.move_left()
                    if event.key == K_RIGHT:
                        self.snake.move_right()
                elif event.type == QUIT:
                    running = False

            self.play()
            time.sleep(.3)

        pass

if __name__ == '__main__':
    game = Game()
    game.run()


from flask import Flask, request, jsonify
from pathlib import Path
import os

app = Flask(__name__)

@app.route('/api/get_content', methods=['POST'])

def ordenar():
    order = ""
    reverse_order = ""
    data = request.get_json(force=True)
    path = data['path']

    if 'order' in data.keys():
        order = str(data['order'])
    if 'reverse_order' in data.keys():
        reverse_order = str(data['reverse_order'])

    if order == "size" and reverse_order == "" or reverse_order == "False":
        j=0
        dic={}
        paths = sorted(Path(path).iterdir(), key=os.path.getsize)
        for i in paths:
            dic[j] = [str(i)]
            j+=1
        return jsonify(dic)

    elif order == "size" and reverse_order == "True":
        j=0
        dic={}
        paths = sorted(Path(path).iterdir(), key=os.path.getsize)
        paths.reverse()
        for i in paths:
            dic[j] = [str(i)]
            j+=1
        return jsonify(dic)

    elif order == "" and reverse_order == "True":
        j=0
        dic={}
        paths = sorted(Path(path).iterdir(), key=os.path.getmtime)
        paths.reverse()
        for i in paths:
            dic[j] = [str(i)]
            j+=1
        return jsonify(dic)

    else:
        j=0
        dic={}
        paths = sorted(Path(path).iterdir(), key=os.path.getmtime)
        for i in paths:
            dic[j] = [str(i)]
            j+=1
        return jsonify(dic)

@app.route('/', methods=['GET'])
def index():
    return jsonify({'Mensaje':'Viva Peru!'})

if __name__ == "__main__":
    app.run('0.0.0.0', 5000, debug=True)


import json
from django.core.management.base import BaseCommand

from meetup.models import (
    MeetupProgram,
    Stage,
    Block,
    Event
)

class Command(BaseCommand):
    def handle(self, *args, **options):
        meetups_filepath = 'meetups.json'

        with open(meetups_filepath, encoding='UTF-8', mode='r') as f:
            meetups = json.loads(f.read())
        for serialized_meetup in meetups:
            meetup, _ = MeetupProgram.objects.get_or_create(
                title=serialized_meetup['title'],
                defaults={
                    'date': serialized_meetup['date'],
                    'start_time': serialized_meetup['start_time'],
                    'end_time': serialized_meetup['end_time'],
                }
            )
            for serialized_stage in serialized_meetup['stages']:
                stage, _ = Stage.objects.get_or_create(
                    title=f'{serialized_stage["title"]}. {meetup.title}',
                    defaults={
                        'program': meetup,
                        'start_time': serialized_stage['start_time'],
                        'end_time': serialized_stage['end_time'],
                    }
                )
                for serialized_block in serialized_stage['blocks']:
                    block, _ = Block.objects.get_or_create(
                        title=f'{serialized_block["title"]}. {serialized_stage["title"]}',
                        defaults={
                            'stage': stage,
                            'start_time': serialized_block['start_time'],
                            'end_time': serialized_block['end_time'],
                        }
                    )
                    for serialized_event in serialized_block['events']:
                        event, _ = Event.objects.get_or_create(
                            title=f'{serialized_event["title"]}. {serialized_stage["title"]}',
                            defaults={
                                'block': block,
                                'start_time': serialized_event['start_time'],
                                'end_time': serialized_event['end_time'],
                            }
                        )


import numpy as np
from matplotlib import pyplot as plt
import random
from scipy.stats import binom
from collections import Counter
import seaborn as sns
random.seed(10)

def simulation(group_id, S_group, N_user, cap, T):
  user_id = []
  for id in group_id:
    U = range(id*S_group, (id+1)*S_group)
    user_id.append(np.random.choice(U))

  nn = np.array([0]*N_user)
  f = np.zeros((cap+1, T+1))
  for i in range(min(cap+1,T+1)):
    f[i,i] = (1.0/S_group)**i
  R = np.zeros((cap+1, T+1))

  for ti in range(1, T+1):
    R[:,ti] = R[:,ti-1]
    U = range(group_id[ti-1]*S_group, (group_id[ti-1]+1)*S_group)
    for i in U:
      nn[i] += 1
      for l in range(cap+1):
        j = nn[i]
        if j > l:
          if f[l,j] == 0:
            f[l,j] = f[l,j-1]*j/(j-l)*(1-1.0/S_group)
        for m in range(l+1, cap+1):
          R[m,ti] += (m-l)*(f[l,j-1]-f[l,j])
        #print (t, R[ti])

  reach_arrays = []
  for i in range(cap):
    c = i+1
    counter = Counter()
    reach = 0
    reach_array = []
    for user in user_id:
      counter[user] += 1
      if counter[user] <= c:
        reach += 1
      reach_array.append(reach)
    reach_arrays.append(reach_array)
  reach_arrays = np.array(reach_arrays)

  return R, reach_arrays

def macro_simulation(N_user = 120, S_group = 6, T = 500, cap = 3):
  N_group = N_user//S_group
  user_id = np.random.choice(N_user, T)
  group_id = user_id//S_group
  Rs = []
  reaches = []
  for i in range(2000):
    R, reach = simulation(group_id, S_group, N_user, cap, T)
    Rs.append(R[:,1:])
    reaches.append(reach)
  return Rs, reaches

T = 180
S_groups = [1,2,3,4,5,6,8,10,12,20,40,60,120]
Rss = []
reachess = []
for S in S_groups:
  Rs, reaches = macro_simulation(S_group = S, T = T)
  Rss.append(Rs)
  reachess.append(reaches)

reachess = np.array(reachess)

rvars = np.var(reachess, axis = 1)
reach_vars = []
for i,S in enumerate(S_groups):
  reach_vars.append(rvars[i,-1,-1])

plt.rc('font', size=16)
res = []
groups = []
for i in [0,1,3,8,10,12]:
  res.append(reach_vars[i])
  groups.append(S_groups[i])
  print (S_groups[i])
fig, ax = plt.subplots()
plt.plot(groups, res, '-o')
plt.ylabel("variance")
plt.xlabel("group size (k)")
plt.savefig('figure2c.svg')


import torch
import torch.nn as nn

class Discriminiator(nn.Module):
    def __init__(self, in_filters=128, model_type='wrn'):
        super(Discriminiator, self).__init__()
        if model_type == 'wrn':
            self.out_filters = 64  # WRN[B, 128, 8 ,8]
        elif model_type == 'res':  # # 由于ResNet的layer4=>[Batch_size, 2048, 4, 4]
            self.out_filters = 256

        self.model = nn.Sequential(
            nn.Conv2d(in_filters, self.out_filters, kernel_size=3, padding=0, stride=2),
            nn.BatchNorm2d(self.out_filters),
            nn.LeakyReLU(0.2, inplace=True),
            nn.Conv2d(self.out_filters, 1, kernel_size=1, padding=0, stride=1),
            nn.Sigmoid(),
        )

    def forward(self, img):
        validity = self.model(img)
        return validity

def test():
    discriminator = Discriminiator(in_filters=128)
    y = discriminator(torch.randn(1, 128, 8, 8))
    # y = discriminator(torch.randn(1, 3, 32, 32))
    print(y.size())  # 1, 1, 7, 7

# test()


import random
import sys
from nltk.translate.bleu_score import sentence_bleu, SmoothingFunction

def main(candidate_file):
	references = []
	with open("save/str_real_data.txt") as fin:
		for line in fin:
			line = line.split()
			references.append(line)
	random.shuffle(references)
	references = references[:1000]

	candidates = []
	with open(candidate_file)as fin:
		for line in fin:
			line = line.split()
			candidates.append(line)

	chencherry = SmoothingFunction()

	weights = [(1, 0, 0, 0), (0.5, 0.5, 0, 0), (0.33, 0.33, 0.33, 0), (0.25, 0.25, 0.25, 0.25)]

	#for i in range(len(weights)):
		#weight = weights[i]
	for ngram in range(2, 6):
		weight = tuple((1. / ngram for _ in range(ngram)))
		scores = []
		for candidate in candidates:
			score = sentence_bleu(references, candidate, weights=weight, smoothing_function=chencherry.method1)
			scores.append(score)
		print(f"{len(weight)}-gram: ", sum(scores) / len(scores))

if __name__ == "__main__":
	main(sys.argv[1])


import argparse
import random
import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from torch.utils.data import DataLoader
from torchvision import datasets, transforms, utils
from tqdm import tqdm
from pathlib import Path
from model import BinarizeMnist, VAE_SingleLayer, VAE_TwoLayer
import os
os.environ["CUDA_VISIBLE_DEVICES"] = "0"

def main(args):
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    model = VAE_TwoLayer()
    model.to(device)
    checkpoint = torch.load(args.resume, map_location=lambda storage, loc: storage)
    model.load_state_dict(checkpoint["model"], strict=False)
    transform_test= transforms.Compose([
        transforms.ToTensor(),
        BinarizeMnist()
    ])
    test_dataset = datasets.MNIST("./MNIST", train=False, download=True,
                                    transform=transform_test)
    test_dataloader = DataLoader(test_dataset, batch_size=1, shuffle=False, drop_last=True,
                                 num_workers=args.num_workers, pin_memory=True)
    N = 1 * 28 * 28
    model.eval()
    average_elbo_favi = average_elbo_naive = average_eblo_appro = average_eblo_accur = 0
    valid_i = 1
    for i, (images, _) in enumerate(test_dataloader, 1):
        images = images.to(device)
        elbo_favi, elbo_naive, elbo_appro, elbo_accur = model.eval_savi(images, args.num_iter, args.learning_rate, "sgvb2", 3470)
        if torch.isnan(elbo_favi).any() or torch.isnan(elbo_naive).any() or torch.isnan(elbo_appro).any() or torch.isnan(elbo_accur).any():
            # ignore nan sample at this time
            continue
        valid_i += 1
        elbo_favi = torch.mean(elbo_favi / N, dim=0)
        elbo_naive = torch.mean(elbo_naive / N, dim=0)
        elbo_appro = torch.mean(elbo_appro / N, dim=0)
        elbo_accur = torch.mean(elbo_accur / N, dim=0)
        average_elbo_favi += elbo_favi.item()
        average_elbo_naive += elbo_naive.item()
        average_eblo_appro += elbo_appro.item()
        average_eblo_accur += elbo_accur.item()
        if i == 500:
            break
    average_elbo_favi /= valid_i
    average_elbo_naive /= valid_i
    average_eblo_appro /= valid_i
    average_eblo_accur /= valid_i
    print("[test] elbo favi: {:.10f}, elbo naive: {:.10f}, elbo approx: {:.10f}, elbo accur: {:.10f}"
        .format(average_elbo_favi * N, average_elbo_naive * N, average_eblo_appro * N, average_eblo_accur * N))

if __name__ == "__main__":

    SEED=3470
    random.seed(SEED)
    np.random.seed(SEED)
    torch.manual_seed(SEED)
    torch.cuda.manual_seed(SEED)
    torch.backends.cudnn.deterministic=True
    parser = argparse.ArgumentParser()
    parser.add_argument("--num-workers", type=int, default=4, help="Number of dataloader workers.")
    parser.add_argument("--num-iter", type=int, default=50, help="Number of training steps.")
    parser.add_argument("--learning-rate", type=float, default=1e-3, help="Learning rate.")
    parser.add_argument("--resume", type=str, default="", help="model load path.")
    args = parser.parse_args()
    main(args)

5.1. Evaluation on Density Estimation/infer.py


import s_cmd
from pprint import pprint
import ngit
import argparse

# #### ##### ##### #####

def info(msg):
    print("[INFO] {}".format(msg))

def arg_parse():
    parser = argparse.ArgumentParser(description='deploy from branch')
    parser.add_argument('-b', metavar="master", help='branch name to deploy', required=True)
    args = parser.parse_args()
#    print(args)
    return args

# #### ##### ##### #####

def main():

    # init
    git_url = "https://github.com/naihsi/test-server.git"

    # arg parse
    _branch_name = arg_parse().b

    # 1. fetch information from git
    _url = git_url
    _branches = ngit.get_branch_info(_url)

    if _branch_name not in _branches:
        info("[INFO] branch {} not in the git".format(_branch_name))
        return

    if s_cmd.pass_dryrun(s_cmd.create_inst, (_branch_name, True)):
        _inst_id = s_cmd.create_inst(_branch_name)
        s_cmd.wait_for("running", _inst_id)
        info("[INFO] {} was launched for branch {}".format(_inst_id, _branch_name))

if __name__ == "__main__":
    main()


from pathlib import Path
import json
import collections
import pandas as pd
import numpy as np
import datetime
import csv

from experiment_utils import find_recent_file

def flatten_dict(d, parent_key='', sep='_'):
    items = []
    for k, v in d.items():
        new_key = parent_key + sep + k if parent_key else k
        if isinstance(v, collections.MutableMapping):
            items.extend(flatten_dict(v, new_key, sep=sep).items())
        else:
            items.append((new_key, v))
    return dict(items)

def read_json(path, prefix, vals=None, flat=True):
    """
    Reads the latest file in path that starts with the prefix, returning a dictionary of the values.

    :param path: folder path, the target file is the most recent file in this folder
                named "<prefix>*.json"
    :param vals: list of the values to read from the file.
    :return: a dictionary of values from the latest experiment parameter file
    """

    params_file = find_recent_file(path, prefix)
    if params_file == -1:
        # no file found
        return {x: None for x in vals}
    with open(str(params_file), 'r') as file:
        params = json.load(file)
    if vals:
        params = {x: params[x] for x in vals}
    if flat:
        params = flatten_dict(params)
    return params

def read_experiment_params(path, vals=None, flat=True):
    """
    Reads the latest experiment parameter file in path, and returns a dictionary of the values.

    :param path: folder path, the experiment parameter file is in this folder and is named
            "experiment_params_<timestamp>.json"
    :param vals: list of the values to read from the file.
    :param flat: if true, then will flatten out nested dicts
    :return: a dictionary of values from the latest experiment parameter file
    """

    if vals is None:
        vals = ['debug', 'name', 'experiment_number', 'dataset', 'model_type',
               'model_path', 'best_model_metric', 'quantization', 'prune_method',
               'prune_compression', 'finetune_epochs', 'attack_method', 'attack_kwargs',
               'email_verbose', 'gpu', 'save_one_checkpoint', 'seed', 'train_from_scratch',
               'train_kwargs', 'prune_kwargs']

    params = read_json(path=path, prefix="experiment_params", vals=vals, flat=flat)

    return params

def read_pruning_metrics(path, vals=None, flat=True):
    if vals is None:

        vals = ['size', 'size_nz', 'compression_ratio', 'flops',
                'flops_nz', 'theoretical_speedup', 'loss']

    prune_folder = str(Path(path) / "prune")
    params = read_json(path=prune_folder, prefix="metrics", vals=vals, flat=flat)

    # # rename args
    # if "size" in params:
    #     params["model_size"] = params["size"]
    #     params.pop("size")
    # else:
    #     params["model_size"] = None
    # if "size_nz" in params:
    #     params["model_size_nz"] = params["size_nz"]
    #     params.pop("size_nz")
    # else:
    #     params["model_size_nz"] = None

    return params

def read_attack_metrics(path, vals=None, flat=True, attack='pgd'):
    if vals is None:
        vals = ['results']
    attack_folder = str(Path(path) / "attacks" / attack)
    params = read_json(path=attack_folder, prefix="train_attack_results", vals=vals, flat=flat)

    # # format args
    # if "results" in params:
    #     if "adv_acc1" in params["results"]:
    #         params["adv_acc1"] = params["results"]["adv_acc1"]
    #     if "adv_acc5" in params["results"]:
    #         params["adv_acc5"] = params["results"]["adv_acc5"]
    #     params.pop("results")
    # else:
    #     params["adv_acc1"] = None
    #     params["adv_acc5"] = None
    #
    # if flat:
    #     params = flatten_dict(params)
    return params

def read_transfer_attack_metrics(path, vals=None, flat=True, attack='pgd'):
    if vals is None:
        vals = ['transfer_results', 'transfer_model_path']
    attack_folder = str(Path(path) / "attacks" / attack / "transfer_attack")
    params = read_json(path=attack_folder, prefix="train_attack_results", vals=vals, flat=flat)

    # # format args
    # if params"transfer_results" is not None:
    #     if "transfer_runtime" in params["transfer_results"]:
    #         params["transfer_results"].pop("transfer_runtime")
    # elif params["transfer_results"] is None:
    #     params["transfer_results"] = {
    #                         "inputs_tested": None,
    #                         "both_correct1": None,
    #                         "both_correct5": None,
    #                         "transfer_correct1": None,
    #                         "transfer_correct5": None,
    #                         "target_correct1": None,
    #                         "target_correct5": None,
    #                     }
    #
    # if flat:
    #     params = flatten_dict(params)
    return params

def read_accuracy_results(path, vals=None):
    """If model has been pruned, read from pruning logs, else from training logs."""

    if vals is None:
        vals = ['train_acc1', 'train_acc5', 'val_acc1', 'val_acc5']

    acc_path = Path(path) / "prune"
    if not acc_path.exists():
        acc_path = Path(path) / "train"
        if not acc_path.exists():
            raise FileNotFoundError(f"Could not find train or prune folder in {path}")

    csv_file = find_recent_file(acc_path, "logs")
    df = pd.read_csv(csv_file, usecols=vals)
    # this assumes no best_model_metric
    params = df.iloc[-1]
    params = params.to_dict()

    return {f"clean_{x}": params[x] for x in params}

def collect_one_experiment_data(path):
    experiment_params = read_experiment_params(path)
    pruning_metrics = read_pruning_metrics(path)
    attack_metrics = read_attack_metrics(path, attack=experiment_params['attack_method'])
    transfer_attack_metrics = read_transfer_attack_metrics(path, attack=experiment_params['attack_method'])
    accuracy_results = read_accuracy_results(path)

    results = {
        **experiment_params,
        **pruning_metrics,
        **attack_metrics,
        **transfer_attack_metrics,
        **accuracy_results,
    }
    return results

def find_experiment_folders(exp_folder, models, dataset='CIFAR10'):

    experiment_folders = []

    for model in models:
        model_path = exp_folder / model / dataset
        experiment_folders.extend(list(model_path.glob(f"{model}*")))

    return experiment_folders

def generate_csv(experiment_number, models=None, timestamp=True):
    if models is None:
        models = ['resnet20', 'googlenet', 'vgg_bn_drop', 'mobilenet_v2']

    experiment_x = Path.cwd() /"experiments" / f"experiment_{experiment_number}"
    experiment_folders = find_experiment_folders(experiment_x, models)

    csv_name = "collected_results.csv"
    if timestamp:
        timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
        csv_name = f"collected_results_{timestamp}.csv"
    csv_path = experiment_x / csv_name

    # want ordered & unique data type
    headers = {}
    data = []
    for exp in experiment_folders:
        results = collect_one_experiment_data(exp)
        headers.update({x:None for x in results})
        data.append(results)

    with open(csv_path, 'w') as file:
        writer = csv.DictWriter(file, fieldnames=headers.keys(), lineterminator='\n')
        writer.writeheader()

        for x in data:
            writer.writerow(x)

def generate_csv_new_workflow(experiment_number, models=None, timestamp=True):
    if models is None:
        models = ['resnet20', 'googlenet', 'vgg_bn_drop', 'mobilenet_v2']

    experiment_x = Path.cwd() /"experiments" / f"experiment_{experiment_number}"
    experiment_folders = find_experiment_folders(experiment_x, models)

    csv_name = "collected_results.csv"
    if timestamp:
        timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
        csv_name = f"collected_results_{timestamp}.csv"
    csv_path = experiment_x / csv_name

    # want ordered & unique data type
    headers = {}
    data = []
    for exp in experiment_folders:
        results = read_json(exp, prefix='experiment_results_', flat=True)
        headers.update({x:None for x in results})
        data.append(results)

    # make sure all data cases have the same headers
    for x in data:
        for key in headers:
            if key not in x:
                x[key] = None

    with open(csv_path, 'w') as file:
        writer = csv.DictWriter(file, fieldnames=headers.keys(), lineterminator='\n')
        writer.writeheader()

        for x in data:
            writer.writerow(x)

    return csv_path

def fill_empty_csv(experiment, path=None):
    """
    Fills in empty prune attack parameters and prune/quantized transfer attacks for
    models which were not pruned.  The transfer attack results will simply be the identity
    function.

    :param experiment:
    :param path:
    :return:
    """

    experiment_folder = Path.cwd() / 'experiments' / f'experiment_{experiment}'

    if path is None:
        path = find_recent_file(experiment_folder, prefix='collected_results')

    df = pd.read_csv(path)

    convert_results = [
        "clean_train_acc1",
        "clean_train_acc5",
        "clean_val_acc1",
        "clean_val_acc5",
        "model_size",
        "model_size_nz",
        "compression_ratio",
        "flops",
        "flops_nz",
        "theoretical_speedup",
        "model_file_size",
        "attack_results_inputs_tested",
        "attack_results_clean_acc1",
        "attack_results_clean_acc5",
        "attack_results_adv_acc1",
        "attack_results_adv_acc5",
        "attack_results_runtime"
    ]

    df_train_scratch = df[df['prune_model_file_size'].isna()]
    for x in convert_results:
        df[f'prune_{x}'] = np.where(df[f'prune_{x}'].isna(), df[f'train_{x}'], df[f'prune_{x}'])

    more_convert_results = [
        ("attack_transfer_both_correct1", "attack_results_adv_acc1"),
        ("attack_transfer_both_correct5", "attack_results_adv_acc5"),
        ("attack_transfer_transfer_correct1", "attack_results_adv_acc1"),
        ("attack_transfer_transfer_correct5", "attack_results_adv_acc5"),
        ("attack_transfer_target_correct1", "attack_results_adv_acc1"),
        ("attack_transfer_target_correct5", "attack_results_adv_acc5"),
    ]

    # columns attack_transfer_transfer_model, attack_transfer_transfer_runtime, and
    #   attack_transfer_inputs_tested are left blank

    for prefix_empty, prefix_replace in [('prune_', 'train_'), ('quantize_', 'quantize_')]:
        for empty, replace in more_convert_results:
            df[f'{prefix_empty}{empty}'] = np.where(
                df[f'{prefix_empty}{empty}'].isna(),
                df[f'{prefix_replace}{replace}'],
                df[f'{prefix_empty}{empty}']
            )

    new_csv = path.parent / f"formatted_{path.name}"
    # with open(new_csv, 'w') as f:
    df.to_csv(new_csv, index=False)
    return new_csv

def retrieve_formatted_csv_df(experiment, cols=None, regenerate=True):
    """
    If regenerate is true, regenerates the files and removes old ones.
    :param experiment:
    :param cols:
    :param replace:
    :return:
    """
    experiment_folder = Path.cwd() / 'experiments' / f'experiment_{experiment}'
    if not regenerate:
        raw_results = find_recent_file(experiment_folder, prefix='collected_results')
        if isinstance(raw_results, int):
            generate_csv_new_workflow(experiment)
        file = find_recent_file(experiment_folder, prefix='formatted_collected_results')
        if isinstance(file, int):
            file = fill_empty_csv(experiment)
    else:
        csvs = list(experiment_folder.glob('*.csv'))
        for f in csvs:
            f.unlink()
        generate_csv_new_workflow(experiment)
        file = fill_empty_csv(experiment)

    return pd.read_csv(file, usecols=cols)

def format_data_for_plot(experiment, metrics, pruning=None, models=None):
    """
    Returns a tuple ({model: groupby dataframe}, {model: big dataframe})

    The first item in the dict is for easy plotting, the second is for combining
    with other experiments

    Can plot the groupby dataframe using

    for pruning_strat, data in groupby_dataframe:
        x_axis = data['prune_compression']
        y_axis = data[metric]
        label = pruning_strat
        plt.plot(x_axis, y_axis, label=label)

    :param experiment: which experiment to get the data from
    :param metrics: list of columns which you want to plot on y axis.
    :param pruning: list of pruning strategies on which collect data
    :param models: list of models on which to collect data
    :return:
    """
    if not isinstance(metrics, list):
        metrics = [metrics]

    if models is None:
        models = ['resnet20', 'googlenet', 'vgg_bn_drop', 'mobilenet_v2']

    if pruning is None:
        pruning = ["RandomPruning", "GlobalMagWeight", "LayerMagWeight", "GlobalMagGrad", "LayerMagGrad",
            "GreedyPGDGlobalMagGrad", "GreedyPGDLayerMagGrad"]

    cols = [
        'name',
        'model_type',
        'prune_method',
        'prune_compression',
    ]
    cols.extend(metrics)
    df_orig = retrieve_formatted_csv_df(experiment, cols=cols)

    final_data = {}
    ungrouped_data = {}

    for model in models:
        df = df_orig[df_orig['model_type'] == model]
        zero_compression_metrics = {}
        for metric in metrics:
            zero_compression_metrics[metric] = df[df['name'] == model][metric].item()
        temp = {**zero_compression_metrics, 'prune_compression': 1}
        for prune_strat in pruning:
            df = df.append({**temp, 'prune_method': prune_strat}, ignore_index=True)
        df = df.sort_values('prune_compression')
        ungrouped_data[model] = df
        final_data[model] = df.groupby(['prune_method'])
        # for key, group in a:
        #     print(key)
        #     print("\n\n\n")
        #     print(group)
    return final_data, ungrouped_data

def format_multiple_experiment_data_for_plot(experiments, metrics, pruning=None, models=None):
    """
    Combines the data from multiple experiment runs into one result of the same format
    as format_data_for_plot(), except that the metrics columns are the averages of the
    results from each experiment, and there is an additional column for each metric which
    is the standard deviation for that metric.

    :param experiments:
    :param metrics:
    :param pruning:
    :param models:
    :return:
    """
    if not isinstance(experiments, list):
        experiments = [experiments]

    if not isinstance(metrics, list):
        metrics = [metrics]

    if models is None:
        models = ['resnet20', 'googlenet', 'vgg_bn_drop', 'mobilenet_v2']

    experiment_data = []
    for experiment_x in experiments:
        experiment_data.append(
            format_data_for_plot(
                experiment=experiment_x,
                metrics=metrics,
                pruning=pruning,
                models=models
            )[1]
        )
        a= 0

    # combine model dataframes
    model_dfs = {}

    for model in models:
        temp = pd.concat([x[model] for x in experiment_data])

        model_dfs[model] = temp.groupby(
            ['prune_method', 'prune_compression'],
            as_index=False,
        ).agg(
            {metrc:['mean', 'std'] for metrc in metrics}
        ).sort_values('prune_compression')

    return model_dfs

if __name__ == '__main__':
    #generate_csv_new_workflow(10)
    # clean_prune_acc(10)
    format_multiple_experiment_data_for_plot(
        experiments=[10,11, 12],
        metrics=['prune_clean_val_acc1', 'prune_attack_results_adv_acc1', 'prune_attack_transfer_target_correct1']
    )
    #'prune_attack_transfer_target_correct_1', 'quantize_attack_transfer_target_correct_1',
    # 'quantize_attack_results_adv_acc1'


from matplotlib import colors
import matplotlib.pyplot as plt
import pandas as pd

array = pd.read_csv("DS9.txt", delimiter=' ',names=['x', 'y'])
graphic = plt.figure(figsize=[5.4, 9.6])
ax = graphic.add_subplot(111)
ax.set_title("Data Set 9")
ax.set_ylabel("Second column")
ax.set_xlabel("First column")
ax.scatter(array.y, array.x, c=["#DD9787"], s=1)
graphic.savefig("DS9 picture.png")
plt.show()


from Eurecom import *
from tdlbp import *
import argparse
from helper.functions import sendEmailMessage, standartParametrization

if __name__ == '__main__':

    parser = argparse.ArgumentParser(description='Process and extract FRGC database')
    parser = standartParametrization(parser)
    args = parser.parse_args()

    if not os.path.exists(args.pathImages):
        os.makedirs(args.pathImages)

    faceDataset = []
    sets = ['s1','s2']
    for s in sets:
        ek = EurecomKinect(args.pathdatabase,s,args.typeoffile,args.faceVariation.split('_'))
        ek.feedTemplates()

        if args.loadNewDepth:
            ek.loadNewDepthImage()
        elif args.loadSymmImages:
            ek.loadSymmFilledImages()

        if args.angles:
            ek.loadRotatedFaces(args.angles.split('_'),args.axis.split('_'))

        faceDataset.append(ek)

    tdlbp = ThreeDLBP(8,14,faceDataset,generateImages=args.generateImages)
    tdlbp.fullPathGallFile = args.pathImages

    if not args.steps is None:
        ppSteps = args.steps.split('_')
        for p in ppSteps:
            className = None
            parameters = None
            kwargsList = None
            if ':' in p:
                parameters = p.split(':')
                className = parameters[0]
                parameters = parameters[1].split(',')
                kwargsList = {}
                for pr in parameters:
                    lParameters = pr.split('=')
                    kwargsList[lParameters[0]] = eval(lParameters[1])

            else:
                className = p

            module = __import__(className)
            class_ = getattr(module,className)
            if kwargsList is None:
                tdlbp.preProcessingSteps = class_()
            else:
                tdlbp.preProcessingSteps = class_(**kwargsList)

    if args.operation in ['both','pp']:
        tdlbp.preProcessing(True,args.parcal)
        sendEmailMessage('Fim do pre-processamento', 'Terminou o pre-processamento EURECOM e LBP')
        #xek.saveTemplateImage()

    if args.operation in ['both', 'fe']:
        tdlbp.featureExtraction(args.points,args.radius,args.parcal,layersUtilize=[1,2,3,4],typeMeasurement=args.typeMeasure,procs=args.quantityProcesses,masks=args.generateMasks,forceImage=args.force,deformValue=args.deformValue)
        sendEmailMessage('Fim dos experimentos','Terminou a extração de características EURECOM e LBP')

    if (args.pathtrainingfile is not None) and (args.operation == 'fe'):
        faceVariationGenerate = { k : ['s1','s2'] for k in args.faceVariation.split('_') }
        galeryData = faceDataset[0].generateDatabaseFile(args.pathtrainingfile,faceVariationGenerate,[faceDataset[1]],'SVMTorchFormat')


from django.urls import path
from . import views
from .views import Blogdetail, Bloghome
app_name='index'
urlpatterns = [
    # path('login/', views.login, name='login'),
    # path('register/', views.register, name='register'),
    path('', views.home, name='home'),
    path('cart/', views.cart, name='cart'),
    path('checkout/', views.checkout, name='checkout'),
    path('updateitem/', views.updateitem, name='updateitem'),
    path('s_and_t/', views.s_and_tlist, name='s_and_t'),
    path('blog/', views.blogger, name='blogger'),
    path('bloghome/', Bloghome.as_view(), name='bloghome'),
    path('article/<int:pk>', Blogdetail.as_view(), name='blogdetail'),
    path('services/', views.services, name='services'),
    path('tools/', views.tools, name='tools')
]