import numpy as np
import tensorflow as tf
from scipy.stats import multivariate_normal as normal

class Equation(object):
    """Base class for defining PDE related function."""

    def __init__(self, dim, total_time, num_time_interval):
        self._dim = dim
        self._total_time = total_time
        self._num_time_interval = num_time_interval
        self._delta_t = (self._total_time + 0.0) / self._num_time_interval
        self._sqrt_delta_t = np.sqrt(self._delta_t)
        self._y_init = None

    def sample(self, num_sample):
        """Sample forward SDE."""
        raise NotImplementedError

    def f_tf(self, t, x, y, z):
        """Generator function in the PDE."""
        raise NotImplementedError

    def g_tf(self, t, x):
        """Terminal condition of the PDE."""
        raise NotImplementedError

    @property
    def y_init(self):
        return self._y_init

    @property
    def dim(self):
        return self._dim

    @property
    def num_time_interval(self):
        return self._num_time_interval

    @property
    def total_time(self):
        return self._total_time

    @property
    def delta_t(self):
        return self._delta_t

def get_equation(name, dim, total_time, num_time_interval):
    try:
        return globals()[name](dim, total_time, num_time_interval)
    except KeyError:
        raise KeyError("Equation for the required problem not found.")

class AllenCahn(Equation):
    def __init__(self, dim, total_time, num_time_interval):
        super(AllenCahn, self).__init__(dim, total_time, num_time_interval)
        self._x_init = np.zeros(self._dim)
        self._sigma = np.sqrt(2.0)

    def sample(self, num_sample):
        dw_sample = normal.rvs(size=[num_sample,
                                     self._dim,
                                     self._num_time_interval]) * self._sqrt_delta_t
        x_sample = np.zeros([num_sample, self._dim, self._num_time_interval + 1])
        x_sample[:, :, 0] = np.ones([num_sample, self._dim]) * self._x_init
        for i in xrange(self._num_time_interval):
            x_sample[:, :, i + 1] = x_sample[:, :, i] + self._sigma * dw_sample[:, :, i]
        return dw_sample, x_sample

    def f_tf(self, t, x, y, z):
        return y - tf.pow(y, 3)

    def g_tf(self, t, x):
        return 0.5 / (1 + 0.2 * tf.reduce_sum(tf.square(x), 1, keep_dims=True))

class HJB(Equation):
    def __init__(self, dim, total_time, num_time_interval):
        super(HJB, self).__init__(dim, total_time, num_time_interval)
        self._x_init = np.zeros(self._dim)
        self._sigma = np.sqrt(2.0)
        self._lambda = 1.0

    def sample(self, num_sample):
        dw_sample = normal.rvs(size=[num_sample,
                                     self._dim,
                                     self._num_time_interval]) * self._sqrt_delta_t
        x_sample = np.zeros([num_sample, self._dim, self._num_time_interval + 1])
        x_sample[:, :, 0] = np.ones([num_sample, self._dim]) * self._x_init
        for i in xrange(self._num_time_interval):
            x_sample[:, :, i + 1] = x_sample[:, :, i] + self._sigma * dw_sample[:, :, i]
        return dw_sample, x_sample

    def f_tf(self, t, x, y, z):
        return -self._lambda * tf.reduce_sum(tf.square(z), 1, keep_dims=True)

    def g_tf(self, t, x):
        return tf.log((1 + tf.reduce_sum(tf.square(x), 1, keep_dims=True)) / 2)

class PricingOption(Equation):
    def __init__(self, dim, total_time, num_time_interval):
        super(PricingOption, self).__init__(dim, total_time, num_time_interval)
        self._x_init = np.ones(self._dim) * 100
        self._sigma = 0.2
        self._mu_bar = 0.06
        self._rl = 0.04
        self._rb = 0.06
        self._alpha = 1.0 / self._dim

    def sample(self, num_sample):
        dw_sample = normal.rvs(size=[num_sample,
                                     self._dim,
                                     self._num_time_interval]) * self._sqrt_delta_t
        x_sample = np.zeros([num_sample, self._dim, self._num_time_interval + 1])
        x_sample[:, :, 0] = np.ones([num_sample, self._dim]) * self._x_init
        # for i in xrange(self._n_time):
        # 	x_sample[:, :, i + 1] = (1 + self._mu_bar * self._delta_t) * x_sample[:, :, i] + (
        # 		self._sigma * x_sample[:, :, i] * dw_sample[:, :, i])
        factor = np.exp((self._mu_bar-(self._sigma**2)/2)*self._delta_t)
        for i in xrange(self._num_time_interval):
            x_sample[:, :, i + 1] = (factor * np.exp(self._sigma * dw_sample[:, :, i])) * x_sample[:, :, i]
        return dw_sample, x_sample

    def f_tf(self, t, x, y, z):
        temp = tf.reduce_sum(z, 1, keep_dims=True) / self._sigma
        return -self._rl * y - (self._mu_bar - self._rl) * temp + (
            (self._rb - self._rl) * tf.maximum(temp - y, 0))

    def g_tf(self, t, x):
        temp = tf.reduce_max(x, 1, keep_dims=True)
        return tf.maximum(temp - 120, 0) - 2 * tf.maximum(temp - 150, 0)

class PricingDefaultRisk(Equation):
    def __init__(self, dim, total_time, num_time_interval):
        super(PricingDefaultRisk, self).__init__(dim, total_time, num_time_interval)
        self._x_init = np.ones(self._dim) * 100.0
        self._sigma = 0.2
        self._rate = 0.02   # interest rate R
        self._delta = 2.0 / 3
        self._gammah = 0.2
        self._gammal = 0.02
        self._mu_bar = 0.02
        self._vh = 50.0
        self._vl = 70.0
        self._slope = (self._gammah - self._gammal) / (self._vh - self._vl)

    def sample(self, num_sample):
        dw_sample = normal.rvs(size=[num_sample,
                                     self._dim,
                                     self._num_time_interval]) * self._sqrt_delta_t
        x_sample = np.zeros([num_sample, self._dim, self._num_time_interval + 1])
        x_sample[:, :, 0] = np.ones([num_sample, self._dim]) * self._x_init
        for i in xrange(self._num_time_interval):
            x_sample[:, :, i + 1] = (1 + self._mu_bar * self._delta_t) * x_sample[:, :, i] + (
                self._sigma * x_sample[:, :, i] * dw_sample[:, :, i])
        return dw_sample, x_sample

    def f_tf(self, t, x, y, z):
        piecewise_linear = tf.nn.relu(
            tf.nn.relu(y - self._vh) * self._slope + self._gammah - self._gammal) + self._gammal
        return (-(1 - self._delta) * piecewise_linear - self._rate) * y

    def g_tf(self, t, x):
        return tf.reduce_min(x, 1, keep_dims=True)

class BurgesType(Equation):
    def __init__(self, dim, total_time, num_time_interval):
        super(BurgesType, self).__init__(dim, total_time, num_time_interval)
        self._x_init = np.zeros(self._dim)
        self._y_init = 1 - 1.0 / (1 + np.exp(0 + np.sum(self._x_init) / self._dim))
        self._sigma = self._dim + 0.0

    def sample(self, num_sample):
        dw_sample = normal.rvs(size=[num_sample,
                                     self._dim,
                                     self._num_time_interval]) * self._sqrt_delta_t
        x_sample = np.zeros([num_sample, self._dim, self._num_time_interval + 1])
        x_sample[:, :, 0] = np.ones([num_sample, self._dim]) * self._x_init
        for i in xrange(self._num_time_interval):
            x_sample[:, :, i + 1] = x_sample[:, :, i] + self._sigma * dw_sample[:, :, i]
        return dw_sample, x_sample

    def f_tf(self, t, x, y, z):
        return (y - (2 + self._dim) / 2.0 / self._dim) * tf.reduce_sum(z, 1, keep_dims=True)

    def g_tf(self, t, x):
        return 1 - 1.0 / (1 + tf.exp(t + tf.reduce_sum(x, 1, keep_dims=True) / self._dim))

class QuadraticGradients(Equation):
    def __init__(self, dim, total_time, num_time_interval):
        super(QuadraticGradients, self).__init__(dim, total_time, num_time_interval)
        self._alpha = 0.4
        self._x_init = np.zeros(self._dim)
        base = self._total_time + np.sum(np.square(self._x_init) / self._dim)
        self._y_init = np.sin(np.power(base, self._alpha))

    def sample(self, num_sample):
        dw_sample = normal.rvs(size=[num_sample,
                                     self._dim,
                                     self._num_time_interval]) * self._sqrt_delta_t
        x_sample = np.zeros([num_sample, self._dim, self._num_time_interval + 1])
        x_sample[:, :, 0] = np.ones([num_sample, self._dim]) * self._x_init
        for i in xrange(self._num_time_interval):
            x_sample[:, :, i + 1] = x_sample[:, :, i] + dw_sample[:, :, i]
        return dw_sample, x_sample

    def f_tf(self, t, x, y, z):
        x_square = tf.reduce_sum(tf.square(x), 1, keep_dims=True)
        base = self._total_time - t + x_square / self._dim
        base_alpha = tf.pow(base, self._alpha)
        derivative = self._alpha * tf.pow(base, self._alpha - 1) * tf.cos(base_alpha)
        term1 = tf.reduce_sum(tf.square(z), 1, keep_dims=True)
        term2 = -4.0 * (derivative ** 2) * x_square / (self._dim ** 2)
        term3 = derivative
        term4 = -0.5 * (
            2.0 * derivative + 4.0 / (self._dim ** 2) * x_square * self._alpha * (
                (self._alpha - 1) * tf.pow(base, self._alpha - 2) * tf.cos(base_alpha) - (
                    self._alpha * tf.pow(base, 2 * self._alpha - 2) * tf.sin(base_alpha)
                    )
                )
            )
        return term1 + term2 + term3 + term4

    def g_tf(self, t, x):
        return tf.sin(
            tf.pow(tf.reduce_sum(tf.square(x), 1, keep_dims=True) / self._dim, self._alpha))

class ReactionDiffusion(Equation):
    def __init__(self, dim, total_time, num_time_interval):
        super(ReactionDiffusion, self).__init__(dim, total_time, num_time_interval)
        self._kappa = 0.6
        self._lambda = 1 / np.sqrt(self._dim)
        self._x_init = np.zeros(self._dim)
        self._y_init = 1 + self._kappa + np.sin(self._lambda * np.sum(self._x_init)) * np.exp(
            -self._lambda * self._lambda * self._dim * self._total_time / 2)

    def sample(self, num_sample):
        dw_sample = normal.rvs(size=[num_sample,
                                     self._dim,
                                     self._num_time_interval]) * self._sqrt_delta_t
        x_sample = np.zeros([num_sample, self._dim, self._num_time_interval + 1])
        x_sample[:, :, 0] = np.ones([num_sample, self._dim]) * self._x_init
        for i in xrange(self._num_time_interval):
            x_sample[:, :, i + 1] = x_sample[:, :, i] + dw_sample[:, :, i]
        return dw_sample, x_sample

    def f_tf(self, t, x, y, z):
        exp_term = tf.exp((self._lambda ** 2) * self._dim * (t - self._total_time) / 2)
        sin_term = tf.sin(self._lambda * tf.reduce_sum(x, 1, keep_dims=True))
        temp = y - self._kappa - 1 - sin_term * exp_term
        return tf.minimum(tf.constant(1.0, dtype=tf.float64), tf.square(temp))

    def g_tf(self, t, x):
        return 1 + self._kappa + tf.sin(self._lambda * tf.reduce_sum(x, 1, keep_dims=True))


import geopandas as gpd
import os
from shapely.ops import unary_union
import numpy as np
import rasterio
from osgeo import gdal, ogr, osr

def rasterize(input_fn, output_fn, xsize, ysize, projection, gt, attribute, nodatavalue, dtype):
    shp = ogr.Open(input_fn)
    shp_lyr = shp.GetLayer()

    output = gdal.GetDriverByName('GTiff').Create(
        output_fn,
        xsize,
        ysize,
        1,
        dtype,
        options=['COMPRESS=LZW']
    )

    if isinstance(projection, int):
        srs = osr.SpatialReference()
        srs.ImportFromEPSG(projection)
        projection = srs.ExportToWkt()

    output.SetProjection(projection)
    output.SetGeoTransform(gt)

    band = output.GetRasterBand(1)
    band.SetNoDataValue(nodatavalue)
    gdal.RasterizeLayer(output, [1], shp_lyr, options=[f"ATTRIBUTE={attribute}"])

    band = None
    output = None
    shp = None

def create_coastal_segments(iso3 = None):
    '''This function selects from OSM coastline and stores them in folders based the individual floodplains.'''

    # Load shapefile of coastline
    path = os.path.join('DataDrive', 'AMENITIES','coastlines-split-4326', 'lines.shp')
    # path = os.path.join('DataDrive', 'AMENITIES','coastlines-test', 'lines.shp')

    coastline = gpd.read_file(path)

    # Load GADM files
    fp =  os.path.join('DataDrive', 'SLR','admin', 'can_flood_gadm_2_merged.shp')
    gadm_merged = gpd.read_file(fp)

    assert coastline.crs == gadm_merged.crs

    if iso3 != None:
        gadm_merged['iso3'] = [keys[:3] for keys in gadm_merged['keys']]
        gadm_filt = gadm_merged[gadm_merged['iso3'] == iso3]

    # Extract floodplains
    floodplains = [key for key in gadm_filt['keys'] if key.endswith('flood_plain')]
    gadm_filt = gadm_filt[gadm_filt['keys'].isin(floodplains)]

    # Small buffer to capture coastline
    gadm_filt = gadm_filt.buffer(0.2)

    # Iterate over floodplains and select coastal segments within the floodplain
    for i, admin_shape in enumerate(gadm_filt):
        coastline_admin = coastline[coastline.within(admin_shape)]
        assert len(coastline_admin) > 0
        # export to shape
        folder = os.path.join('DataDrive', 'AMENITIES', 'PROCESSED', floodplains[i])
        if not os.path.exists(folder):
            os.mkdir(folder)
        coastline_admin.to_file(os.path.join(folder, 'coastline.shp'))

def create_cells_adjecent_to_coast():
    '''This function rasterizes the masked shoreline projections data. It then creates a raster in which 1km cells that are adjecent to the coast have value 1, other cells have value 0.
    '''
    # Load population raster for extent, crs and cellsize
    population_fn =  os.path.join('DataDrive', 'POPULATION', 'GHS_POP_2015.tif')

    # Load masked shoreline projections
    path = os.path.join('DataDrive', 'EROSION', 'PROCESSED', 'shoreline_projections_2050_RCP45.npy')

    if not os.path.exists(path):
        raise ValueError('Clipped shoreline projections not found. First run prepare_input_data.py')

    shoreline_projections = np.load(path)

    # transform to points
    location_gpd = gpd.GeoDataFrame(np.full(shoreline_projections.shape[0], 1, dtype=np.int16), geometry=gpd.points_from_xy(shoreline_projections[:, 1], shoreline_projections[:, 0])).set_crs(epsg = 4326)
    location_gpd.columns = [str(column) for column in location_gpd.columns]
    input_fn = os.path.join('DataDrive', 'EROSION', 'PROCESSED', 'sandy_beaches.shp')
    location_gpd.to_file(input_fn)
    output_fn = os.path.join('DataDrive', 'EROSION', 'PROCESSED', 'sandy_beaches.tif')

    with rasterio.open(population_fn) as src:
                rasterize(
                    input_fn,
                    output_fn,
                    src.width,
                    src.height,
                    4326,
                    src.transform.to_gdal(),
                    '0',
                    -1,
                    gdal.GDT_Int32
                )

if __name__ == '__main__':
    # create_coastal_segments(iso3='FRA')
    create_cells_adjecent_to_coast()


import argparse
import numpy as np
import os
import os.path as osp

from dataloaders import make_data_loader
from dataloaders.utils import decode_segmap
from dataloaders.utils import decode_seg_map_sequence
from modeling.deeplab import *
from modeling.sync_batchnorm.replicate import patch_replication_callback
from utils.loss import SegmentationLosses
from utils.metrics import Evaluator
from mask_handler import get_rail_from_mask
from torchvision.utils import save_image
from tqdm import tqdm

class Tester(object):
    def __init__(self, args):
        self.args = args

        # Define DataLoader
        kwargs = {'num_workers': args.workers, 'pin_memory': True}
        self.train_loader, self.val_loader, self.test_loader, self.nclass = make_data_loader(args, **kwargs)

        # Define network
        self.model = DeepLab(num_classes=self.nclass,
                             backbone=args.backbone,
                             output_stride=args.out_stride,
                             sync_bn=args.sync_bn,
                             freeze_bn=args.freeze_bn)

        # Define Criterion
        weight = None
        self.criterion = SegmentationLosses(weight=weight, cuda=args.cuda).build_loss(mode=args.loss_type)

        # Define Evaluator
        self.evaluator = Evaluator(self.nclass)

        # Using cuda
        if args.cuda:
            self.model = torch.nn.DataParallel(self.model, device_ids=self.args.gpu_ids)
            patch_replication_callback(self.model)
            self.model = self.model.cuda()

        # Resuming checkpoint
        self.best_pred = 0.0
        if args.resume is not None:
            if not os.path.isfile(args.resume):
                raise RuntimeError("=> no checkpoint found at '{}'".format(args.resume))
            checkpoint = torch.load(args.resume)
            args.start_epoch = checkpoint['epoch']
            if args.cuda:
                self.model.module.load_state_dict(checkpoint['state_dict'])
            else:
                self.model.load_state_dict(checkpoint['state_dict'])
            self.best_pred = checkpoint['best_pred']
            print("=> loaded checkpoint '{}' (epoch {})"
                  .format(args.resume, checkpoint['epoch']))

    def visualize_image_by_pred(self, pred):
        rail, rail_type = get_rail_from_mask(torch.max(pred[:4], 1)[1].detach().cpu().numpy()[0].view())
        return rail_type

    # def visualize_image(self, dataset, i, image, target, pred):
    def visualize_image(self, dataset, target):
        '''
        save_image(image[:4].clone().cpu().data, osp.join(self.args.output_dir, '%d_image.jpg' % i), 2, normalize=True)
        save_image(decode_seg_map_sequence(torch.max(pred[:4], 1)[1].detach().cpu().numpy(),
                                           dataset=dataset), osp.join(self.args.output_dir, '%d_pred.jpg' % i), 2,
                   normalize=False, range=(0, 255))
                '''

        # rail, rail_type = get_rail_from_mask(torch.max(pred[:4], 1)[1].detach().cpu().numpy()[0].view())

        rail, rail_type = get_rail_from_mask(torch.squeeze(target[:4], 1).detach().cpu().numpy()[0].view())

        '''
        rail = decode_segmap(rail, self.args.dataset)
        import matplotlib.pyplot as plt
        plt.figure()
        plt.title('display')
        plt.subplot(211)
        plt.imshow(rail)
        target = decode_segmap(torch.squeeze(target[:4], 1).detach().cpu().numpy()[0].view(), self.args.dataset)
        plt.subplot(212)
        plt.imshow(target)
        plt.show()
        '''

        '''
        rail = torch.from_numpy(np.array(decode_seg_map_sequence([rail], dataset=dataset)))

        save_image(rail,
                   osp.join(self.args.output_dir, '%d_type_is_%s.jpg' % (i, rail_type)), 2, normalize=False,
                   range=(0, 255))

        save_image(decode_seg_map_sequence(torch.squeeze(target[:4], 1).detach().cpu().numpy(),
                                           dataset=dataset), osp.join(self.args.output_dir, '%d_truth.jpg' % i), 2,
                   normalize=False, range=(0, 255))
        '''
        return rail_type

    def test_tensor(self, tensor):
        print(tensor.shape)

    def validation(self):
        if not osp.exists(self.args.output_dir) or not osp.isdir(self.args.output_dir):
            print('%s does not exist or is not a dir' % self.args.output_dir)
            exit(1)
        self.model.eval()
        self.evaluator.reset()
        tbar = tqdm(self.val_loader, desc='\r')
        test_loss = 0.0
        for i, sample in enumerate(tbar):
            image, target = sample['image'], sample['label']
            if self.args.cuda:
                image, target = image.cuda(), target.cuda()
            with torch.no_grad():
                output = self.model(image)
            loss = self.criterion(output, target)
            test_loss += loss.item()
            tbar.set_description('Test loss: %.3f' % (test_loss / (i + 1)))
            self.visualize_image(self.args.dataset, i, image, target, output)
            pred = output.data.cpu().numpy()
            target = target.cpu().numpy()
            pred = np.argmax(pred, axis=1)
            # Add batch sample into evaluator
            self.evaluator.add_batch(target, pred)
            if i == 19:
                break

def trans(rail_type):
    assert rail_type != 'unknown'
    map = {'straight': 0,
           'left': 1,
           'right': 2}
    return map[rail_type]

def run_model(tester):
    for i, sample in enumerate(tester.val_loader):
        image = sample['image'].cuda()
        with torch.no_grad():
            output = tester.model(image)

def run_func(tester):
    truth = []
    with open('/home/shinriitin/GraduationProject/test.ans', 'r') as f:
        lines = f.readlines()
        for line in lines:
            if line == '\n':
                break
            truth.append(trans(line.split('\n')[0]))
    # print(truth)
    pred = []
    for i, sample in enumerate(tester.val_loader):
        image = sample['image'].cuda()
        with torch.no_grad():
            output = tester.model(image)
        # target = sample['label'].cuda()
        pred.append(trans(tester.visualize_image_by_pred(output)))
    # print(pred)
    for i in range(len(truth)):
        if truth[i] != pred[i]:
            print(i, truth[i], pred[i])

    from sklearn.metrics import accuracy_score
    from sklearn.metrics import precision_score
    from sklearn.metrics import recall_score
    from sklearn.metrics import f1_score
    print('acc:', accuracy_score(truth, pred))
    print('macro precision:', precision_score(truth, pred, average="macro"))
    print('micro precision:', precision_score(truth, pred, average="micro"))
    print('weighted precision:', precision_score(truth, pred, average="weighted"))
    print('macro recall:', recall_score(truth, pred, average="macro"))
    print('micro recall:', recall_score(truth, pred, average="micro"))
    print('weighted recall:', recall_score(truth, pred, average="weighted"))
    print('macro f1:', f1_score(truth, pred, average="macro"))
    print('micro f1:', f1_score(truth, pred, average="micro"))
    print('weighted f1:', f1_score(truth, pred, average="weighted"))

def test_func(args):
    tester = Tester(args)
    tester.model.eval()
    run_func(tester)

def run_func_only(tester):
    for i, sample in enumerate(tester.val_loader):
        target = sample['label'].cuda()
        tester.visualize_image(tester.args.dataset, target)

def test_fps(args):
    import time
    tester = Tester(args)
    tester.model.eval()
    times = 10
    start = time.time()
    for i in range(times):
        run_func_only(tester)
    t2 = time.time() - start
    print('User Time = %.5f s, FPS = %.5f' % (t2, 500 * times / t2))

def main():
    parser = argparse.ArgumentParser(description="PyTorch DeepLabV3Plus Testing")
    parser.add_argument('--backbone', type=str, default='resnet',
                        choices=['resnet', 'xception', 'drn', 'mobilenet'],
                        help='backbone name (default: resnet)')
    parser.add_argument('--out-stride', type=int, default=16,
                        help='network output stride (default: 16)')
    parser.add_argument('--sync-bn', type=bool, default=None,
                        help='whether to use sync bn (default: auto)')
    parser.add_argument('--freeze-bn', type=bool, default=False,
                        help='whether to freeze bn parameters (default: False)')
    parser.add_argument('--dataset', type=str, default='coco',
                        choices=['coco'],
                        help='dataset name (default: coco)')
    parser.add_argument('--workers', type=int, default=4,
                        metavar='N', help='DataLoader threads')
    parser.add_argument('--base-size', type=int, default=513,
                        help='base image size')
    parser.add_argument('--crop-size', type=int, default=513,
                        help='crop image size')
    parser.add_argument('--loss-type', type=str, default='ce',
                        choices=['ce', 'focal'],
                        help='loss func type (default: ce)')
    # training hyper params
    parser.add_argument('--batch-size', type=int, default=None,
                        metavar='N', help='input batch size for \
                                    training (default: auto)')
    parser.add_argument('--test-batch-size', type=int, default=None,
                        metavar='N', help='input batch size for \
                                    testing (default: auto)')
    # cuda, seed and logging
    parser.add_argument('--no-cuda', action='store_true', default=
    False, help='disables CUDA training')
    parser.add_argument('--gpu-ids', type=str, default='0',
                        help='use which gpu to train, must be a \
                            comma-separated list of integers only (default=0)')
    parser.add_argument('--seed', type=int, default=1, metavar='S',
                        help='random seed (default: 1)')
    # checking point
    parser.add_argument('--resume', type=str, default=None,
                        help='put the path to resuming file if needed')
    parser.add_argument('--checkname', type=str, default=None,
                        help='set the checkpoint name')
    # output dir
    parser.add_argument('--output_dir', type=str, default='test_result',
                        help='test result output directory')

    args = parser.parse_args()
    args.cuda = not args.no_cuda and torch.cuda.is_available()
    if args.cuda:
        try:
            args.gpu_ids = [int(s) for s in args.gpu_ids.split(',')]
        except ValueError:
            raise ValueError('Argument --gpu_ids must be a comma-separated list of integers only')

    if args.sync_bn is None:
        if args.cuda and len(args.gpu_ids) > 1:
            args.sync_bn = True
        else:
            args.sync_bn = False

    test_func(args)

if __name__ == '__main__':
    main()


import urllib2
import re
import copy
from BeautifulSoup import BeautifulSoup

class DataFetcher:
    def __init__(self, rss_url):
        self._markup_massage = copy.copy(BeautifulSoup.MARKUP_MASSAGE)
        self._markup_massage += [(re.compile('''), lambda m: "'")]
        self._rss_url = rss_url

    @staticmethod
    def remove_html_tags(html):
        return "".join(re.split("<[^>]*>", html))

    def get_rss_content(self):
        response = urllib2.urlopen(self._rss_url)
        html = response.read()
        return html

    def get_article_titles(self):
        html = self.get_rss_content()
        soup = BeautifulSoup(html,
                             convertEntities=BeautifulSoup.HTML_ENTITIES,
                             markupMassage=self._markup_massage)

        article_titles = []
        for tag in soup.findAll('title'):
            title = self.remove_html_tags(tag.prettify())
            article_titles.append(title.strip())

        return article_titles

if __name__ == "__main__":

    rss_links = [
        "http://rss.nytimes.com/services/xml/rss/nyt/MiddleEast.xml",
    ]
    article_titles = []
    for rss in rss_links:
        df = DataFetcher(rss)
        article_titles += df.get_article_titles()

    f = open("data", 'a')
    for title in article_titles:
        inp = raw_input("Type read(r), yes(y) or no(n): " + title + "\n")
        if not inp:
            continue
        if inp[0] == "r":
            inp = "read"
        elif inp[0] == "y":
            inp = "yes"
        else:
            inp = "no"

        f.write(title + "\n")
        f.write(inp + "\n")

    f.close()


"""ProjectsApp URL Configuration

Created by Harris Christiansen on 10/02/16.
"""
from django.conf.urls import url

from . import views

urlpatterns = [
    url(r'^project/all$', views.getProjects, name='Projects'),
    url(r'^project/form$', views.getProjectForm, name='ProjectsForm'),
    url(r'^project$', views.getProject, name='Project'),
    url(r'^project/join$', views.joinProject, name='JoinProject'),
    url(r'^project/unjoin$', views.unjoinProject, name='UnjoinProject'),
    url(r'^project/edit$', views.editProject, name='EditProject'),
    url(r'^bookmarks$', views.viewBookmarks, name='Bookmarks')
]


import pygame
import block_library
import random

class Good_Block(block_library.Block):

    def update(self):
        x = 1
        self.rect.x += x

        if self.rect.x > 720:
            self.reset_pos()
    def reset_pos(self):
        if self.rect.x > 720:
            x = random.randrange(-200, -20)
            y = random.randrange(0, 401)
            self.rect.x = x
            self.rect.y = y


import time
from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager

driver = webdriver.Chrome(ChromeDriverManager().install())
driver.get(url='http://127.0.0.1:8000/admin')

def element_is_clickable():
    driver.find_element_by_xpath('//*[@id="id_username"]').send_keys("razor")
    driver.find_element_by_xpath('//*[@id="id_password"]').send_keys("asg123")
    time.sleep(2)
    driver.find_element_by_xpath('//*[@id="login-form"]/div[3]/input').click()
    time.sleep(2)
    driver.find_element_by_xpath('//*[@id="user-tools"]/a[1]').click()
    driver.find_element_by_xpath('/html/body/nav/div/a/span').click()

#element_is_clickable()

def respons_check(w,file):
    height = 768
    driver.set_window_size(w,height)
    driver.save_screenshot(file)

respons_check(900,"test900.png")
respons_check(1200,"test1200.png")
respons_check(1800,"test1800.png")
respons_check(600,"test600.png")


#!/usr/bin/env python3
import time
from bs4 import element
from selenium import webdriver
from selenium.webdriver.support.select import Select
import re

class UItest():
    def __init__(self):
        self.option = webdriver.ChromeOptions()
        self.option.add_experimental_option('detach', True)

    def openPage(self):
        self.driver = webdriver.Chrome('E:/all py/chrome-selenium-python-master/chromedriver.exe', options=self.option)
        self.driver.get('https://www.way2automation.com/angularjs-protractor/webtables/')

        self.driver.find_element_by_xpath('/html/body/table/tbody/tr[3]/td[11]/button').click()
        time.sleep(2)
        self.driver.find_element_by_xpath('/html/body/div[3]/div[3]/button[2]').click()
        time.sleep(0.5)

        if 'Novak' in self.driver.page_source:
            print('Tagert not deleted')
        else:
            print('Tagert deleted')

if __name__ == "__main__":
    UItest().openPage()


from functools import partial

from devices.traffic import TrafficLight

from utils import soundplayer
from conf import raw_conf, o_conf
from utils.logger import logger
from utils.getclass import get_class
from utils.message import exception_summary
from utils.schedulesetter import ScheduleSetter
import sys

states = raw_conf()['states']

class Indicator:
    """
    an indicator associates builds/status of services with indicators (traffic lights, sounds etc)
    an indicator might for example:

      - read the API functional tests Bamboo build, play a sound if they have
        failing test(s), and display the build status on a traffic light
      - make a sound and write to a log if a service is down

    each indicator has a 'state'-- that encompasses Bamboo responsiveness, test failures,
    presence of configured traffic lights, internal exceptions
    states are divided into ERROR, WARNING and NONE that are configurable, and currently
    affect the logging level
    """

    def __init__(self, indicator_name, settings):
        self.indicator_name = indicator_name
        self.state = None
        self.settings = settings
        self.unhandled_exception_raised_previously = False
        self.listeners = []
        self.setup_listeners()
        self.setup_devices()

    def setup_listeners(self):
        for listener_config in self.settings['listeners']:
            for listener_type, settings in listener_config.items():
                listener = self.schedule_listener(listener_type, settings, self.find_schedule(settings))
                self.listeners.append(listener)

    def schedule_listener(self, name, settings, schedule_location):
        try:
            class_name = 'listeners.{0}.{1}'.format(name, name.capitalize())
            listener = get_class(class_name)(self.indicator_name, name, settings)
            wrapped_listener = partial(self.run_wrapper, listener)
            ScheduleSetter(job=wrapped_listener, schedule_settings=schedule_location)
            return listener
        except NameError as e:
            message = "{indicator}: implementation for listener type '{listener}' " \
                      "is not available or incomplete\n" \
                      "Exception: {exception}\n"
            message = message.format(indicator=self.indicator_name, listener=name, exception=e)
            logger.error(message)

    def find_schedule(self, listener_settings):

        if listener_settings.get('schedule'):
            return listener_settings
        elif self.settings.get('schedule'):
            return self.settings
        else:
            return o_conf()

    def run_wrapper(self, listener):
        logger.info("Running indicator '{indicator}', {clazz} listener '{listener}' ...".format(indicator=listener.indicator_name,
            listener=listener.name, clazz=listener.listener_class))
        try:
            listener.poll()
        except Exception as e:
            logger.error('{indicator}: {listener}: unhandled exception as follows:\n{exception}'
                .format(indicator=listener.indicator_name, listener=listener.name, exception=exception_summary()))

    def setup_devices(self):
        settings = self.settings.get('trafficlight')
        if settings:
            self.trafficlight = TrafficLight(self.indicator_name, settings)
        else:
            self.trafficlight = None

    def run(self):
        state = self.get_state()
        self.show_change(state)
        if self.trafficlight:
            self.trafficlight.set_lights(state)
        self.state = state

        # except Exception as e:
        #     logger.error('Signal {signal}: Unhandled internal exception. '
        #                  'Could be configuration problem or bug.\n{exception}'
        #                  .format(signal=self.indicator_name, exception=e.args))
        #     self.internal_exception(e)
            # NB traffic light update not shown until unhandled exception clear for one complete pass
            # logger.error('Waiting {0} secs\n'.format(configuration['errorheartbeat_secs']))
            # sleep(configuration['errorheartbeat_secs'])

    def get_state(self):
        # for listener in self.listeners:
            # logger.info("{indicator}: polling '{name}' tests"
            #             .format(indicator=self.indicator_name, name=listener.name))
            # logger.info("  {0}:{1}:{2}".format(listener.tests_ok(), listener.comms_ok(), listener.has_changed()))
        comms_failures = any(not listener.comms_ok() for listener in self.listeners)
        test_failures = any(not listener.tests_ok() for listener in self.listeners)
        state = 1 if test_failures else 0
        state += 2 if comms_failures else 0
        return o_conf().states[state]

    def signal_unhandled_exception(self, e):
        logger.error("Indicator {indicator}: "
                     "internal exception occurred:\nException as follows:\n{exception}"
                     .format(indicator=self.indicator_name, exception=exception_summary()))
        if not self.unhandled_exception_raised_previously:
            self.show_change('internalexception')
        self.unhandled_exception_raised_previously = True

    def show_change(self, state):
        severities = o_conf().severities
        errors = severities['errors']
        is_error = state in errors
        warnings = severities['warnings']
        is_warning = state in warnings
        change_to_from_error = is_error != (self.state in errors)
        change_to_from_warning = is_warning != (self.state in warnings)
        change_of_error_level = change_to_from_error or change_to_from_warning
        self.show_by_traffic_light(state)
        self.show_by_sound(change_of_error_level, is_error, is_warning)
        self.show_by_logging(change_to_from_error, change_to_from_warning, state)

    def show_by_traffic_light(self, state):
        state_changed = state != self.state
        if self.trafficlight:
            if state_changed:
                self.trafficlight.state_change()
            else:
                self.trafficlight.blink()
            self.trafficlight.set_lights(state)

    def show_by_sound(self, change_of_error_level, is_error, is_warning):
        if change_of_error_level:
            sound = self.settings['sounds']
            if is_error or is_warning:
                wav = sound['failures']
            else:
                wav = sound['greenbuild']
            soundplayer.playwav(wav)

    def show_by_logging(self, change_to_from_error, change_to_from_warning, state):
        if change_to_from_error:
            level = 'ERROR'
        elif change_to_from_warning:
            level = 'WARNING'
        else:
            level = 'NONE'
        if not self.state and state != self.state:
            message = "State changing from '{previous}' to '{current}'" \
                .format(previous=self.state, current=state)
            logger_method = {'ERROR': logger.error,
                             'WARNING': logger.warn,
                             'NONE': logger.info}
            logger_method[level](message)


# Google Kick Start : Problem 2
# L Shape Plot

import numpy as np
def count(x, y):
  return max((min(x//2, y)+min(x,y//2)-2),0)

def solve():
  R, C = map(int, input().split())
  G = []
  for _ in range(R):
    G.append(list(map(int, input().split())))
  G = np.array(G)
  dim = G.shape
  l = np.zeros(dim)
  t = np.zeros(dim)
  r = np.zeros(dim)
  b = np.zeros(dim)

  for i in range(dim[0]):
    lt = 0
    rt = 0
    for j in range(dim[1]):
      if G[i][j] == 0:
        l[i][j] = 0
        lt = 0
      else:
        lt += 1
        l[i][j] = lt

      if G[i][C-1-j] == 0:
        r[i][C-1-j] = 0
        rt = 0
      else:
        rt += 1
        r[i][C-1-j] = rt

  for j in range(dim[1]):
    tt = 0
    bt = 0
    for i in range(dim[0]):
      if G[i][j] == 0:
        t[i][j] = 0
        tt = 0
      else:
        tt += 1
        t[i][j] = tt

      if G[R-1-i][j] == 0:
        b[R-1-i][j] = 0
        bt = 0
      else:
        bt += 1
        b[R-1-i][j] = bt
  tl = 0
  tr = 0
  bl = 0
  br = 0
  for i in range(dim[0]):
    for j in range(dim[1]):
      tl += count(l[i][j],t[i][j])
      tr += count(r[i][j],t[i][j])
      bl += count(l[i][j],b[i][j])
      br += count(r[i][j],b[i][j])

  return tl+tr+bl+br

for i in range(int(input())):
  print("Case #{}: {}".format(i+1, solve()))


from urllib.request import urlopen

from girder import logger
from girder.api.rest import RestException
from girder.constants import AccessType
from girder.models.folder import Folder

from . import DataONELocations, DataONENotATaleError
from ..import_providers import ImportProvider
from ..data_map import DataMap
from ..file_map import FileMap
from ..import_item import ImportItem
from ..entity import Entity
from ...models.tale import Tale
from .register import \
    D1_lookup, \
    extract_metadata_docs, \
    get_documents, \
    get_package_pid, \
    get_package_list, \
    extract_data_docs, \
    extract_resource_docs, \
    check_multiple_metadata

ALL_LOCATIONS = [DataONELocations.prod_cn, DataONELocations.dev_mn, DataONELocations.dev_cn]

class DataOneImportProvider(ImportProvider):
    def __init__(self):
        super().__init__('DataONE')

    def matches(self, entity: Entity) -> bool:
        url = entity.getValue()
        try:
            package_pid = get_package_pid(url, entity['base_url'])
        except RestException:
            return False
        return package_pid is not None

    def lookup(self, entity: Entity) -> DataMap:
        # just wrap D1_lookup for now
        # this does not seem to properly resolve individual files. If passed something like
        # https://cn.dataone.org/cn/v2/resolve/urn:uuid:9266a118-78b3-48e3-a675-b3dfcc5d0fc4,
        # it returns the parent dataset, which, as a user, I'd be annoyed with
        dataMap = D1_lookup(entity.getValue(), entity['base_url'])
        dataMap.repository = self.name
        return dataMap

    def listFiles(self, entity: Entity) -> FileMap:
        result = get_package_list(entity.getValue(), entity['base_url'])
        return FileMap.fromDict(result)

    def getDatasetUID(self, doc: object, user: object) -> str:
        if 'folderId' in doc:
            # It's an item, grab the parent which should contain all the info
            doc = Folder().load(doc['folderId'], user=user, level=AccessType.READ)
        # obj is a folder at this point use its meta
        return doc['meta']['identifier']

    def _listRecursive(self, user, pid: str, name: str, base_url: str = DataONELocations.prod_cn,
                       progress=None):
        """Create a package description (Dict) suitable for dumping to JSON."""
        if progress:
            progress.update(increment=1, message='Processing package {}.'.format(pid))

        # query for things in the resource map. At this point, it is assumed that the pid
        # has been correctly identified by the user in the UI.

        docs = get_documents(pid, base_url)

        # Filter the Solr result by TYPE so we can construct the package
        metadata = extract_metadata_docs(docs)
        data = extract_data_docs(docs)
        children = extract_resource_docs(docs)

        # Add in URLs to resolve each metadata/data object by
        self._addResolutionUrls(metadata, base_url)
        self._addResolutionUrls(data, base_url)

        # Determine the folder name. This is usually the title of the metadata file
        # in the package but when there are multiple metadata files in the package,
        # we need to figure out which one is the 'main' or 'documenting' one.
        primary_metadata = [doc for doc in metadata if 'documents' in doc]

        check_multiple_metadata(primary_metadata)

        # Create a Dict to store folders' information
        # the data key is a concatenation of the data and any metadata objects
        # that aren't the main or documenting metadata
        primary_identifier = primary_metadata[0]['identifier']
        data += [doc for doc in metadata if doc['identifier'] != primary_identifier]
        if not name:
            name = primary_metadata[0]['title']

        yield ImportItem(
            ImportItem.FOLDER,
            name,
            identifier=primary_identifier,
            meta={
                "dsRelPath": "/",
            }
        )

        for fileObj in data:
            name = fileObj.get("fileName", fileObj["identifier"])
            yield ImportItem(
                ImportItem.FILE,
                name,
                identifier=primary_identifier,
                url=fileObj["url"],
                size=int(fileObj["size"]),
                mimeType=fileObj["formatId"],
                meta={
                    "checksum": {fileObj["checksumAlgorithm"].lower(): fileObj["checksum"]},
                    "dsRelPath": f"/{name}",  # D1 packages are flat...
                    "directIdentifier": fileObj["identifier"],
                },
            )

        # Recurse and add child packages if any exist
        if children is not None and len(children) > 0:
            for child in children:
                logger.debug('Registering child package, {}'.format(child['identifier']))
                yield from self._listRecursive(
                    user, child['identifier'], None, base_url=base_url, progress=progress)

        yield ImportItem(ImportItem.END_FOLDER)
        logger.debug('Finished registering dataset')

    def _addResolutionUrls(self, docs, base_url):
        """
        Combines the base coordinating node URL with the resolve endpoint and identifier
        :param docs: List of metadata/data objects
        :param base_url: The coordinating node base URL (including the version)
        :return: None
        """
        for d in docs:
            d['url'] = "{}/{}/{}".format(base_url, 'resolve', d['identifier'])

    def import_tale(self, data_map, user, force=False):
        existing_tale_id = Tale().findOne(
            query={
                "creatorId": user["_id"],
                "publishInfo.pid": {"$eq": data_map.doi},
            },
            fields={"_id"},
        )
        if existing_tale_id and not force:
            return Tale().load(existing_tale_id["_id"], user=user)

        if not data_map.tale:
            raise DataONENotATaleError(data_map)

        docs = get_documents(data_map.dataId, data_map.base_url)
        for doc in docs:
            if doc.get("formatType") == "METADATA":
                metadata = doc
            elif doc.get("formatType") == "DATA":
                zipfile = doc
        file_url = f"{data_map.base_url}/object/{zipfile['identifier']}"

        def stream_zipfile(chunk_size):
            with urlopen(file_url) as src:
                while True:
                    data = src.read(chunk_size)
                    if not data:
                        break
                    yield data

        publishInfo = [
            {
                "pid": metadata["identifier"],
                "uri": metadata["dataUrl"],  # NOTE: it's wrong for test data
                "date": metadata["dateUploaded"],  # convert to date?
                "repository_id": data_map.dataId,
                "repository": "DataONE",
            }
        ]
        relatedIdentifiers = [
            {"relation": "IsDerivedFrom", "identifier": data_map.doi}
        ]
        return Tale().createTaleFromStream(
            stream_zipfile,
            user=user,
            publishInfo=publishInfo,
            relatedIdentifiers=relatedIdentifiers,
        )


from contextlib import contextmanager

from sqlalchemy import create_engine
from sqlalchemy.exc import InvalidRequestError
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker, scoped_session

from settings import POSTGRES_URI, REDSHIFT_URI

postgres_engine = create_engine(POSTGRES_URI)
redshift_engine = create_engine(REDSHIFT_URI)

postgres_session = scoped_session(sessionmaker(postgres_engine))
postgres_base = declarative_base(postgres_engine)
postgres_base.query = postgres_session.query_property()

redshift_session = scoped_session(sessionmaker(redshift_engine))
redshift_base = declarative_base(redshift_engine)
redshift_base.query = redshift_session.query_property()

@contextmanager
def redshift_session_context():
    """A helper method for keeping the state of an connection with the database
    separate from the work being done, and ensuring that the session is always
    cleaned up after use."""

    transactional_session = redshift_session()
    try:
        yield transactional_session
        transactional_session.commit()
    except InvalidRequestError:
        pass
    except:
        transactional_session.rollback()
        raise
    finally:
        transactional_session.close()


from parse import *
from Dropset import *
from Tree import *
from Taxon import *
from Bipartition import *
import numpy as np

'''
Function calculating dropset of two bipartitions in bitvector format
'''

def get_drops(ind_bip, s_bip):
    # calculate the dropsets
    a = ind_bip & s_bip
    b = (~ind_bip) & (~s_bip)
    c = ind_bip & (~s_bip)
    d = (~ind_bip) & s_bip

    # choose the dropset representation with smaller bits set
    _ab = a.count() + b.count()
    _cd = c.count() + d.count()

    if _ab < _cd:
        drop = a | b
    else:
        drop = c | d

    # raise flag that we have to deal with a matching dropset
    if drop.count() == 0:
        indices = [-1]
    else:
        # now get the indices of bits set to 1 as a list
        indices = np.argwhere(drop == np.amax(drop)).flatten()

    return indices

'''
Load all bipartitions and calculate the dropset

Parameters:
save - bool, determines if the dropsets are saved in a file called drops.txt
start_tree - index of tree from which we should start extracting bips
end_tree - index of tree till which we extract bips (start_tree < end_tree!)
file - which file should be loaded

returns

- dictionary with all dropsets
- all trees and their bips
- taxa_list containing every taxon and where it appears
'''

def calculate_drops(save, file, limit_drops):
    # count all dropsets
    drops_count = 0
    comparisons_count = 0

    # Decides if we save dropsets into a file called drops.txt
    save_in_file = save

    # Read the bips file from RAxML (second argument: number of trees)
    [trees, mxtips, n_tree] = read_bips(file)

    # Dictionary saving all dropsets
    dropsets_dict = {}
    # Array saving all taxon
    taxa_list = [Taxon(i) for i in range(mxtips + 1)]

    # store all dropsets in a file called sets
    if save_in_file:
        f = open("drops.txt", "w")

    # Iterate through all trees
    for i in range(len(trees)):
        # for i in range(start_tree, end_tree):
        print("calculating dropset for tree", i)

        # Extract the bips
        s_bips = trees[i]['s_bips']
        ind_bips = trees[i]['ind_bips']
        s_treeList = trees[i]['local_to_global']

        # Save information into Taxon for faster retrieval
        for taxon_id in s_treeList:
            taxa_list[taxon_id].add_tree(i)

        # Dictionary to detect multiple bips
        # trees[i]['s_bips_dict'] = {}
        # s_bips_dict = trees[i]['s_bips_dict']

        # global_to_local maps global to local ids
        trees[i]['global_to_local'] = {}
        g2l_mapping = trees[i]['global_to_local']
        for local_index, global_index in enumerate(s_treeList):
            g2l_mapping[global_index] = local_index

        trees[i]["Tree"] = Tree(i, g2l_mapping, s_treeList)
        tree = trees[i]["Tree"]
        s_bips_dict = tree.get_bips()

        count = 0
        maxcount = len(s_bips) * len(ind_bips)
        comparisons_count += maxcount

        for s_id, s_bip in enumerate(s_bips):
            # sanity check that we have unambigious bitvector representation
            assert (s_bip[0] == False)

            # convert it to string
            key = s_bip.to01()
            # create the bipartition and store it into a dictionary
            s_bip_el = Bipartition((i, s_id), s_bip)
            s_bips_dict[key] = s_bip_el

            # look at all possible combinations of ind_bip and s_bip and calculate the dropset
            for ind_id, ind_bip in enumerate(ind_bips):
                # second sanity check for standard representation
                assert (ind_bip[0] == False)
                count += 1
                # print("looking at",count,"/",maxcount)
                # calculate dropsets and get the indices
                indices = get_drops(ind_bip, s_bip)
                if len(indices) > limit_drops:
                    pass
                else:
                    # raised flag, we deal with a matching bipartition
                    if indices[0] == -1:
                        s_bip_el.set_matching(True)
                    else:
                        # get the global representation of the dropset
                        drop = [s_treeList[i] for i in indices]
                        # sort the dropset for unique key
                        drop = sorted(drop)
                        key = str(drop)

                        # if dropset already exists then get it
                        if key in dropsets_dict:
                            drop_e = dropsets_dict[key]
                            drop_e.add_s_bip(s_bip_el)

                        # otherwise create a new dropset
                        else:
                            drops_count += 1  # one more unique dropset
                            drop_e = Dropset(drop, s_bip_el)
                            dropsets_dict[key] = drop_e

                        # Store the dropset into taxon for easier searching
                        for taxon_id in drop:
                            # Take care not to store duplicates
                            if drop_e not in taxa_list[taxon_id].get_dropsets():
                                taxa_list[taxon_id].add_dropset(drop_e)

                    if save_in_file:
                        # save tree index and s_bip, ind_bip index
                        f.write(str(i) + " " + str(s_id) + " " + str(ind_id) + "\n")
                        f.write(str(drop) + "\n")

    print("Ok, everything is fine")
    if save_in_file:
        f.close()
    print("Extracted", drops_count, "unique dropsets from", comparisons_count, "comparisons")

    return dropsets_dict, trees, taxa_list


'''
Make a colorbar as a separate figure.
'''
from pacfm.model import file_provider

from matplotlib import pyplot
import matplotlib as mpl

def drawColorbars(*intervals):
    #(nrows, ncols, plot_number)
    fig = pyplot.figure(figsize=(7,3), frameon=False)
    for i in range(len(intervals)):
        #ax1 = fig.add_axes([0.05, 0.80, 0.9, 0.15])
        #ax2 = fig.add_axes([0.05, 0.475, 0.9, 0.15])
        ax= fig.add_subplot(len(intervals),1,i+1)

        cmap = mpl.cm.YlOrRd
        norm = mpl.colors.Normalize(vmin=intervals[i][0], vmax=intervals[i][1])

        cb1 = mpl.colorbar.ColorbarBase(ax, cmap=cmap,
                                           norm=norm,
                                           orientation='horizontal')

        #norm = mpl.colors.Normalize(vmin=5, vmax=10)
        #cb2 = mpl.colorbar.ColorbarBase(ax2, cmap=cmap,
        #                                     norm=norm,
        #                                     spacing='proportional',
        #                                     orientation='horizontal')
        #cb2.set_label('Discrete intervals, some other units')

    cb1.set_label('Some Units')
    #pyplot.show()
    pyplot.savefig(file_provider["output"]["colorbar_svg"])

intervals= [(1,10),(5,20)]
drawColorbars(*intervals)


import re

def get_positions(positions):
    """Get integer (or list of integer) representation of user-supplied 'positions'.
    Args:
        -- positions: user-supplied string, which is an integer (or a list of integers).
    Returns:
        -- int_positions: an integer (or a list of integers)."""
    positions = positions.strip()
    list_positions = re.split(r'[\[\(,\)\]]',positions)
    int_positions = [int(tmp) for tmp in list_positions[1:-1]]
    return int_positions


import scapy.all as scapy
import argparse

def get_arguments():
    parser = argparse.ArgumentParser()
    parser.add_argument('-t', '--target', dest='target',
                        help='Target IP / IP range.')

    options = parser.parse_args()
    if not options.target:
        parser.error(
            '[-] Please specify an IP / IP range, use --help for more info.')

    return options

def scan(ip):
    arp_request = scapy.ARP(pdst=ip)
    dst = 'ff:ff:ff:ff:ff:ff'
    broadcast = scapy.Ether(dst=dst)
    arp_request_broadcast = broadcast / arp_request
    answered_list = scapy.srp(
        arp_request_broadcast, timeout=1, verbose=False)[0]

    clients_list = []

    for element in answered_list:
        client_dict = {'ip': element[1].psrc, 'mac': element[1].hwsrc}
        clients_list.append(client_dict)

    return clients_list

def print_sesult(results_list):
    print('IP\t\t\tMAC Address')
    print('-------------------------------------')

    for client in results_list:
        print(f'{client["ip"]}\t\t{client["mac"]}')

options = get_arguments()

ip = options.target
# ip = "10.0.2.1/24"
# ip = "192.168.10.1/24"

scan_result = scan(ip)
print_sesult(scan_result)


import torch.nn as nn
import torch.nn.functional as F
import math
import torch
from torch.nn.parameter import Parameter
from torch.nn.modules.module import Module
from DeepRobust.deeprobust.graph import utils
from DeepRobust.deeprobust.graph.defense import GCN
from tqdm import tqdm
import scipy.sparse as sp
import numpy as np

class GCNSVD(GCN):

    def __init__(self, nfeat, nhid, nclass, dropout=0.5, lr=0.01, weight_decay=5e-4, with_relu=True, with_bias=True, device='cpu'):

        super(GCNSVD, self).__init__(nfeat, nhid, nclass, dropout, lr, weight_decay, with_relu, with_bias, device=device)
        self.device = device

    def fit(self, features, adj, labels, idx_train, idx_val=None, k=50, train_iters=200, initialize=True, verbose=True):

        modified_adj = self.truncatedSVD(adj, k=k)
        # modified_adj_tensor = utils.sparse_mx_to_torch_sparse_tensor(self.modified_adj)
        features, modified_adj, labels = utils.to_tensor(features, modified_adj, labels, device=self.device)

        self.modified_adj = modified_adj
        self.features = features
        self.labels = labels
        super().fit(features, modified_adj, labels, idx_train, idx_val, train_iters=train_iters, initialize=initialize, verbose=verbose)

    def truncatedSVD(self, data, k=50):
        print('=== GCN-SVD: rank={} ==='.format(k))
        if sp.issparse(data):
            data = data.asfptype()
            U, S, V = sp.linalg.svds(data, k=k)
            print("rank_after = {}".format(len(S.nonzero()[0])))
            diag_S = np.diag(S)
        else:
            U, S, V = np.linalg.svd(data)
            U = U[:, :k]
            S = S[:k]
            V = V[:k, :]
            print("rank_before = {}".format(len(S.nonzero()[0])))
            diag_S = np.diag(S)
            print("rank_after = {}".format(len(diag_S.nonzero()[0])))

        return U @ diag_S @ V

class GCNJaccard(GCN):

    def __init__(self, nfeat, nhid, nclass, binary_feature=True, dropout=0.5, lr=0.01, weight_decay=5e-4, with_relu=True, with_bias=True, device='cpu'):

        super(GCNJaccard, self).__init__(nfeat, nhid, nclass, dropout, lr, weight_decay, with_relu, with_bias, device=device)
        self.device = device
        self.binary_feature = binary_feature

    def fit(self, features, adj, labels, idx_train, idx_val=None, threshold=0.01, train_iters=200, initialize=True, verbose=True):
        self.threshold = threshold
        modified_adj = self.drop_dissimilar_edges(features, adj)
        # modified_adj_tensor = utils.sparse_mx_to_torch_sparse_tensor(self.modified_adj)
        features, modified_adj, labels = utils.to_tensor(features, modified_adj, labels, device=self.device)
        self.modified_adj = modified_adj
        self.features = features
        self.labels = labels
        super().fit(features, modified_adj, labels, idx_train, idx_val, train_iters=train_iters, initialize=initialize, verbose=verbose)

    def drop_dissimilar_edges(self, features, adj):
        if not sp.issparse(adj):
            adj = sp.csr_matrix(adj)
        modified_adj = adj.copy().tolil()
        # preprocessing based on features

        print('=== GCN-Jaccrad ===')
        # isSparse = sp.issparse(features)
        edges = np.array(modified_adj.nonzero()).T
        removed_cnt = 0
        for edge in tqdm(edges):
            n1 = edge[0]
            n2 = edge[1]
            if n1 > n2:
                continue

            # if isSparse:
            if self.binary_feature:
                J = self._jaccard_similarity(features[n1], features[n2])

                if J < self.threshold:
                    modified_adj[n1, n2] = 0
                    modified_adj[n2, n1] = 0
                    removed_cnt += 1
            else:
                # For not binary feature, use cosine similarity
                C = self._cosine_similarity(features[n1], features[n2])
                if C < self.threshold:
                    modified_adj[n1, n2] = 0
                    modified_adj[n2, n1] = 0
                    removed_cnt += 1
        print('removed %s edges in the original graph' % removed_cnt)
        return modified_adj

    def _jaccard_similarity(self, a, b):
        intersection = a.multiply(b).count_nonzero()
        J = intersection * 1.0 / (a.count_nonzero() + b.count_nonzero() - intersection)
        return J

    def _cosine_similarity(self, a, b):
        inner_product = (features[n1] * features[n2]).sum()
        C = inner_product / np.sqrt(np.square(a).sum() + np.square(b).sum())
        return C


# Haoxuan Li
# Student ID: 10434197
from .Student import Student
from .instructor import Instructor
from .grade import Grade
import os
from haoxuanli_810_09.Utils import Utils
from prettytable import PrettyTable
from operator import itemgetter

class Repository:
    def __init__(self, init_dir: str):
        self.students = dict()  # key: student_CWID  value: student(OBJ)
        self.instructors = dict()  # key: instructor_CWID  value: instructor(OBJ)
        self.grades = dict()  # key: (student_CWID, course_name)  value: grade(OBJ)
        self.students_path = os.path.join(init_dir, "students.txt")
        self.instructors_path = os.path.join(init_dir, "instructors.txt")
        self.grades_path = os.path.join(init_dir, "grades.txt")
        self.read_student(self.students_path)
        self.read_instructors(self.instructors_path)
        self.read_grades(self.grades_path)

    def read_student(self, path: str):
        for cwid, name, major in Utils.read_lines(path):
            student = Student(cwid, name, major)
            self.students[student.cwid] = student
        return self.students

    def read_instructors(self, path: str):
        for cwid, name, dept in Utils.read_lines(path):
            instructor = Instructor(cwid, name, dept)
            self.instructors[instructor.cwid] = instructor
        return self.instructors

    def read_grades(self, path: str):
        for s_cwid, course_name, score, ins_id in Utils.read_lines(path):
            grade = Grade(s_cwid, course_name, score, ins_id)
            self.grades[grade.stu_id, grade.course_name] = grade
        return self.grades

    def show_students(self):
        student_summary_table = PrettyTable()
        student_summary_table.field_names = Student.get_fields()
        sorted_students = sorted(self.students.items(), key=itemgetter(0))  # <-------------- here!
        for item in sorted_students:
            student_summary_table.add_row(item[1].pt_show())  # <-------------- here!
        print(student_summary_table.get_string())

    def show_instructors(self):
        instructor_summary_table = PrettyTable()
        instructor_summary_table.field_names = Instructor.get_fields()
        sorted_instructors = sorted(self.instructors.items(), key=itemgetter(0))  # <-------------- here!
        for item in sorted_instructors:
            for data in item[1].pt_show():  # <-------------- here!
                instructor_summary_table.add_row(data)
        print(instructor_summary_table.get_string())

    def analysis_grades(self):
        for grade in self.grades.values():
            self.students.get(grade.stu_id).add_course(grade.course_name, grade.score)
            self.instructors.get(grade.ins_id).add_course(grade.course_name)


def container_run(platform: str, nvidia_runtime: bool, docker_registry: str, shared_memory_size: str, local_ccache_dir: str, command: List[str], cleanup: Cleanup, dry_run: bool=False) -> int:
    'Run command in a container'
    container_wait_s = 600
    environment = {
        'CCACHE_MAXSIZE': '500G',
        'CCACHE_TEMPDIR': '/tmp/ccache',
        'CCACHE_DIR': '/work/ccache',
        'CCACHE_LOGFILE': '/tmp/ccache.log',
    }
    jenkins_env_vars = ['BUILD_NUMBER', 'BUILD_ID', 'BUILD_TAG']
    environment.update({k: os.environ[k] for k in jenkins_env_vars if (k in os.environ)})
    environment.update({k: os.environ[k] for k in ['CCACHE_MAXSIZE'] if (k in os.environ)})
    tag = get_docker_tag(platform=platform, registry=docker_registry)
    mx_root = get_mxnet_root()
    local_build_folder = buildir()
    os.makedirs(local_build_folder, exist_ok=True)
    os.makedirs(local_ccache_dir, exist_ok=True)
    logging.info('Using ccache directory: %s', local_ccache_dir)
    docker_client = docker.from_env()
    docker_cmd_list = [get_docker_binary(nvidia_runtime), 'run', '--cap-add', 'SYS_PTRACE', '--rm', '--shm-size={}'.format(shared_memory_size), '-v', '{}:/work/mxnet'.format(mx_root), '-v', '{}:/work/build'.format(local_build_folder), '-v', '{}:/work/ccache'.format(local_ccache_dir), '-u', '{}:{}'.format(os.getuid(), os.getgid()), '-e', 'CCACHE_MAXSIZE={}'.format(environment['CCACHE_MAXSIZE']), '-e', 'CCACHE_TEMPDIR={}'.format(environment['CCACHE_TEMPDIR']), '-e', 'CCACHE_DIR={}'.format(environment['CCACHE_DIR']), '-e', 'CCACHE_LOGFILE={}'.format(environment['CCACHE_LOGFILE']), '-ti', tag]
    docker_cmd_list.extend(command)
    docker_cmd = ' \\\n\t'.join(docker_cmd_list)
    logging.info('Running %s in container %s', command, tag)
    logging.info('Executing the equivalent of:\n%s\n', docker_cmd)
    ret = 0
    if (not dry_run):
        signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGINT, signal.SIGTERM})
        runtime = None
        if nvidia_runtime:
            runtime = 'nvidia'
        container = docker_client.containers.run(tag, runtime=runtime, detach=True, command=command, shm_size=shared_memory_size, user='{}:{}'.format(os.getuid(), os.getgid()), cap_add='SYS_PTRACE', volumes={
            mx_root: {
                'bind': '/work/mxnet',
                'mode': 'rw',
            },
            local_build_folder: {
                'bind': '/work/build',
                'mode': 'rw',
            },
            local_ccache_dir: {
                'bind': '/work/ccache',
                'mode': 'rw',
            },
        }, environment=environment)
        logging.info('Started container: %s', trim_container_id(container.id))
        cleanup.add_container(container)
        signal.pthread_sigmask(signal.SIG_UNBLOCK, {signal.SIGINT, signal.SIGTERM})
        stream = container.logs(stream=True, stdout=True, stderr=True)
        sys.stdout.flush()
        for chunk in stream:
            sys.stdout.buffer.write(chunk)
            sys.stdout.buffer.flush()
        sys.stdout.flush()
        stream.close()
        try:
            logging.info('Waiting for status of container %s for %d s.', trim_container_id(container.id), container_wait_s)
            wait_result = container.wait(timeout=container_wait_s)
            logging.info('Container exit status: %s', wait_result)
            ret = wait_result.get('StatusCode', 200)
        except Exception as e:
            logging.exception(e)
            ret = 150
        try:
            logging.info('Stopping container: %s', trim_container_id(container.id))
            container.stop()
        except Exception as e:
            logging.exception(e)
            ret = 151
        try:
            logging.info('Removing container: %s', trim_container_id(container.id))
            container.remove()
        except Exception as e:
            logging.exception(e)
            ret = 152
        cleanup.remove_container(container)
        containers = docker_client.containers.list()
        if containers:
            logging.info('Other running containers: %s', [trim_container_id(x.id) for x in containers])
    return ret


import pygame
from pygame.sprite import Group

class Scoreboard:
    """A class to report scoring information"""

    def __init__(self, settings, screen, stats, start_screen):
        self.screen = screen
        self.screen_rect = screen.get_rect()
        self.settings = settings
        self.stats = stats
        self.start = start_screen

        # Font settings
        self.text_color = (255, 255, 255)
        self.bg_color = (100, 100, 200)
        self.font = pygame.font.SysFont(None, 48)

        # Declare attributes in init but initialize in other methods
        self.score_image = None
        self.score_rect = None
        self.value_image = None
        self.value_rect = None
        self.lives_str_image = None
        self.lives_str_rect = None
        self.lives_image = None
        self.lives_rect = None
        self.coins_image = None
        self.coins_rect = None
        self.num_coins_image = None
        self.num_coins_rect = None
        self.world_image = None
        self.world_rect = None
        self.curr_image = None
        self.curr_image_rect = None
        self.hs_image = None
        self.hs_rect = None

        # Prepare the initial score images
        self.prep_score()
        self.prep_lives()
        self.prep_coins()
        self.prep_world()

    def prep_score(self):
        """Turn score into a rendered image"""
        score = "SCORE"
        rounded_score = round(self.stats.score, -1)
        value = "{:,}".format(rounded_score)
        self.score_image = self.font.render(score, True, self.text_color, self.bg_color)
        self.value_image = self.font.render(value, True, self.text_color, self.bg_color)

        # Display score at top left
        self.score_rect = self.score_image.get_rect()
        self.score_rect.left = self.screen_rect.left + 20
        self.score_rect.top = self.screen_rect.top

        self.value_rect = self.value_image.get_rect()
        self.value_rect.left = self.screen_rect.left + 65
        self.value_rect.top = 30

    def prep_lives(self):
        lives_str = "LIVES"
        self.lives_str_image = self.font.render(lives_str, True, self.text_color, self.bg_color)
        lives = str(self.stats.lives)
        self.lives_image = self.font.render(lives, True, self.text_color, self.bg_color)

        # Display at top right
        self.lives_str_rect = self.lives_str_image.get_rect()
        self.lives_str_rect.right = self.screen_rect.right - 20
        self.lives_str_rect.top = self.screen_rect.top

        self.lives_rect = self.lives_image.get_rect()
        self.lives_rect.right = self.screen_rect.right - 65
        self.lives_rect.top = 30

    def prep_coins(self):
        coins = "COINS"
        num_coins = str(self.stats.coins)
        self.coins_image = self.font.render(coins, True, self.text_color, self.bg_color)
        self.num_coins_image = self.font.render(num_coins, True, self.text_color, self.bg_color)

        # Display coins at middle-left
        self.coins_rect = self.coins_image.get_rect()
        self.coins_rect.left = self.screen_rect.left + 350
        self.coins_rect.top = self.screen_rect.top

        self.num_coins_rect = self.value_image.get_rect()
        self.num_coins_rect.left = self.screen_rect.left + 390
        self.num_coins_rect.top = 30

    def prep_world(self):
        world = "WORLD"
        curr_world = self.stats.world
        self.world_image = self.font.render(world, True, self.text_color, self.bg_color)
        self.curr_image = self.font.render(curr_world, True, self.text_color, self.bg_color)

        # Display world at middle-right
        self.world_rect = self.world_image.get_rect()
        self.world_rect.right = self.screen_rect.right - 350
        self.world_rect.top = self.screen_rect.top

        self.curr_image_rect = self.curr_image.get_rect()
        self.curr_image_rect.right = self.screen_rect.right - 390
        self.curr_image_rect.top = 30

    def show_score(self):
        """Draw scoreboard to screen"""
        self.screen.blit(self.score_image, self.score_rect)
        self.screen.blit(self.value_image, self.value_rect)
        self.screen.blit(self.lives_str_image, self.lives_str_rect)
        self.screen.blit(self.lives_image, self.lives_rect)
        self.screen.blit(self.coins_image, self.coins_rect)
        self.screen.blit(self.num_coins_image, self.num_coins_rect)
        self.screen.blit(self.world_image, self.world_rect)
        self.screen.blit(self.curr_image, self.curr_image_rect)

    def show_hs(self):
        self.screen.fill((100, 100, 200))
        value = str(self.stats.high_score)
        hs = "HIGH SCORE: " + value
        self.hs_image = self.font.render(hs, True, self.text_color, self.bg_color)

        self.hs_rect = self.hs_image.get_rect()
        self.hs_rect.center = self.screen_rect.center
        self.hs_rect.top = self.screen_rect.top

        self.start.blit_back()
        self.screen.blit(self.hs_image, self.hs_rect)
        pygame.display.flip()


from selenium import webdriver
from selenium.webdriver.common.by import By
import os
import csv
from sds_parser.parser import sds_parser, ChemicalData
from tqdm import tqdm
from time import sleep

class SigmaConfigs:

    BASEURL = "https://www.sigmaaldrich.com/catalog/search?term=potassium&interface=All&N=0&mode=match%20partialmax&lang=en&region=US&focus=product"
    SDS_FOLDER = os.getcwd() + '/sigma_aldrich_sds/'
    PROFILE = webdriver.FirefoxProfile('/home/ari/.mozilla/firefox/fqt1kmcn.default-1544991237172')

    def __init__(self):
        self.url = self.BASEURL

def sigma_scraper():

    clear_sds_folder()

    sigma = SigmaConfigs()

    with open('sigma_sds_data.csv', 'w+') as csv_file:
        csv_writer = csv.writer(csv_file)
        csv_writer.writerow([category.name for category in ChemicalData.CATEGORIES])

        while sigma.url:

            if sigma.url != SigmaConfigs.BASEURL:
                print('scanning next page!')

            scan_page(sigma.url, csv_writer)

            sigma.url = get_next_page_url(sigma.url)

def scan_page(url, csv_writer):

    # open new window
    driver = webdriver.Firefox(SigmaConfigs.PROFILE)
    driver.get(url)
    driver.implicitly_wait(30)

    link_actions = []

    sds_buttons = driver.find_elements_by_class_name('msdsBulletPoint')
    for idx, button in enumerate(sds_buttons):
        link_action = button.get_attribute('href')
        link_actions.append(link_action)
        print(link_action)

    for action in link_actions:
        try:
            driver.execute_script(action)
        except:
            print('Null JS action')
        sleep(5)
        driver.get(url)
        driver.implicitly_wait(30)

        # gather data and write row to csv if file exists
        if len(os.listdir(SigmaConfigs.SDS_FOLDER)) != 0:
            chemical_data = get_data_from_pdf()
            if chemical_data:
                csv_writer.writerow(chemical_data)
            clear_sds_folder()

def clear_sds_folder():

    for file in os.listdir(SigmaConfigs.SDS_FOLDER):
        os.remove(os.path.join(SigmaConfigs.SDS_FOLDER, file))

def get_data_from_pdf():
    old_file_name = SigmaConfigs.SDS_FOLDER + os.listdir(SigmaConfigs.SDS_FOLDER)[0]
    new_file_name = old_file_name.split('.')[0] + '.pdf'
    os.rename(old_file_name, new_file_name)
    chemical_data = sds_parser(new_file_name)
    return chemical_data

def get_next_page_url(url):
    driver = webdriver.Firefox(SigmaConfigs.PROFILE)
    driver.implicitly_wait(30)
    driver.get(url)
    driver.implicitly_wait(30)
    next_page_button = driver.find_element(By.XPATH, '//a[@rel="next"]')
    next_page_url = ''
    if next_page_button:
        next_page_url = SigmaConfigs.BASEURL + next_page_button.get_attribute('href')
    driver.close()
    return next_page_url

def get_num_sds_on_page(url):
    driver = webdriver.Firefox(SigmaConfigs.PROFILE)
    driver.implicitly_wait(30)
    driver.get(url)
    driver.implicitly_wait(30)
    sds_count = len(driver.find_elements_by_class_name('msdsBulletPoint'))
    driver.close()
    return sds_count

if __name__ == '__main__':
    sigma_scraper()


# ----------------------------------------------------------------------------
# Copyright (c) 2016--, QIIME development team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file LICENSE, distributed with this software.
# ----------------------------------------------------------------------------

import unittest
import json
import tempfile
import tarfile
import os

from sklearn.externals import joblib

from .._taxonomic_classifier import (
    TaxonomicClassifier, TaxonomicClassifierDirFmt, JSONFormat, PickleFormat)
from . import FeatureClassifierTestPluginBase

class TestTypes(FeatureClassifierTestPluginBase):
    def test_taxonomic_classifier_semantic_type_registration(self):
        self.assertRegisteredSemanticType(TaxonomicClassifier)

    def test_taxonomic_classifier_semantic_type_to_format_registration(self):
        self.assertSemanticTypeRegisteredToFormat(
            TaxonomicClassifier, TaxonomicClassifierDirFmt)

class TestFormats(FeatureClassifierTestPluginBase):
    package = 'q2_feature_classifier.tests'

    def test_taxonomic_classifier_dir_fmt(self):
        format = self._setup_dir(['sklearn_pipeline.tar',
                                  'preprocess_params.json'],
                                 TaxonomicClassifierDirFmt)
        # Should not error
        format.validate()

class TestTransformers(FeatureClassifierTestPluginBase):
    package = 'q2_feature_classifier.tests'

    def test_taxo_class_dir_fmt_to_taxo_class_result(self):
        transformer = self.get_transformer(TaxonomicClassifierDirFmt, dict)
        input = self._setup_dir(['sklearn_pipeline.tar',
                                 'preprocess_params.json'],
                                TaxonomicClassifierDirFmt)

        obs = transformer(input)
        exp = ['params', 'pipeline']

        self.assertSetEqual(set(obs.keys()), set(exp))

    def test_taxo_class_result_to_taxo_class_dir_fmt(self):
        transformer = self.get_transformer(dict, TaxonomicClassifierDirFmt)
        params_filepath = self.get_data_path('preprocess_params.json')
        pipeline_filepath = self.get_data_path('sklearn_pipeline.tar')

        with open(params_filepath) as fh:
            params = json.load(fh)

        def read_pipeline(pipeline_filepath):
            with tarfile.open(pipeline_filepath) as tar:
                dirname = tempfile.mkdtemp()
                tar.extractall(dirname)
                pipeline = joblib.load(os.path.join(dirname,
                                       'sklearn_pipeline.pkl'))
                for fn in tar.getnames():
                    os.unlink(os.path.join(dirname, fn))
                os.rmdir(dirname)
            return pipeline
        pipeline = read_pipeline(pipeline_filepath)

        exp = {'params': params, 'pipeline': pipeline}

        obs = transformer(exp)

        preprocess_params = obs.preprocess_params.view(JSONFormat)
        sklearn_pipeline = obs.sklearn_pipeline.view(PickleFormat)

        with preprocess_params.open() as fh:
            obs_params = json.load(fh)
        obs_pipeline = read_pipeline(str(sklearn_pipeline))

        obs = {'params': obs_params, 'pipeline': obs_pipeline}

        self.assertEqual(obs['params'], exp['params'])
        self.assertTrue(obs['pipeline'])

if __name__ == "__main__":
    unittest.main()


from netgan.netgan import *
from netgan import utils

from sklearn.metrics import roc_auc_score, average_precision_score
from matplotlib import pyplot as plt
import scipy.sparse as sp
import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
import tensorflow as tf
import numpy as np
import time

def fit(adj):
    '''does the thing
        parameters:
            adj (scipy sparse csr): adjacency matrix for the input graph
        output:
            model (?): the trained model
    '''
    lcc = utils.largest_connected_components(adj)
    adj = adj[lcc,:][:,lcc]
    n = adj.shape[0]

    val_share = 0.1
    test_share = 0.05

    # split the graph into train/test/validation
    train_ones, val_ones, val_zeros, test_ones, test_zeros = utils.train_val_test_split_adjacency(adj, val_share, test_share, undirected=True, connected=True, asserts=True)

    # generate the training graph and ensure it is symmetric
    train_graph = sp.coo_matrix((np.ones(len(train_ones)),(train_ones[:, 0],train_ones[:,1]))).tocsr()
    assert (train_graph.toarray() == train_graph.toarray().T).all()

    rw_len = 16
    batch_size = 128

    walker = utils.RandomWalker(train_graph, rw_len, p=1, q=1, batch_size=batch_size)

    # define the model
    model = NetGAN(n, rw_len, walk_generator=walker.walk, \
            gpu_id=0, use_gumbel=True, disc_iters=3, \
            W_down_generator_size=128, W_down_discriminator_size=128, \
            l2_penalty_generator=1e-7, l2_penalty_discriminator=5e-5, \
            generator_layers=[40], discriminator_layers=[30], \
            temp_start=5, learning_rate=0.0003)

    # stopping criterion can be one of 'val' or 'eo'
    stopping_criterion = 'val'
    if stopping_criterion == 'eo':
        stopping = 0.5
    else:
        stopping = None

    eval_every = 3
    #max_iters = 30000
    max_iters = 4

    # train the model
    log_dict = model.train(A_orig=adj, val_ones=val_ones, val_zeros=val_zeros, \
            stopping=stopping, eval_every=eval_every, max_patience=5, max_iters=max_iters)

    sample_walks = model.generate_discrete(10000, reuse=True)

    samples = []
    for x in range(60):
        samples.append(sample_walks.eval({model.tau: 0.5}))
        #if (x + 1) % 10 == 0:
        #    print(x + 1)

    random_walks = np.array(samples).reshape([-1, rw_len])
    scores_matrix = utils.score_matrix_from_random_walks(random_walks, n).tocsr()

    return scores_matrix, train_graph.sum()

def gen(scores, tg_sum):
    return utils.graph_from_scores(scores, tg_sum)

def main():
    A, _X_obs, _z_obs = utils.load_npz('data/cora_ml.npz')
    A = A + A.T
    A[A > 1] = 1

    scores, tg_sum = fit(A)

    sampled_graph = gen(scores, tg_sum)
    print(sampled_graph)
    np.savetxt('wew2.dat', sampled_graph, fmt='%d')

    #print('--------------------------')
    #print(scores)
    #print(tg_sum)
    #np.save('./scores', scores)

main()


import subprocess
from pathlib import Path

DATA = [
    "dist/libogg.dll;.",
    "dist/libvorbis.dll;.",
    "src/modlunky2/VERSION;.",
    "src/modlunky2/static;static",
    "dist/s99-client.exe;.",
]
BASE_DIR = Path(__file__).parent.resolve()

def run_pyinstaller():
    pyinstaller_args = ["pyinstaller.exe", f"{BASE_DIR / 'pyinstaller-cli.py'}"]
    for data in DATA:
        pyinstaller_args.extend(["--add-data", data])

    pyinstaller_args.extend(
        [
            "--name=modlunky2",
            f"--icon={BASE_DIR / 'src/modlunky2/static/images/icon.ico'}",
            "--clean",
            "--onedir",
            "--noconsole",
            "--noconfirm",
        ]
    )

    process = subprocess.Popen(pyinstaller_args)
    process.communicate()

def build_launcher():
    process = subprocess.Popen(
        [
            "cargo",
            "build",
            f"--manifest-path={BASE_DIR / 'src/launcher/Cargo.toml'}",
            "--release",
        ]
    )
    process.communicate()

def main():
    run_pyinstaller()
    build_launcher()

if __name__ == "__main__":
    main()


import matplotlib
matplotlib.use('Agg')
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages
import pickle
import sys

colors_for_matplotlib = ['r', 'g', 'b', 'c', 'm', 'y', 'k']

ENGINES_NAMES = ["Venture", "Bher", "MIT-Church"]

engine_names_dict = {}
engine_names_dict['Venture'] = 'Venture'
engine_names_dict['Bher'] = 'Transformational Compiler (Bher)'
engine_names_dict['MIT-Church'] = 'MIT-Church'

what_varying = {}
what_varying['T'] = "# of dimensions"
what_varying['W'] = "# of datapoints"

# fig = plt.figure(0)
# plt.clf()
# plt.title("...")
# plt.xlabel(GRAPH_GROUP[variant_factor]['X'])
# plt.ylabel(GRAPH_GROUP[variant_factor]['Y'])

plt.legend(('Line 1', 'Line 2'), 'upper left')

subplot_id = 0
for engine_name in sorted(ENGINES_NAMES):
  GRAPH_GROUP = pickle.load(open(engine_name, "r"))
  for variant_factor in GRAPH_GROUP.keys():
    subplot_id += 1
    plt.subplot(3, 2, subplot_id)
    plt.grid('on')

    variant_engine_id = -1
    for variant_engine in GRAPH_GROUP[variant_factor]['data'].keys():
      variant_engine_id += 1
      attempts = 0
      for THIS_ATTEMPT in GRAPH_GROUP[variant_factor]['data'][variant_engine]:
        attempts += 1
        mydict = THIS_ATTEMPT
        XS = []
        YS = []
        for key in sorted(mydict.iterkeys()):
          XS += [mydict[key]['x']]
          YS += [mydict[key]['y']]

        plt.title(engine_names_dict[engine_name])
        if "NotOpt" in variant_engine:
          color = "g"
        elif "Opt" in variant_engine:
          color = "b"
        elif "Unco" in variant_engine:
          color = "r"
        plt.plot(XS, YS, color, label = variant_engine + ", attempt #" + str(attempts), alpha=1.0)
        plt.xlabel(what_varying[variant_factor])
        plt.ylabel("Time per sweep")

# pie = plt.pie([0,0], labels=["a","b"])
# l = plt.legend(loc="upper left")
# for group in pie:
    # for x in group:
        # x.set_visible(False)

plt.tight_layout()

# plt.clf()
# l1 = plt.plot([0.3,0.3,0.3], [0.3,0.3,0.3], "r", label = "Uncollapsed")
# l2 = plt.plot([0.3,0.3,0.3], [0.3,0.3,0.3], "g", label = "Collapsed")
# l3 = plt.plot([0.3,0.3,0.3], [0.3,0.3,0.3], "b", label = "Collapsed (optimized, Venture only)")
# l = plt.legend(loc="upper left")

plt.savefig("./result.pdf", format='pdf', bbox_inches='tight')


import sys
import os
import stat
import errno
import time
import shutil
import subprocess
import popenasync

try:
    from shutil import ignore_patterns, copytree
    # Running Python > 2.5
except ImportError:
    # Running Python <= 2.5
    # Imported from Python 2.7 module shutil
    import fnmatch
    def ignore_patterns(*patterns):
        def _ignore_patterns(path, names):
            ignored_names = []
            for pattern in patterns:
                ignored_names.extend(fnmatch.filter(names, pattern))
            return set(ignored_names)
        return _ignore_patterns
    def copytree(src, dst, symlinks=False, ignore=None):
        names = os.listdir(src)
        if ignore is not None:
            ignored_names = ignore(src, names)
        else:
            ignored_names = set()

        os.makedirs(dst)
        errors = []
        for name in names:
            if name in ignored_names:
                continue
            srcname = os.path.join(src, name)
            dstname = os.path.join(dst, name)
            try:
                if symlinks and os.path.islink(srcname):
                    linkto = os.readlink(srcname)
                    os.symlink(linkto, dstname)
                elif os.path.isdir(srcname):
                    copytree(srcname, dstname, symlinks, ignore)
                else:
                    # Will raise a SpecialFileError for unsupported file types
                    shutil.copy2(srcname, dstname)
            # catch the Error from the recursive copytree so that we can
            # continue with other files
            except shutil.Error as err:
                errors.extend(err.args[0])
            except EnvironmentError as why:
                errors.append((srcname, dstname, str(why)))
        try:
            shutil.copystat(src, dst)
        except OSError as why:
            if WindowsError is not None and isinstance(why, WindowsError):
                # Copying file access times may fail on Windows
                pass
            else:
                errors.extend((src, dst, str(why)))
        if errors:
            raise Error(errors)

def replace_in_file(src, dst, vars):
    f = open(src, "rt")
    content =  f.read()
    f.close()
    for k in vars:
        content = content.replace(k, vars[k])
    f = open(dst, "wt")
    f.write(content)
    f.close()

def run_process(args, logger=None):
    def log(buffer, checkNewLine):
        endsWithNewLine = False
        if buffer.endswith('\n'):
            endsWithNewLine = True
        if checkNewLine and buffer.find('\n') == -1:
            return buffer
        lines = buffer.splitlines()
        buffer = ''
        if checkNewLine and not endsWithNewLine:
            buffer = lines[-1]
            lines = lines[:-1]
        for line in lines:
            if not logger is None:
                logger.info(line.rstrip('\r'))
            else:
                print(line.rstrip('\r'))
        return buffer

    shell = False
    if sys.platform == "win32":
        shell = True

    proc = popenasync.Popen(args,
        stdin = subprocess.PIPE,
        stdout = subprocess.PIPE,
        stderr = subprocess.STDOUT,
        universal_newlines = 1,
        shell = shell,
        env = os.environ)

    log_buffer = None;
    while proc.poll() is None:
        log_buffer = log(proc.read_async(wait=0.1, e=0), False)
    if log_buffer:
        log(log_buffer, False)

    proc.wait()
    return proc.returncode

# LINKS:
#   http://stackoverflow.com/questions/1213706/what-user-do-python-scripts-run-as-in-windows/1214935#1214935
def rmtree(filename):
    def handleRemoveReadonly(func, path, exc):
        excvalue = exc[1]
        if func in (os.rmdir, os.remove) and excvalue.errno == errno.EACCES:
            os.chmod(path, stat.S_IRWXU| stat.S_IRWXG| stat.S_IRWXO) # 0777
            func(path)
        else:
            raise
    shutil.rmtree(filename, ignore_errors=False, onerror=handleRemoveReadonly)

# LINKS:
#   http://snippets.dzone.com/posts/show/6313
def find_executable(executable, path=None):
    """Try to find 'executable' in the directories listed in 'path' (a
    string listing directories separated by 'os.pathsep'; defaults to
    os.environ['PATH']).  Returns the complete filename or None if not
    found
    """
    if path is None:
        path = os.environ['PATH']
    paths = path.split(os.pathsep)
    extlist = ['']
    if os.name == 'os2':
        (base, ext) = os.path.splitext(executable)
        # executable files on OS/2 can have an arbitrary extension, but
        # .exe is automatically appended if no dot is present in the name
        if not ext:
            executable = executable + ".exe"
    elif sys.platform == 'win32':
        pathext = os.environ['PATHEXT'].lower().split(os.pathsep)
        (base, ext) = os.path.splitext(executable)
        if ext.lower() not in pathext:
            extlist = pathext
    for ext in extlist:
        execname = executable + ext
        if os.path.isfile(execname):
            return execname
        else:
            for p in paths:
                f = os.path.join(p, execname)
                if os.path.isfile(f):
                    return f
    else:
        return None


'''
Created on 2016/03/07

@author: kot
'''

import numpy as np

from engines.classifier import sgdclassifier
from actions.gen_pf_bollinger import *
import lib.common_funcs as f
import lib.kabuka_funcs as kf
import lib.tables as tbl
import lib.sqlrun as sql
from config import *

class SupervisedClassifier(object):
    '''
    classdocs
    '''
    ma_classifier = None
    mi_classifier = None
    prgname = ""
    chart_prg = ""
    meigaras = []
    startd = ""
    endd = ""
    ml_algorithm = ""
    kabuka_generator = None
    table_name = ""
    #sd_table = ""

    def __init__(self, prgname, meigaras=[]):
        '''
        Constructor
        '''
        rootd = f.get_json_data(prgname)
        d = rootd["play_fields"]
        self.chart_prg = d["prg_name"]
        self.ml_algorithm = d["ml_algorithm"]
        startd = d["startd"]
        endd = d["endd"]
        self.tmp_table_name = d["tmp_table"]
        self.table_name = d["table"]
        #self.sd_table = d["sd_table"]
        self.meigaras = meigaras
        self.startd = startd
        self.endd = endd

        tbl.create_table_from_tmpl("pf", [self.table_name])

        #if self.sd_table != "":
        #    tbl.create_table_from_tmpl("sd", [self.sd_table])

        #self.kabuka_generator = kf.g_get_kabuka(meigaras, startd, endd)

        classifier = None
        if self.ml_algorithm == "sgd_classifier":
            self.ma_classifier = sgdclassifier.SGDClassifier("buy", meigaras)
            self.mi_classifier = sgdclassifier.SGDClassifier("sel", meigaras)

    def learn(self):
        old_code = ""
        dl, ol, hl, ll, cl, vl = ([], [], [], [], [], [])
        X = []
        yma = []
        ymi = []
        kabuka_generator = kf.g_get_kabuka(self.meigaras, self.startd, self.endd)

        for line in kabuka_generator:
            (code, date, open, high, low, close, volume) = line
            if code != old_code and old_code != "":
                if self.chart_prg == "gen_bollinger_field":
                    tpl = gen_pf_bollinger(dl, ol, hl, ll, cl, vl)
                    if len(tpl) == 4:
                        (d, xx, ma, mi) = tpl
                    debug = False
                    if debug:
                        ma_labels = self.ma_classifier.predict(xx)
                        mi_labels = self.mi_classifier.predict(xx)
                X.extend(xx)
                yma.extend(ma)
                ymi.extend(mi)
                dl, ol, hl, ll, cl, vl = ([], [], [], [], [], [])

            dl.append(date)
            ol.append(open)
            hl.append(high)
            ll.append(low)
            cl.append(close)
            vl.append(volume)

            old_code = code

        self.ma_classifier.fit(X, yma)
        self.ma_classifier.save()
        self.mi_classifier.fit(X, ymi)
        self.mi_classifier.save()

    def predict2tmp(self):
        old_code = ""
        dl, ol, hl, ll, cl, vl = ([], [], [], [], [], [])
        kabuka_generator = kf.g_get_kabuka(self.meigaras, self.startd, self.endd)

        for line in kabuka_generator:
            (code, date, open, high, low, close, volume) = line
            if code != old_code and old_code != "":
                if self.chart_prg == "gen_bollinger_field":
                    tpl = gen_pf_bollinger(dl, ol, hl, ll, cl, vl)
                    if len(tpl) == 4:
                        (d, X, Y) = tpl
                    else:
                        continue
                if len(X) > 0:
                    labels = np.array(self.classifier.predict(X))
                else:
                    continue
                data = []
                for i in range(len(d)):
                    data.append([old_code, d[i], "", labels[i], Y[i][0], Y[i][1]])
                if len(data) > 0:
                    tbl.arr2table(data, self.table_name)
                data = []
                dl, ol, hl, ll, cl, vl = ([], [], [], [], [], [])

            dl.append(date)
            ol.append(open)
            hl.append(high)
            ll.append(low)
            cl.append(close)
            vl.append(volume)

            old_code = code

    def tmp2pf(self):
        fieldids = {}
        strsql = "select fieldid, size, avg(max_price), std(max_price), avg(min_price), \
std(min_price) from trade.pf_bollinger group by fieldid, size;"
        generator = sql.g_get_data(strsql)
        for row in generator:
            (fieldid, size, avg_max, std_max, avg_min, std_min) = row
            fieldids["%s-%d" % (fieldid, size)] = 1

    def create_score_dist(self):
        pass


import re

message = "Call me 415-555-1011 tomorrow, or at 415-555-9999 for my office line"

#create regex object
phoneNumRegex = re.compile(r"\d\d\d-\d\d\d-\d\d\d\d")
mo = phoneNumRegex.search(message)
#get matched string
print(mo.group())
mo = phoneNumRegex.findall(message)
print(mo)


# Copyright (c) 2011-2014 Berkeley Model United Nations. All rights reserved.
# Use of this source code is governed by a BSD License (see LICENSE).

from django import template
register = template.Library()

@register.filter
def is_advisor(user):
    try:
        return user.is_advisor()
    except:
        return False

@register.filter
def is_chair(user):
    try:
        return user.is_chair()
    except:
        return False


### In this program we have to  calculate the area in sq.feet,
### After calculation we have to convert it into acre by dividing area by 43560.

l = float(input("Enter the length of the field in feets >> ")) # Taken l to represent length of field.
b = float(input("Enter the width of the field in feets >> ")) # Taken b to represent width of field.
area = l * b / 43560
print("The area of field in acre is %.3f"%area)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models
import wagtail.wagtailcore.fields
import wagtail.wagtailcore.blocks

class Migration(migrations.Migration):

    dependencies = [
        ('v1', '0114_update_menu_item_link_fields'),
    ]

    operations = [
        migrations.AlterField(
            model_name='menuitem',
            name='nav_footer',
            field=wagtail.wagtailcore.fields.StreamField([(b'nav_footer', wagtail.wagtailcore.blocks.StructBlock([(b'draft', wagtail.wagtailcore.blocks.BooleanBlock(help_text=b'If checked, this block will only show on our sharing site (Content).', default=False, required=False, label=b'Mark block as draft')), (b'content', wagtail.wagtailcore.blocks.RichTextBlock(required=False, features=[b'link']))], label=b'Menu footer'))], blank=True),
        ),
    ]


import autograd.numpy as np
from autograd import grad

def sigmoid(x):
    return 1/(1+np.exp(-x))

class LogisticRegression(object):
    def __init__(self, X_train, y_train, learning_rate):
        self.X_train = X_train
        self.y_train = y_train
        self.samples = self.X_train.shape[0]
        self.features = self.X_train.shape[1]
        self.learning_rate = learning_rate
        self.iterations = 100
        self.theta = None
        self.gradient_func = grad(self.training_loss)

    def training_loss(self, theta):
        return self.cost(theta)

    def initialise_theta(self):
        params = np.random.rand(self.features, 1)
        self.theta = params

    def cost(self, theta):
        Yy = sigmoid(self.X_train.dot(theta))
        cost = 0.
        for i in range(self.samples):
            cost -= (self.y_train[i]*np.log10(Yy[i]) + (1-self.y_train[i])*np.log10(1 - Yy[i]))
        return cost/self.samples

    def update_theta(self):
        current_theta = self.theta
        current_theta -= self.gradient_func(current_theta)*self.learning_rate
        self.theta = current_theta
        return current_theta

    def train(self):
        theta = None
        self.initialise_theta()
        error = 0.00001
        for i in xrange(self.iterations):
            print('')
            theta = self.update_theta()
            print('Iteration -  '+ str(i+1))
            print('')
            if self.MSE(theta) <= error:
                break
        print('### Training complete')

    def predict(self, X_test):
        '''
        Assumes thats X_test has the bias festure added to it, however an assert is done
        '''
        assert(X_test.shape[1] == self.theta.shape[0])
        predictions = X_test.dot(self.theta)
        self.predictions = predictions
        return predictions