from email.mime import image
import os
from common.input_distortion import InputDistortion
import numpy as np
from common.arguments import parse_args
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import os
import sys
import errno
import logging
import datetime
from common.camera import *
from common.loss   import *
from common.data   import *
from common.render import *
from common.model  import *
from common.evaluate import evaluate, run_evaluation
from common.model_factory import get_model, initialize_model
from common.generators import ChunkedGenerator, UnchunkedGenerator, EvaluateGenerator
from time import time
from common.utils import deterministic_random, load_cfg_from_file, set_momentum
from tqdm import tqdm
import os.path as osp
# Parse argument
args = parse_args()
# Load arguments from file
args = load_cfg_from_file(args, args.cfg_file)
print(args, "\n")
# Set GPU device
os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"]=args.gpu
# Making the experiments reproducible
import random
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)

# Create checkpoint directory
if args.checkpoint == "auto":
    checkpoint_root = "checkpoint"
    args.checkpoint = osp.join(checkpoint_root, "%s-%s-a%s-b%d-dj_%s-dp_%s-df%s-lss_exc_%s-conf_%s%s%s" % (
        args.model, args.keypoints, args.architecture, args.batch_size,
        args.train_distortion_type, args.train_distortion_parts, args.train_distortion_temporal,
        args.loss_ignore_parts,
        'det' if ('hrnet' in args.keypoints) and  args.drop_conf_score == False else args.train_gen_conf_score,
        '_' + args.loss if args.loss != 'mpjpe' else '',
        '_smthconf' if args.smooth_conf_score else ''
        ))
print("Saving checkpoint to ", args.checkpoint, "\n")
try:
    # Create checkpoint directory if it does not exist
    os.makedirs(args.checkpoint)
except OSError as e:
    if e.errno != errno.EEXIST:
        raise RuntimeError('Unable to create checkpoint directory:', args.checkpoint)

# Prepare input distortion instance
inp_distr = InputDistortion(args)
# Prepare dataset
dtf = DataFetcher(args)
# Fetch validation dataset
cameras_valid, poses_valid, poses_valid_2d = dtf.fetch_test()
# Apply distortion on validation data input
poses_valid_2d = [inp_distr.get_test_inputs(i) for i in poses_valid_2d]

model_pos, model_pos_train, checkpoint = initialize_model(args,
    num_joints_in  = poses_valid_2d[0].shape[-2] if len(poses_valid_2d) > 0 else 17,
    num_joints_out = dtf.dataset.skeleton().num_joints(),
    in_features    =  poses_valid_2d[0].shape[-1] if len(poses_valid_2d) > 0 else 3 if args.drop_conf_score == False else 2,
    chk_filename   = os.path.join(args.checkpoint, args.resume if args.resume else args.evaluate)
)

model_traj = None
# Generate test data
if hasattr(model_pos, 'receptive_field'):
    receptive_field = model_pos.receptive_field()
    pad = (receptive_field-1)//2 # Padding on each side
else:
    receptive_field = 1
    for i in args.architecture.split(','):
        receptive_field *= int(i)
        pad = (receptive_field-1)//2
print('INFO: Receptive field: {} frames'.format(receptive_field))

if args.causal:
    print('INFO: Using causal convolutions')
    causal_shift = pad
else:
    causal_shift = 0

if args.test_fixed_size_input:
    test_generator = ChunkedGenerator(args.batch_size // args.stride, cameras_valid, poses_valid, poses_valid_2d, args.stride,
                                  pad=pad, causal_shift=causal_shift, augment=args.test_time_augmentation, shuffle=False,
                                  kps_left=dtf.kps_left, kps_right=dtf.kps_right, joints_left=dtf.joints_left, joints_right=dtf.joints_right)
else:
    test_generator = UnchunkedGenerator(cameras_valid, poses_valid, poses_valid_2d,
                                        pad=pad, causal_shift=causal_shift, augment=False,
                                        kps_left=dtf.kps_left, kps_right=dtf.kps_right, joints_left=dtf.joints_left, joints_right=dtf.joints_right)
print('INFO: Testing on {} frames'.format(test_generator.num_frames()))

################### TRAINING ###################
if not args.evaluate:
    # Prepare log dir
    import wandb
    wandb.init(project="robustvp3d", name=args.wandb_name, config=args)

    logging.basicConfig(filename= #"test.log",
    osp.join(args.checkpoint, "%s.log" % datetime.datetime.now()),
    filemode='a',
    format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s',
    datefmt='%H:%M:%S',
    level=logging.INFO)
    logging.getLogger().addHandler(logging.StreamHandler())
    # Save args to log file
    logging.info(args)

    # Fetch train data
    cameras_train, poses_train, poses_train_2d = dtf.fetch_train()
    # Apply distortion on input training data
    # from matplotlib import pyplot as plt
    # plt.plot(poses_train_2d[0][:,11,0])
    poses_train_2d = [inp_distr.get_train_inputs(i) for i in poses_train_2d]
    # Prepare optimizers
    lr = args.learning_rate
    optimizer = optim.Adam(model_pos_train.parameters(), lr=lr, amsgrad=True)
    lr_decay = args.lr_decay

    min_loss = 1e4
    losses_3d_train = []
    losses_3d_train_eval = []
    losses_3d_valid = []

    epoch = 0
    initial_momentum = 0.1
    final_momentum = 0.001
    train_generator = ChunkedGenerator(args.batch_size//args.stride, cameras_train, poses_train, poses_train_2d, args.stride,
                                       pad=pad, causal_shift=causal_shift, shuffle=True, augment=args.data_augmentation,
                                       kps_left=dtf.kps_left, kps_right=dtf.kps_right, joints_left=dtf.joints_left, joints_right=dtf.joints_right)
    train_generator_eval = UnchunkedGenerator(cameras_train, poses_train, poses_train_2d,
                                              pad=pad, causal_shift=causal_shift, augment=False)
    print('INFO: Training on {} frames'.format(train_generator_eval.num_frames()))
    if args.resume:
        epoch = checkpoint['epoch']
        if 'optimizer' in checkpoint and checkpoint['optimizer'] is not None:
            optimizer.load_state_dict(checkpoint['optimizer'])
            train_generator.set_random_state(checkpoint['random_state'])
        else:
            print('WARNING: this checkpoint does not contain an optimizer state. The optimizer will be reinitialized.')

        lr = checkpoint['lr']

    print('** Note: reported losses are averaged over all frames and test-time augmentation is not used here.')
    print('** The final evaluation will be carried out after the last training epoch.')

    # Pos model only
    while epoch < args.epochs:
        start_time = time()
        epoch_loss_3d_train = 0
        epoch_loss_traj_train = 0
        epoch_loss_2d_train_unlabeled = 0
        N = 0
        N_semi = 0
        model_pos_train.train()
        # Regular supervised scenario
        for _, batch_3d, batch_2d in tqdm(train_generator.next_epoch(), total=train_generator.num_batches):
            inputs_3d = torch.from_numpy(batch_3d.astype('float32'))
            inputs_2d = torch.from_numpy(batch_2d.astype('float32'))
            if torch.cuda.is_available():
                inputs_3d = inputs_3d.cuda()
                inputs_2d = inputs_2d.cuda()
            inputs_3d[:, :, 0] = 0

            optimizer.zero_grad()
            # inputs_2d = inp_distr.smooth_conf_scr(inputs_2d)
            # Predict 3D poses
            predicted_3d_pos = model_pos_train(inputs_2d)
            # Select joints for computing losses
            predicted_3d_pos = inp_distr.get_loss_joints(predicted_3d_pos)
            inputs_3d        = inp_distr.get_loss_joints(inputs_3d)

            if   args.loss == 'mpjpe':
                loss_3d_pos = mpjpe(predicted_3d_pos, inputs_3d)
            elif args.loss == 'l1':
                loss_3d_pos = L1_loss(predicted_3d_pos, inputs_3d)
            elif args.loss == "conf_mpjpe":
                loss_3d_pos = conf_mpjpe(predicted_3d_pos, inputs_3d, inputs_2d[...,-1])
            epoch_loss_3d_train += inputs_3d.shape[0]*inputs_3d.shape[1] * loss_3d_pos.item()
            N += inputs_3d.shape[0]*inputs_3d.shape[1]

            loss_total = loss_3d_pos
            loss_total.backward()
            optimizer.step()
            wandb.log({"loss_3d_pos": loss_3d_pos.item()*1000})

        losses_3d_train.append(epoch_loss_3d_train / N)

        # End-of-epoch evaluation
        with torch.no_grad():
            model_pos.load_state_dict(model_pos_train.state_dict())
            model_pos.eval()
            epoch_loss_3d_valid = 0
            epoch_loss_traj_valid = 0
            epoch_loss_2d_valid = 0
            N = 0

            if not args.no_eval:
                # Evaluate on test set
                for cam, batch, batch_2d in test_generator.next_epoch():
                    inputs_3d = torch.from_numpy(batch.astype('float32'))
                    inputs_2d = torch.from_numpy(batch_2d.astype('float32'))
                    if args.smooth_conf_score == True or "PoseFormer" in args.model:
                        inputs_2d, inputs_3d = eval_data_prepare(inputs_2d, inputs_3d, receptive_field)

                    if torch.cuda.is_available():
                        inputs_3d = inputs_3d.cuda()
                        inputs_2d = inputs_2d.cuda()
                    inputs_traj = inputs_3d[:, :, :1].clone()
                    inputs_3d[:, :, 0] = 0

                    # Smooth conf. score (if enabled)
                    # inputs_2d = inp_distr.smooth_conf_scr(inputs_2d)
                    # Predict 3D poses
                    predicted_3d_pos = model_pos(inputs_2d)
                    # Select joints for evaluation
                    predicted_3d_pos = inp_distr.get_eval_joints(predicted_3d_pos)
                    inputs_3d        = inp_distr.get_eval_joints(inputs_3d)
                    try:
                        loss_3d_pos = mpjpe(predicted_3d_pos, inputs_3d)
                    except:
                        import ipdb; ipdb.set_trace()
                        loss_3d_pos = mpjpe(predicted_3d_pos[:, :inputs_3d.shape[1]], inputs_3d)
                    epoch_loss_3d_valid += inputs_3d.shape[0]*inputs_3d.shape[1] * loss_3d_pos.item()
                    N += inputs_3d.shape[0]*inputs_3d.shape[1]

                losses_3d_valid.append(epoch_loss_3d_valid / N)

                # Evaluate on training set, this time in evaluation mode
                epoch_loss_3d_train_eval = 0
                epoch_loss_traj_train_eval = 0
                epoch_loss_2d_train_labeled_eval = 0
                N = 1 # SHOULD BE ZERO!!
                # for cam, batch, batch_2d in train_generator_eval.next_epoch():
                #     if batch_2d.shape[1] == 0:
                #         # This can only happen when downsampling the dataset
                #         continue

                #     inputs_3d = torch.from_numpy(batch.astype('float32'))
                #     inputs_2d = torch.from_numpy(batch_2d.astype('float32'))

                #     if args.smooth_conf_score == True or "PoseFormer" in args.model:
                #         inputs_2d, inputs_3d = eval_data_prepare(inputs_2d, inputs_3d, receptive_field)

                #     if torch.cuda.is_available():
                #         inputs_3d = inputs_3d.cuda()
                #         inputs_2d = inputs_2d.cuda()
                #     inputs_traj = inputs_3d[:, :, :1].clone()
                #     inputs_3d[:, :, 0] = 0
                #     # Smooth conf. score (if enabled)
                #     # inputs_2d = inp_distr.smooth_conf_scr(inputs_2d)
                #     # Compute 3D poses
                #     predicted_3d_pos = model_pos(inputs_2d)
                #     # Select joints for computing losses
                #     predicted_3d_pos = inp_distr.get_loss_joints(predicted_3d_pos)
                #     inputs_3d        = inp_distr.get_loss_joints(inputs_3d)

                #     loss_3d_pos = mpjpe(predicted_3d_pos, inputs_3d)
                #     epoch_loss_3d_train_eval += inputs_3d.shape[0]*inputs_3d.shape[1] * loss_3d_pos.item()
                #     N += inputs_3d.shape[0]*inputs_3d.shape[1]

                losses_3d_train_eval.append(epoch_loss_3d_train_eval / N)
                wandb.log({"loss_3d_pos_valid": losses_3d_valid[-1]*1000})
                # Evaluate 2D loss on unlabeled training set (in evaluation mode)
                epoch_loss_2d_train_unlabeled_eval = 0
                N_semi = 0

        elapsed = (time() - start_time)/60

        if args.no_eval:
            logging.info('[%d] time %.2f lr %f 3d_train %f' % (
                    epoch + 1,
                    elapsed,
                    lr,
                    losses_3d_train[-1] * 1000))
        else:
            logging.info('[%d] time %.2f lr %f 3d_train %f 3d_eval %f 3d_valid %f' % (
                    epoch + 1,
                    elapsed,
                    lr,
                    losses_3d_train[-1] * 1000,
                    losses_3d_train_eval[-1] * 1000,
                    losses_3d_valid[-1]  *1000))

        # Decay learning rate exponentially
        lr *= lr_decay
        for param_group in optimizer.param_groups:
            param_group['lr'] *= lr_decay
        epoch += 1

        # Decay BatchNorm momentum
        # momentum = initial_momentum * np.exp(-epoch/args.epochs * np.log(initial_momentum/final_momentum))
        # if args.parallel:
        #     set_momentum(model_pos_train, momentum)
        # else:
        #     model_pos_train.set_bn_momentum(momentum)

        # Save checkpoint if necessary
        def save_ckpt(chk_path):
            print('Saving checkpoint to', chk_path)
            torch.save({
                'epoch': epoch,
                'lr': lr,
                'random_state': train_generator.random_state(),
                'optimizer': optimizer.state_dict(),
                'model_pos': model_pos_train.state_dict(),
                'model_traj': None,
                'random_state_semi':  None,
            }, chk_path)
        if epoch % args.checkpoint_frequency == 0:
            save_ckpt(os.path.join(args.checkpoint, 'epoch_{}.bin'.format(epoch)))
        if len(losses_3d_valid) > 0:
            if  losses_3d_valid[-1] * 1000 < min_loss:
                save_ckpt(os.path.join(args.checkpoint, 'best.bin'.format(epoch)))
                min_loss = losses_3d_valid[-1] * 1000

        # Save training curves after every epoch, as .png images (if requested)
        if args.export_training_curves and epoch > 3:
            if 'matplotlib' not in sys.modules:
                import matplotlib
                matplotlib.use('Agg')
                import matplotlib.pyplot as plt

            plt.figure()
            epoch_x = np.arange(3, len(losses_3d_train)) + 1
            plt.plot(epoch_x, losses_3d_train[3:], '--', color='C0')
            plt.plot(epoch_x, losses_3d_train_eval[3:], color='C0')
            plt.plot(epoch_x, losses_3d_valid[3:], color='C1')
            plt.legend(['3d train', '3d train (eval)', '3d valid (eval)'])
            plt.ylabel('MPJPE (mm)')
            plt.xlabel('Epoch')
            plt.xlim((3, epoch))
            plt.savefig(os.path.join(args.checkpoint, 'loss_3d.png'))
            plt.close('all')

def gen_animation(model_pos, input_keypoints):
    ground_truth = None
    if args.viz_subject in dtf.dataset.subjects() and args.viz_action in dtf.dataset[args.viz_subject]:
        if 'positions_3d' in dtf.dataset[args.viz_subject][args.viz_action]:
            ground_truth = dtf.dataset[args.viz_subject][args.viz_action]['positions_3d'][args.viz_camera].copy()

    if ground_truth is None:
        print('INFO: this action is unlabeled. Ground truth will not be rendered.')
    gen = UnchunkedGenerator(None, None, [input_keypoints],
                             pad=pad, causal_shift=causal_shift, augment=args.test_time_augmentation,
                             kps_left=dtf.kps_left, kps_right=dtf.kps_right, joints_left=dtf.joints_left, joints_right=dtf.joints_right)

    # First set of perdiction
    prediction = evaluate(model_pos, gen, inp_distr, dtf, return_predictions=True)

    if model_traj is not None and ground_truth is None:
        raise NotImplementedError

    if args.viz_export is not None:
        print('Exporting joint positions to', args.viz_export)
        # Predictions are in camera space
        np.save(args.viz_export, prediction)

    if ground_truth is not None:
        # Reapply trajectory
        trajectory = ground_truth[:, :1]
        ground_truth[:, 1:] += trajectory
        prediction += trajectory

    # Invert camera transformation
    cam = dtf.dataset.cameras()[args.viz_subject][args.viz_camera]
    if ground_truth is not None:
        prediction = camera_to_world(prediction, R=cam['orientation'], t=cam['translation'])
        ground_truth = camera_to_world(ground_truth, R=cam['orientation'], t=cam['translation'])
    else:
        # If the ground truth is not available, take the camera extrinsic params from a random subject.
        # They are almost the same, and anyway, we only need this for visualization purposes.
        for subject in dtf.dataset.cameras():
            if 'orientation' in dtf.dataset.cameras()[subject][args.viz_camera]:
                rot = dtf.dataset.cameras()[subject][args.viz_camera]['orientation']
                break
        prediction = camera_to_world(prediction, R=rot, t=0)
        # We don't have the trajectory, but at least we can rebase the height
        prediction[:, :, 2] -= np.min(prediction[:, :, 2])
    return prediction, ground_truth

################### RENDERING ###################
if args.render:
    print('Rendering...')
    input_keypoints       = dtf.keypoints[args.viz_subject][args.viz_action][args.viz_camera].copy()
    noisy_input_keypoints = inp_distr.get_test_inputs(input_keypoints)
    # Predict from clean inputs:
    # prediction, ground_truth = gen_animation(input_keypoints)
    prediction_source_1, ground_truth = gen_animation(model_pos, noisy_input_keypoints)
    # Second set of prediction
    model_pos_2, _, _ = initialize_model(args,
        num_joints_in  = poses_valid_2d[0].shape[-2] if len(poses_valid_2d) > 0 else 17,
        num_joints_out = dtf.dataset.skeleton().num_joints(),
        in_features    =  poses_valid_2d[0].shape[-1] if len(poses_valid_2d) > 0 else 3 if args.drop_conf_score == False else 2,
        chk_filename   = os.path.join(args.checkpoint_2, args.resume if args.resume else args.evaluate)
    )
    prediction_source_2, _ = gen_animation(model_pos_2, noisy_input_keypoints)

    anim_output = {
        'Model 1': prediction_source_1,
        'Model 2': prediction_source_2
        }
    if ground_truth is not None and not args.viz_no_ground_truth:
        anim_output['Ground truth'] = ground_truth
    from common.h36m_dataset import h36m_cameras_intrinsic_params
    cam = h36m_cameras_intrinsic_params[args.viz_camera]

    # input_keypoints       = image_coordinates(input_keypoints[..., :2], w=cam['res_w'], h=cam['res_h'])
    noisy_input_keypoints = image_coordinates(noisy_input_keypoints[..., :2], w=cam['res_w'], h=cam['res_h'])
    ignore_joints = None
    # ignore_joints = np.where(inp_distr.get_mask(inp_distr.body_parts) == 0)[0]
    from common.visualization import dist_render_animation, render_animation

    dist_render_animation(noisy_input_keypoints, dtf.keypoints_metadata, anim_output,
                        dtf.dataset.skeleton(), dtf.dataset.fps(), args.viz_bitrate, cam['azimuth'], args.viz_output,
                        limit=args.viz_limit, downsample=args.viz_downsample, size=args.viz_size,
                        input_video_path=args.viz_video, viewport=(cam['res_w'], cam['res_h']),
                        input_video_skip=args.viz_skip, ignore_joints=ignore_joints)
else:
    print('Evaluating...')

    if not args.by_subject:
        run_evaluation(model_pos, actions=dtf.all_actions, args=args, dtf=dtf, inp_distr=inp_distr, action_filter=dtf.action_filter)
    else:
        for subject in dtf.all_actions_by_subject.keys():
            print('Evaluating on subject', subject)
            run_evaluation(model_pos, actions=dtf.all_actions_by_subject[subject], args=args, dtf=dtf, inp_distr=inp_distr, action_filter=dtf.action_filter)
            print('')


from __future__ import print_function
import numpy as np
import pandas as pd
import os
from PIL import Image

iris_path="Data/IRIS/iris.data"
earthquake_path="Data/Earthquake/Earthquakes"
osuleaf_path="Data/OSULeaf/OSULeaf"
face_path="/media/parthosarothi/OHWR/Dataset/UCI_CMU_Faces"

def load_iris_data(iris_csv):
    f=open(iris_csv)
    line=f.readline()
    iris_df=pd.DataFrame()
    while line:
        info=line.strip("\n").split(",")
        if(len(info)==5):
            label=info[-1]
            features=info[:4]
            features.append(label)
            #print(features)
            df = pd.DataFrame([features], columns=['SL', 'SW', 'PL', 'PW', 'class'])
            iris_df=iris_df.append(df)
        line=f.readline()
    return iris_df

def load_and_split_iris(iris_path,split_ratio):
    iris_df = load_iris_data(iris_path)
    class_labels = list(set(iris_df['class']))
    print(class_labels)
    train_set=pd.DataFrame()
    test_set=pd.DataFrame()
    for cl in class_labels:
        class_loc=iris_df.loc[iris_df['class']==cl]
        nb_samples=len(class_loc)
        class_loc=class_loc.sample(nb_samples)
        #print(class_loc)
        test_volume=int(nb_samples*split_ratio)
        train_volume=nb_samples-test_volume
        #print(class_loc.iloc[train_volume:,:])
        train_set=train_set.append(class_loc.iloc[:train_volume,:])
        test_set=test_set.append(class_loc.iloc[train_volume:,:])

    nbtests=len(test_set)
    test_set=test_set.sample(nbtests)

    nbtrain = len(train_set)
    train_set = train_set.sample(nbtrain)
    print("Train samples=%d Test samples=%d"%(nbtrain,nbtests))
    return train_set,test_set,class_labels

def make_one_hot(batch_labels,all_labels):
    total_classes=len(all_labels)
    total_samples=len(batch_labels)
    one_hot=np.zeros([total_samples,total_classes],dtype=float)
    for ts in range(total_samples):
        one_hot[ts][all_labels.index(batch_labels[ts])]=1
    return one_hot

def load_earthquake_data(earthquake_csv):
    f=open(earthquake_csv)
    line=f.readline()
    sequence_length=[]
    labels=[]
    features=[]
    while line:
        info=line.strip("\n").split(",")
        label=info[0]
        label_one_hot=[0,0]
        label_one_hot[int(label)]=1
        feat=info[1:]
        nbfeatures=len(feat)
        sequence_length.append(nbfeatures)
        labels.append(label_one_hot)
        features.append(feat)
        line=f.readline()
    max_length=max(sequence_length)
    min_length=min(sequence_length)
    print("Maximum sequence length: ",max_length," minimum sequence length: ",min_length)
    return np.asarray(features),np.asarray(labels),sequence_length

def load_timeseries_data(timeseries_csv,nbclass,class_label_from_0=False):
    f=open(timeseries_csv)
    line=f.readline()
    sequence_length=[]
    labels=[]
    features=[]
    while line:
        info=line.strip("\n").split(",")
        label=info[0]
        label_one_hot=np.zeros([nbclass])
        if(class_label_from_0):
            label_index=int(label)
        else:
            label_index=int(label)-1
        label_one_hot[label_index]=1
        feat=info[1:]
        nbfeatures=len(feat)
        sequence_length.append(nbfeatures)
        labels.append(label_one_hot)
        features.append(feat)
        line=f.readline()
    max_length=max(sequence_length)
    min_length=min(sequence_length)
    print("Maximum sequence length: ",max_length," minimum sequence length: ",min_length)
    return np.asarray(features),np.asarray(labels),sequence_length

def split_cmu_face_images(face_path,resolution,test_split):
    all_files=[]
    for root,sd,files in os.walk(face_path):
        for fn in files:
            # print(fn)
            if(fn[-3:]=='pgm'):
                filename = fn[:-4]
                attributes=filename.split("_")
                if(attributes[-1]==resolution):
                    all_files.append(os.path.join(root,fn))
    total=len(all_files)
    nbtests=int(total*test_split)
    nbtrain=total-nbtests
    train_data=all_files[:nbtrain]
    test_data=all_files[nbtrain:]
    print("Directory scan complete: %d samples Train %d Test %d" % (total,len(train_data),len(test_data)))
    return train_data,test_data

def load_image(imfile):
    print("Loading %s"%imfile)
    img=Image.open(imfile).convert('L')
    w_c,h_r=img.size
    pixels=img.getdata()
    imagemat=np.reshape(pixels,[h_r,w_c])
    return imagemat

def load_x_y_cmu(files,class_labels):
    x_ = []
    y_ = []
    for fl in files:
        imgmat = load_image(fl)
        x_.append(imgmat)
        info = fl.split(".")[0].split("_")[-2]
        one_hot_label = [0, 0]
        one_hot_label[class_labels.index(info)] = 1.0
        y_.append(one_hot_label)
        print("\t Label %s" % info)
    x_ = np.asarray(x_)
    y_ = np.asarray(y_)
    return x_, y_

def load_cmu_face_images(face_path,resolution,test_split,class_labels):
    train,test=split_cmu_face_images(face_path,resolution,test_split)
    x_train,y_train=load_x_y_cmu(train,class_labels)
    x_test,y_test=load_x_y_cmu(test,class_labels)
    print("Train: X ",x_train.shape," Y ",y_train.shape)
    print("Test: X ", x_test.shape, " Y ", y_test.shape)
    return x_train,y_train,x_test,y_test


import os
import sys

# PostGreSQL
import psycopg2

instrument_path_base = '/var/r1/'
www_path_base = '/var/www/html'

ac_config_dict = {
	'aircraft.tailnumber' : os.getenv("AIRCRAFT"),
	'project.name' : os.getenv("PROJECT"),
	'project.path' : os.getenv("PROJ_DIR")
	}

def connect_db():
  return psycopg2.connect(database="real-time", user="ads", host="acserver")

# For ground testing.
def connect_db_ground():
  return psycopg2.connect(database="real-time-C130", user="ads",
                          host="eol-rt-data.ml-ext.ucar.edu")

def get_config(key):

  # Open real-time onboard database
  db = connect_db()
  cur = db.cursor()

  # Get flight number from database.
  cur.execute("SELECT value FROM global_attributes WHERE key='FlightNumber';")
  rows = cur.fetchall()
  ac_config_dict['project.flightnumber'] = rows[0][0]

  # Assemble camera path.
  cpath = instrument_path_base + ac_config_dict['project.name']
  cpath += '/camera_images/flight_number_'
  cpath += ac_config_dict['project.flightnumber']

  ac_config_dict['cameras.path'] = cpath

  ac_config_dict['toga.path'] = instrument_path_base + \
                                ac_config_dict['project.name'] + '/toga'
  ac_config_dict['mtp.path'] = instrument_path_base + \
                               ac_config_dict['project.name'] + '/mtp'
  ac_config_dict['dropsonde.raw_path'] = instrument_path_base + '/dropsondes'
  ac_config_dict['dropsonde.skewt_path'] = www_path_base + '/skewt'

  db.close()

  return ac_config_dict.get(key, None)


from __future__ import print_function, division
from _cython_feature_detectors import *
import numpy as np
import pandas as pd
import copy
from scipy import stats
import scipy.optimize
import matplotlib.dates as mdates
import math, datetime
from slicedpy.normal import Normal
from slicedpy.powerstate import PowerState, PowerSegment
from slicedpy.datastore import DataStore
from slicedpy import utils
from pda.channel import _indicies_of_periods

"""
.. module:: feature_detectors
   :synopsis: Functions for detecting features in power data.

   This file implements feature detectors which are written in pure
   Python.  Cython feature detectors are in
   cython/_cython_feature_detectors.pyx.  This file also holds helper functions
   for pre-processing prior to using feature detectors.
"""

###############################################################################
# SPIKE HISTOGRAM FUNCTIONS
###############################################################################

def get_merged_spikes(fdiff):
    """Merge consecutive forward difference values of the same sign.

    Args:
        fdiff (1D np.ndarray): forward difference of power.
            e.g. calculated by np.diff

    Returns:
        merged_fdiff (1D np.ndarray).  Will be zero prior to each merged spike.
    """

    sign_comparison = (fdiff[:-1] * fdiff[1:]) > 0
    merged_fdiff = copy.copy(fdiff)
    accumulator = 0
    for i in range(0,merged_fdiff.size-1):
        if sign_comparison[i] == True:
            if accumulator == 0:
                accumulator = fdiff[i] + fdiff[i+1]
            else:
                accumulator += fdiff[i+1]
            merged_fdiff[i] = 0
        else:
            if accumulator != 0:
                merged_fdiff[i] = accumulator
                accumulator = 0

    # Handle last element if necessary
    if accumulator != 0:
        merged_fdiff[-1] = accumulator

    return merged_fdiff

def get_merged_spikes_pandas(series):
    return pd.Series(get_merged_spikes(series.values), index=series.index)

def spike_histogram(series, merge_spikes=True, window_duration=60, n_bins=8):
    """
    Args:
        * series (pd.Series): watts
        * merge_spikes (bool): Default = True
        * window_duration (float): Width of each window in seconds
        * n_bins (int): number of bins per window.

    Returns:
        spike_hist, bin_edges:
            spike_hist (pd.DataFrame):
                index is pd.DateTimeIndex of start of each time window
                columns are 2-tuples of the bin edges in watts (int)
            bin_edges (list of ints):
    """
    fdiff = series.diff()
    if merge_spikes:
        fdiff = get_merged_spikes_pandas(fdiff)

    abs_fdiff = np.fabs(fdiff)
    freq = (window_duration, 'S')
    date_range, boundaries = _indicies_of_periods(fdiff.index,
                                                  freq=freq)
    bin_edges = np.concatenate(([0], np.exp(np.arange(1,n_bins+1))))
    bin_edges = np.round(bin_edges).astype(int)
    cols = zip(bin_edges[:-1], bin_edges[1:])
    spike_hist = pd.DataFrame(index=date_range, columns=cols)

    for date_i, date in enumerate(date_range):
        start_i, end_i = boundaries[date_i]
        chunk = abs_fdiff[start_i:end_i]
        spike_hist.loc[date] = np.histogram(chunk, bins=bin_edges)[0]

    return spike_hist, bin_edges

def spike_histogram_bin_to_data_coordinates(bin_data, scale_x=1,
                                            binary_items=False):
    """make a 2d matrix suitable for clustering where each row stores the
    coordinates of a single data point.  Each x coordinate is the
    ordinal timestamp (in ordinal time where 1 == 1st Jan 0001; 2 =
    2nd Jan 0001 etc), each y coordinate is the count.  Elements with
    count == 0 are ignored.

    Args:
      * bin_data (pd.Series): one bin from the spike histogram.
      * scale_x (float): multiply ordinal time by this value.
      * binary_items (bool): just put a 1 in the second column.

    Returns:
      * X (np.ndarray, dim=2, np.float64)

    """
    nonzero_i = np.nonzero(bin_data)[0]
    X = np.empty((nonzero_i.size, 2), dtype=np.float64)
    X[:,0] = mdates.date2num(bin_data[nonzero_i].index) * scale_x
    X[:,1] = 1 if binary_items else bin_data[nonzero_i].values
    return X

###############################################################################
# multiple_linear_regressions
###############################################################################

def multiple_linear_regressions(series, window_size=10):
    """
    Args:
        series (pd.Series): power.
        window_size (int): Width of each windows in number of samples.
            Must be multiple of 2.  Windows are overlapping:

    ::

                2222
              11113333
    Return:
        np.ndarray(n_windows, 4): the 4 columns are:
          1. slope
          2. intercept
          3. :math:`\\text{r_value} ^2`
          4. std_err
    """
    assert((window_size % 2) == 0)
    data = series.values
    half_window_size = int(window_size / 2)
    n_windows = int(data.size / half_window_size) - 1
    x = np.arange(window_size)
    results = np.empty((n_windows, 4))
    start_i = 0
    end_i = window_size
    for i in range(n_windows):
        window = data[start_i:end_i]
        start_i += half_window_size
        end_i += half_window_size
        slope, intercept, r_value, p_value, std_err = stats.linregress(x, window)
        results[i] = (slope, intercept, r_value**2, std_err)

    idx_i = range(0, half_window_size*n_windows, half_window_size)
    idx = series.index[idx_i]
    return pd.DataFrame(results, index=idx,
                        columns=['slope','intercept','r_squared','std_err'])

###############################################################################
# DECAYS
###############################################################################

def ttest_both_halves(watts, start, end):
    """
    Test if the left and right half of the watts masked `start` and `end`
    to see if they have the same mean or not.

    Returns two-tailed p-value.

    Args:
      * watts (:class:`pandas.Series`)
      * start (:class:`pandas.Timestamp`)
      * end (:class:`pandas.Timestamp`)
    """
    width = end - start
    half_way = start + width.__div__(2)
    left = watts[(watts.index >= start) & (watts.index < half_way)]
    right = watts[(watts.index >= half_way) & (watts.index < end)]
    return stats.ttest_ind(left, right)[1]

def linregress(watts, start, end):
    """
    Linear regression of data masked `start` and `end`

    Args
      * watts (:class:`pandas.Series`)
      * start (:class:`pandas.Timestamp`)
      * end (:class:`pandas.Timestamp`)

    Returns:
      * ``slope`` (in units of watts per second)
      * ``r_value``
      * ``p_value``
      * ``stderr``
    """
    ss = watts[(watts.index >= start) & (watts.index < end)]
    x = mdates.date2num(ss.index) * mdates.SEC_PER_DAY
    (slope, _, r_value,
     p_value, stderr) = stats.linregress(x, ss.values)
    return slope, r_value, p_value, stderr

###############################################################################
# SPIKE THEN DECAY
###############################################################################

def spike_indices(data, min_spike_size, max_spike_size=None):
    """Find spikes between min_spike_size and max_spike_size."""
    fdiff = np.diff(data)
    fdiff = get_merged_spikes(fdiff)
    if max_spike_size is None:
        spike_indices = np.where(fdiff > min_spike_size)[0]
    else:
        assert(max_spike_size > min_spike_size)
        spike_indices = np.where((fdiff > min_spike_size) &
                                 (fdiff < max_spike_size))[0]
    spike_indices += 1
    return spike_indices

def spike_then_decay(series, min_spike_size=300, max_spike_size=None,
                     decay_window=10, mode='poly'):
    """
    Args:
      * series (pd.Series): watts
      * min_spike_size (float): watts
      * max_spike_size (float or None): optional.  watts.
      * decay_window (float): seconds
      * mode (str): 'linear' or 'poly'

    Returns:
      * pd.DataFrame with 'end' and 'decay' columns set.
    """
    def linear(x, values):
        (slope, intercept, r_value,
         p_value, stderr) = stats.linregress(x, values)
        return intercept, slope

    curve = lambda x, c, m: c + (m / x)
    def poly(x, values):
        popt, pconv = scipy.optimize.curve_fit(curve, (x-x[0])+1, values,
                                               p0=(values.min(), 281.5))
        return popt[0], popt[1] # intercept, slope

    if mode=='linear':
        regression_func = linear
    elif mode=='poly':
        regression_func = poly
    else:
        raise Exception('Mode \'' + mode + '\' not recognised')

    spike_idxs = spike_indices(series.values[:-decay_window],
                               min_spike_size, max_spike_size)

    timedelta = datetime.timedelta(seconds=decay_window)

    # For each spike, do regression of next decay_window values
    features = []
    for spike_i in spike_idxs:
        start = series.index[spike_i]
        end = start + timedelta
        chunk = series[spike_i:]
        chunk = chunk[chunk.index < end]
        x = mdates.date2num(chunk.index) * mdates.SEC_PER_DAY
        intercept, slope = regression_func(x, chunk.values)
        f = {'end':end, 'slope':slope, 'intercept':intercept}
        features.append(f)

    return pd.DataFrame(features, index=series.index[spike_idxs])

#############################################################################
# POWER SEGMENT DETECTORS
#############################################################################

def relative_deviation_power_sgmnts(
                  series,
                  rdt=0.05, # relative deviation threshold
                  window_size=10):
    """power segment detector designed to find "power segments" in the face of
    a rapidly oscillating signal (e.g. a washing machine's motor).

    Break ``series`` into chunks, each of size ``window_size``.
    Calculate the mean of the first chunk. Calculate the mean
    deviation of the second chunk against the mean of the first chunk.
    If this deviation is above ``rdt`` then we've left a candidate power
    segment.  If we've been through 2 or more chunks then store this
    power segment.  Repeat.

    """

    # TODO: Optimise:
    # * convert code to Cython
    # * don't calculate ps.mean() from scratch every iteration

    watts = series.values

    def mean_relative_deviation(next_chunk, ps_mean):
        """Convert to absolute value _after_ calculating the mean.
        The idea is that rapid oscillations should cancel themselves out."""
        return math.fabs((next_chunk - ps_mean).mean() / ps_mean)

    n_chunks = int(watts.size / window_size)
    print("n_chunks =", n_chunks)
    ps_start_i = 0
    ps_end_i = window_size
    idx = [] # index for dataframe
    power_sgmnts = []
    for chunk_i in range(n_chunks-2):
        ps = watts[ps_start_i:ps_end_i]
        next_chunk_end_i = (chunk_i+2)*window_size
        next_chunk = watts[ps_end_i:next_chunk_end_i]
        if mean_relative_deviation(next_chunk, ps.mean()) > rdt:
            # new chunk marks the end of the power segment
            if (ps_end_i - ps_start_i) / window_size > 1:
                idx.append(series.index[ps_start_i])
                power_sgmnts.append({'end': series.index[ps_end_i-1],
                                     'power': Normal(ps)})
            ps_start_i = ps_end_i
        ps_end_i = next_chunk_end_i

    return pd.DataFrame(power_sgmnts, index=idx)

def min_max_power_sgmnts(data, max_deviation=20, initial_window_size=30,
                         look_ahead=3, max_ptp=1000):
    """power segment detector which looks for periods with similar min
    and max values.

    Find "power segments" by first taking an initial window of size
    ``initial_window_size``.  Check that this window is "sane" by
    splitting it in half and comparing the min and max of the two
    halves.  If this window is sane then take the average of
    ``look_ahead`` samples ahead of the end of the window and see if
    this average is within ``max_deviation`` of the min and max of the
    window.  If it is then add ``look_ahead`` onto the window and
    repeat.  Also take the tail of the window and check that the mean
    of the front of the window falls within ``max_deviation`` of the
    min and max of the tail (this helps to make sure we end the power
    segment early enough in a situation where we go from a section with a
    large min and max to a section with a similar mean but smaller min
    and max).

    Args:
      * data (pd.Series or np.ndarray): watts
      * max_deviation (float): watts
      * initial_window_size (int): number of samples
      * look_ahead (int): number of samples
      * max_ptp (float): max peak to peak in watts

    Returns:
      * a pd.DataFrame with columns:
        * ``end``
        * ``power``: a DataStore(model=Normal())
    """
    if isinstance(data, pd.Series):
        watts = data.values
        index = data.index
    elif isinstance(data, np.ndarray):
        watts = data
        index = np.arange(watts.shape[0])
    else:
        raise Exception('data is of wrong type!')

    ps_start_i = 0
    ps_end_i = initial_window_size
    idx = [] # index for dataframe
    power_sgmnts = []
    half_window = int(initial_window_size / 2)
    n = watts.size - look_ahead

    def append_power_segment(ps_start_i, ps_end_i, ps):
        idx.append(index[ps_start_i])
        ds = DataStore(model=Normal())
        ds.append(ps)
        power_sgmnts.append({'end': index[ps_end_i-1],
                             'power': ds})

    while True:
        if ps_end_i >= n:
            break

        ps = watts[ps_start_i:ps_end_i]
        end_of_ps = False
        if ps.size == initial_window_size:
            # this is an initial chunk so test it's a sane
            # chunk by comparing the means of the left and right side
            # of this chunk
            halfway = ps_start_i + half_window
            left = watts[ps_start_i:halfway]
            right = watts[halfway:ps_end_i]
            if (math.fabs(left.min() - right.min()) > max_deviation or
                math.fabs(left.max() - right.max()) > max_deviation or
                ps.ptp() > max_ptp):
                ps_start_i += 1
                ps_end_i += 1
                continue
        else:
            # Take an ``initial_window_size`` chunk from the tail
            # and do some checks.
            tail_split = ps_end_i-initial_window_size
            front = watts[ps_start_i:tail_split]
            tail = watts[tail_split:ps_end_i]
            if (front.mean() < tail.min() - max_deviation or
                front.mean() > tail.max() + max_deviation):
#            if (abs(tail.std() - front.std()) > tail.std()*30):
                end_of_ps = True
                ps_end_i = tail_split
                ps = front

        ahead = watts[ps_end_i:ps_end_i+look_ahead]
        if (ahead.mean() < ps.min() - max_deviation or
            ahead.mean() > ps.max() + max_deviation or
            end_of_ps):
            # We've come to the end of a candidate power segment
            append_power_segment(ps_start_i, ps_end_i, ps)
            ps_start_i = ps_end_i
            ps_end_i = ps_start_i + initial_window_size
        else:
            ps_end_i += look_ahead

    # add last power segment, if necessary
    if ps_end_i > ps_start_i + initial_window_size:
        append_power_segment(ps_start_i, ps_end_i, ps)

    return pd.DataFrame(power_sgmnts, index=idx)

def min_max_two_halves_power_sgmnts(series,
                                    max_deviation=20, # watts
                                    initial_window_size=20, # int: n samples
                                    max_ptp=1000 # max peak to peak in watts
                                    ):
    watts = series.values
    ps_start_i = 0
    ps_end_i = initial_window_size
    idx = [] # index for dataframe
    power_sgmnts = []
    while True:
        if ps_end_i >= watts.size:
            break

        ps = watts[ps_start_i:ps_end_i]

        # test it's a sane
        # chunk by comparing the means of the left and right side
        halfway = int((ps_start_i + ps_end_i) / 2)
        left = watts[ps_start_i:halfway]
        right = watts[halfway:ps_end_i]

        if (math.fabs(left.min() - right.min()) > max_deviation or
            math.fabs(left.max() - right.max()) > max_deviation or
            ps.ptp() > max_ptp):
            if ps_end_i == ps_start_i + initial_window_size:
                ps_start_i += 1
                ps_end_i += 1
            else:
                # We've come to the end of a candidate power segment
                idx.append(series.index[ps_start_i])
                power_sgmnts.append({'end': series.index[ps_end_i-1],
                                     'power': Normal(ps)})
                ps_start_i = ps_end_i
                ps_end_i = ps_start_i + initial_window_size
        else:
            ps_end_i += 1

    return pd.DataFrame(power_sgmnts, index=idx)

def mean_chunk_power_sgmnts(series, max_deviation=10, window_size=10):
    watts = series.values
    ps_start_i = 0
    idx = [] # index for dataframe
    power_sgmnts = []
    n_chunks = int(watts.size / window_size)
    for chunk_i in range(1,n_chunks-2):
        ps_end_i = window_size * chunk_i
        ps = watts[ps_start_i:ps_end_i]
        next_chunk = watts[ps_end_i:ps_end_i+window_size]
        if (next_chunk.mean() > ps.mean() + max_deviation or
            next_chunk.mean() < ps.mean() - max_deviation):
            # We've come to the end of a candidate power segment
            idx.append(series.index[ps_start_i])
            power_sgmnts.append({'end': series.index[ps_end_i-1],
                                 'power': Normal(ps)})
            ps_start_i = ps_end_i

    return pd.DataFrame(power_sgmnts, index=idx)

def minimise_mean_deviation_power_sgmnts(series,
                                         max_deviation=20, # watts
                                         initial_window_size=30, # int: number of samples
                                         look_ahead=150, # int: number of samples
                                         max_ptp=1000 # max peak-to-peak watts
                                         ):
    watts = series.values
    ps_start_i = 0
    ps_end_i = initial_window_size
    idx = [] # index for dataframe
    power_sgmnts = []
    half_window = int(initial_window_size / 2)
    n = watts.size - look_ahead
    while True:
        if ps_end_i >= n:
            break

        # this is an initial chunk so test it's a sane
        # chunk by comparing the means of the left and right side
        # of this chunk
        ps = watts[ps_start_i:ps_end_i]
        if ps_end_i == ps_start_i + initial_window_size:
            halfway = ps_start_i + half_window
            left = watts[ps_start_i:halfway]
            right = watts[halfway:ps_end_i]
            if (math.fabs(left.min() - right.min()) > max_deviation or
                math.fabs(left.max() - right.max()) > max_deviation or
                ps.ptp() > max_ptp):
                ps_start_i += 1
                ps_end_i += 1
                continue

        # Now creep forwards from ps_end_i and find the index which
        # gives the lowest deviation of the mean of the power segment
        # against the mean of the look ahead chunk.
        min_deviation = np.finfo(np.float64).max
        i_of_lowest = None
        ps_mean = ps.mean()
        for look_ahead_i in range(ps_end_i+1, ps_end_i+look_ahead):
            ahead = watts[ps_end_i:look_ahead_i]
            # TODO: don't recalculate ahead.mean() from scratch every iteration
            deviation = math.fabs(ahead.mean() - ps_mean)
            if deviation <= min_deviation:
                min_deviation = deviation
                i_of_lowest = look_ahead_i

        # Now we've found the index of the minimum deviation
        if min_deviation <= max_deviation:
            ps_end_i = i_of_lowest
        else:
            # End of power segment
            idx.append(series.index[ps_start_i])
            power_sgmnts.append({'end': series.index[ps_end_i-1],
                                 'power': Normal(ps)})
            ps_start_i = ps_end_i
            ps_end_i = ps_start_i + initial_window_size

    return pd.DataFrame(power_sgmnts, index=idx)

#############################################################################
# MERGE FEATURES
#############################################################################

def merge_features(pwr_sgmnts, decays, spike_histogram):
    """Associate features with each other to produce a list of
    "signature power states".

    Args:
      * pwr_sgmnts: pd.DataFrame with columns 'end' and 'power'
      * decays: pd.DataFrame
      * spike_histogram: pd.DataFrame

    Returns:
      List of PowerSegments.  See PowerSegment class for a list of attributes.
    """
    merged = []
    for start, pwr_seg in pwr_sgmnts.iterrows():
        pwr_state = PowerSegment(start=start,
                                 end=pwr_seg['end'],
                                 power=pwr_seg['power'])

        # DECAYS:
        # Assume decays to be within some constant number of
        # seconds around the start. Set start time of
        # powerstate to be start time of decay.
        max_time_diff = datetime.timedelta(seconds=20)
        i_of_nearest_decay = utils.find_nearest(decays, target=start,
                                                max_time_diff=max_time_diff)
        if i_of_nearest_decay is not None:
            pwr_state.slope = decays.iloc[i_of_nearest_decay]['slope']
            pwr_state.intercept = decays.iloc[i_of_nearest_decay]['intercept']
            pwr_state.start = decays.index[i_of_nearest_decay]

        # SPIKE HISTOGRAM:
        # Just take all.  If the power state is shorter in duration than
        # a single spike histogram window then this might be empty.
        cropped_spike_hist = spike_histogram[(spike_histogram.index >=
                                              pwr_state.start) &
                                             (spike_histogram.index <
                                              pwr_state.end)]
        pwr_state.spike_histogram = cropped_spike_hist.values

        merged.append(pwr_state)

    return merged


"""
This main script runs a simulation and saves the output to ./figures/

Example use case with animated output:
python main.py --num_iters 100 --plot 1

Example use case without animated output:
python main.py --num_iters 100 --plot 0
"""
import argparse
import os
import time

import matplotlib.pyplot as plt

import CAAF.model as model

parser = argparse.ArgumentParser()
parser.add_argument(
    "-n",
    "--num_iters",
    help="number of iterations to run the model",
    type=int,
    required=True,
)
parser.add_argument(
    "-p",
    "--plot",
    help="saves animation of simulation in a .mp4 file",
    type=int,
    default=0,
)
args = parser.parse_args()

num_iters = args.num_iters
plot = args.plot

parameters = {
    "row_size": 200,
    "col_size": 200,
    "refractory_period": 50,
    "driving_period": 220,
    "prob_not_fire": 0.05,
    "prob_con": 0.09,
    "prob_def": 0.05,
}

if __name__ == "__main__":

    heart = model.Heart(**parameters)
    os.makedirs('figures', exist_ok=True)

    start_time = time.time()
    heart.update(num_iters=num_iters, plot=plot)
    duration = time.time() - start_time
    print("Time: {0} s".format(duration))

    # Number of active cells with time
    fig, ax = plt.subplots(figsize=[5, 5])
    ax.plot(heart.num_active_cells)
    fig.savefig("./figures/num_active_cells.png")


# -*- coding: utf-8 -*-
# __auther__ : "johnny leaf"
# __date__: "2018.8.9"
# __project name__ : "dianping_spider"
# __CODE__: Python3

import time
import datetime
import sys

from Edp_excel import report
from Edp_excel import DataAn
from Spider_edp import dpspider

LoginUrl = 'https://e.dianping.com'

AcounDic = {
    '测试': [],
    # '八大处亚运村': ['bjykyl', 'ykyl180225', '90030589'],
    # '八大处平安门诊部': ['13883278696', 'bjpa888', '96640607'],
    # '煤炭总医院': ['mtzyyzx', 'meitan000', '98380431'],
    # '北京炫美': ['BJxuanmei123456', 'xmzx123456777', '92742255'],
    # '成都健丽': ['cdjianli', 'cdjs12345', '73082729'],
    # '成都美瑞': ['MeiruiTF', '93963llq', '8352512'],
    # '成都僮颜': ['tongyan88888', 'tongyan61', '98951640'],
    # '德尔美客': ['deermeike', '30826rpo', '97312957'],
    # '瑷珊': ['ieshan23', '37878jmd', '69046169'],
    # '和谐同方': ['HX73357653', '65787488', '4255672'],
    # '时光': ['tjsgzx520', 'tjsgzx2015', '59241747'],
    # '凤凰怡美': ['fenghuangyimei', 'fhymfh1234', '90290461']
}

def GetStartEnddate(label):
    timelist = []
    today = datetime.date.today()
    if label == '半月报':
        day = 1
        timelist.append(str(
            datetime.date(
                today.year - (
                    today.month == 1
                    ), today.month - day or day+11, 1)))
        timelist.append(str(today))
    elif label == '月报':
        starttime = datetime.date(today.year - (today.month == 1), today.month - 2 or 13, 1)
        endtime = datetime.date(today.year - (today.month == 1), today.month - 1 or 12, 31 or 30)
        timelist.append(str(starttime))
        timelist.append(str(endtime))
    else:
        timelist = label.split(' ')
    return timelist

def main(TimeInfo):
    for acount in AcounDic:
        DataAn.main(TimeInfo, acount)
        # res = dpspider.Get_CookeandSession(LoginUrl, AcounDic[acount])
        # print('%s 已取得Cookies参数' % acount)
        # if res:
        #     flowresult = dpspider.Getflowdata(res, AcounDic[acount])
        #     print('%s 已爬取流量数据' % acount)
        #     chatreuslt = dpspider.Getchatdata(res, AcounDic[acount], TimeInfo)
        #     print('%s 已爬取数咨询据' % acount)
        #     appointmentresult = dpspider.GetAppointresult(res, AcounDic[acount])
        #     print('%s 已爬取预约数据' % acount)
        #     SaleOnlineresult = dpspider.GetSaleOnlineresult(res)
        #     print('%s 已爬取线上销售数据' % acount)
        #     CommentResult = dpspider.GetCommentResult(res, AcounDic[acount])
        #     print('%s 已爬取体验报告' % acount)
        #     report.Report_main(flowresult, chatreuslt, appointmentresult, SaleOnlineresult, CommentResult, acount)
        #     print('====%s报表制作完成====' % acount)
        #     print('==========================')
        #     DataAn.main(TimeInfo, acount)
        # else:
        #     time.sleep(1)

def GetUserinput(inputinfo):
    if inputinfo == '半月报':
        timeresult = GetStartEnddate('半月报')
        print('==========%s模式==========' % inputinfo)
    elif inputinfo == '月报':
        timeresult = GetStartEnddate('月报')
        print('==========%s模式==========' % inputinfo)
    else:
        timeresult = GetStartEnddate(inputinfo)
        print('======%s至%s=====' % (timeresult[0], timeresult[1]))
    return timeresult

if __name__ == '__main__':
    userInput = sys.argv[1]
    TimeInfo = GetUserinput(userInput)
    main(TimeInfo)


# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import atexit
import collections
import itertools
import json
import math
import os
import re
import runpy
import sys
from collections import OrderedDict
from ctypes import c_int

from . import shared_telemetry_utils as utils
from .shared_telemetry_utils import ParserError

atexit.register(ParserError.exit_func)

# Constants.
MAX_LABEL_LENGTH = 20
MAX_LABEL_COUNT = 100
MAX_KEY_COUNT = 30
MAX_KEY_LENGTH = 20
MIN_CATEGORICAL_BUCKET_COUNT = 50
CPP_IDENTIFIER_PATTERN = "^[a-z][a-z0-9_]+[a-z0-9]$"

ALWAYS_ALLOWED_KEYS = [
    "kind",
    "description",
    "operating_systems",
    "expires_in_version",
    "alert_emails",
    "keyed",
    "releaseChannelCollection",
    "bug_numbers",
    "keys",
    "record_in_processes",
    "record_into_store",
    "products",
]

BASE_DOC_URL = (
    "https://firefox-source-docs.mozilla.org/toolkit/components/" "telemetry/telemetry/"
)
HISTOGRAMS_DOC_URL = BASE_DOC_URL + "collection/histograms.html"
SCALARS_DOC_URL = BASE_DOC_URL + "collection/scalars.html"

GECKOVIEW_STREAMING_SUPPORTED_KINDS = [
    "linear",
    "exponential",
    "categorical",
]

# parse_histograms.py is used by scripts from a mozilla-central build tree
# and also by outside consumers, such as the telemetry server.  We need
# to ensure that importing things works in both contexts.  Therefore,
# unconditionally importing things that are local to the build tree, such
# as buildconfig, is a no-no.
try:
    import buildconfig

    # Need to update sys.path to be able to find usecounters.
    sys.path.append(os.path.join(buildconfig.topsrcdir, "dom/base/"))
except ImportError:
    # Must be in an out-of-tree usage scenario.  Trust that whoever is
    # running this script knows we need the usecounters module and has
    # ensured it's in our sys.path.
    pass

def linear_buckets(dmin, dmax, n_buckets):
    ret_array = [0] * n_buckets
    dmin = float(dmin)
    dmax = float(dmax)
    for i in range(1, n_buckets):
        linear_range = (dmin * (n_buckets - 1 - i) + dmax * (i - 1)) / (n_buckets - 2)
        ret_array[i] = int(linear_range + 0.5)
    return ret_array

def exponential_buckets(dmin, dmax, n_buckets):
    log_max = math.log(dmax)
    bucket_index = 2
    ret_array = [0] * n_buckets
    current = dmin
    ret_array[1] = current
    for bucket_index in range(2, n_buckets):
        log_current = math.log(current)
        log_ratio = (log_max - log_current) / (n_buckets - bucket_index)
        log_next = log_current + log_ratio
        next_value = int(math.floor(math.exp(log_next) + 0.5))
        if next_value > current:
            current = next_value
        else:
            current = current + 1
        ret_array[bucket_index] = current
    return ret_array

allowlists = None

def load_allowlist():
    global allowlists
    try:
        parsers_path = os.path.realpath(os.path.dirname(__file__))
        # The parsers live in build_scripts/parsers in the Telemetry module, while
        # the histogram-allowlists file lives in the root of the module. Account
        # for that when looking for the allowlist.
        # NOTE: if the parsers are moved, this logic will need to be updated.
        telemetry_module_path = os.path.abspath(
            os.path.join(parsers_path, os.pardir, os.pardir)
        )
        allowlist_path = os.path.join(
            telemetry_module_path, "histogram-allowlists.json"
        )
        with open(allowlist_path, "r") as f:
            try:
                allowlists = json.load(f)
                for name, allowlist in allowlists.items():
                    allowlists[name] = set(allowlist)
            except ValueError:
                ParserError("Error parsing allowlist: %s" % allowlist_path).handle_now()
    except IOError:
        allowlists = None
        ParserError("Unable to parse allowlist: %s." % allowlist_path).handle_now()

class Histogram:
    """A class for representing a histogram definition."""

    def __init__(self, name, definition, strict_type_checks=False):
        """Initialize a histogram named name with the given definition.
        definition is a dict-like object that must contain at least the keys:

         - 'kind': The kind of histogram.  Must be one of 'boolean', 'flag',
           'count', 'enumerated', 'linear', or 'exponential'.
         - 'description': A textual description of the histogram.
         - 'strict_type_checks': A boolean indicating whether to use the new, stricter type checks.
                                 The server-side still has to deal with old, oddly typed
                                 submissions, so we have to skip them there by default.
        """
        self._strict_type_checks = strict_type_checks
        self._is_use_counter = name.startswith("USE_COUNTER2_")
        if self._is_use_counter:
            definition.setdefault("record_in_processes", ["main", "content"])
            definition.setdefault("releaseChannelCollection", "opt-out")
            definition.setdefault("products", ["firefox", "fennec"])
        self.verify_attributes(name, definition)
        self._name = name
        self._description = definition["description"]
        self._kind = definition["kind"]
        self._keys = definition.get("keys", [])
        self._keyed = definition.get("keyed", False)
        self._expiration = definition.get("expires_in_version")
        self._labels = definition.get("labels", [])
        self._record_in_processes = definition.get("record_in_processes")
        self._record_into_store = definition.get("record_into_store", ["main"])
        self._products = definition.get("products")
        self._operating_systems = definition.get("operating_systems", ["all"])

        self.compute_bucket_parameters(definition)
        self.set_nsITelemetry_kind()
        self.set_dataset(definition)

    def name(self):
        """Return the name of the histogram."""
        return self._name

    def description(self):
        """Return the description of the histogram."""
        return self._description

    def kind(self):
        """Return the kind of the histogram.
        Will be one of 'boolean', 'flag', 'count', 'enumerated', 'categorical', 'linear',
        or 'exponential'."""
        return self._kind

    def expiration(self):
        """Return the expiration version of the histogram."""
        return self._expiration

    def nsITelemetry_kind(self):
        """Return the nsITelemetry constant corresponding to the kind of
        the histogram."""
        return self._nsITelemetry_kind

    def low(self):
        """Return the lower bound of the histogram."""
        return self._low

    def high(self):
        """Return the high bound of the histogram."""
        return self._high

    def n_buckets(self):
        """Return the number of buckets in the histogram."""
        return self._n_buckets

    def keyed(self):
        """Returns True if this a keyed histogram, false otherwise."""
        return self._keyed

    def keys(self):
        """Returns a list of allowed keys for keyed histogram, [] for others."""
        return self._keys

    def dataset(self):
        """Returns the dataset this histogram belongs into."""
        return self._dataset

    def labels(self):
        """Returns a list of labels for a categorical histogram, [] for others."""
        return self._labels

    def record_in_processes(self):
        """Returns a list of processes this histogram is permitted to record in."""
        return self._record_in_processes

    def record_in_processes_enum(self):
        """Get the non-empty list of flags representing the processes to record data in"""
        return [utils.process_name_to_enum(p) for p in self.record_in_processes()]

    def products(self):
        """Get the non-empty list of products to record data on"""
        return self._products

    def products_enum(self):
        """Get the non-empty list of flags representing products to record data on"""
        return [utils.product_name_to_enum(p) for p in self.products()]

    def operating_systems(self):
        """Get the list of operating systems to record data on"""
        return self._operating_systems

    def record_on_os(self, target_os):
        """Check if this probe should be recorded on the passed os."""
        os = self.operating_systems()
        if "all" in os:
            return True

        canonical_os = utils.canonical_os(target_os)

        if "unix" in os and canonical_os in utils.UNIX_LIKE_OS:
            return True

        return canonical_os in os

    def record_into_store(self):
        """Get the non-empty list of stores to record into"""
        return self._record_into_store

    def ranges(self):
        """Return an array of lower bounds for each bucket in the histogram."""
        bucket_fns = {
            "boolean": linear_buckets,
            "flag": linear_buckets,
            "count": linear_buckets,
            "enumerated": linear_buckets,
            "categorical": linear_buckets,
            "linear": linear_buckets,
            "exponential": exponential_buckets,
        }

        if self._kind not in bucket_fns:
            ParserError(
                'Unknown kind "%s" for histogram "%s".' % (self._kind, self._name)
            ).handle_later()

        fn = bucket_fns[self._kind]
        return fn(self.low(), self.high(), self.n_buckets())

    def compute_bucket_parameters(self, definition):
        bucket_fns = {
            "boolean": Histogram.boolean_flag_bucket_parameters,
            "flag": Histogram.boolean_flag_bucket_parameters,
            "count": Histogram.boolean_flag_bucket_parameters,
            "enumerated": Histogram.enumerated_bucket_parameters,
            "categorical": Histogram.categorical_bucket_parameters,
            "linear": Histogram.linear_bucket_parameters,
            "exponential": Histogram.exponential_bucket_parameters,
        }

        if self._kind not in bucket_fns:
            ParserError(
                'Unknown kind "%s" for histogram "%s".' % (self._kind, self._name)
            ).handle_later()

        fn = bucket_fns[self._kind]
        self.set_bucket_parameters(*fn(definition))

    def verify_attributes(self, name, definition):
        general_keys = ALWAYS_ALLOWED_KEYS + ["low", "high", "n_buckets"]

        table = {
            "boolean": ALWAYS_ALLOWED_KEYS,
            "flag": ALWAYS_ALLOWED_KEYS,
            "count": ALWAYS_ALLOWED_KEYS,
            "enumerated": ALWAYS_ALLOWED_KEYS + ["n_values"],
            "categorical": ALWAYS_ALLOWED_KEYS + ["labels", "n_values"],
            "linear": general_keys,
            "exponential": general_keys,
        }
        # We removed extended_statistics_ok on the client, but the server-side,
        # where _strict_type_checks==False, has to deal with historical data.
        if not self._strict_type_checks:
            table["exponential"].append("extended_statistics_ok")

        kind = definition["kind"]
        if kind not in table:
            ParserError(
                'Unknown kind "%s" for histogram "%s".' % (kind, name)
            ).handle_later()
        allowed_keys = table[kind]

        self.check_name(name)
        self.check_keys(name, definition, allowed_keys)
        self.check_keys_field(name, definition)
        self.check_field_types(name, definition)
        self.check_allowlisted_kind(name, definition)
        self.check_allowlistable_fields(name, definition)
        self.check_expiration(name, definition)
        self.check_label_values(name, definition)
        self.check_record_in_processes(name, definition)
        self.check_products(name, definition)
        self.check_operating_systems(name, definition)
        self.check_record_into_store(name, definition)

    def check_name(self, name):
        if "#" in name:
            ParserError(
                'Error for histogram name "%s": "#" is not allowed.' % (name)
            ).handle_later()

        # Avoid C++ identifier conflicts between histogram enums and label enum names.
        if name.startswith("LABELS_"):
            ParserError(
                'Error for histogram name "%s":  can not start with "LABELS_".' % (name)
            ).handle_later()

        # To make it easier to generate C++ identifiers from this etc., we restrict
        # the histogram names to a strict pattern.
        # We skip this on the server to avoid failures with old Histogram.json revisions.
        if self._strict_type_checks:
            if not re.match(CPP_IDENTIFIER_PATTERN, name, re.IGNORECASE):
                ParserError(
                    'Error for histogram name "%s": name does not conform to "%s"'
                    % (name, CPP_IDENTIFIER_PATTERN)
                ).handle_later()

    def check_expiration(self, name, definition):
        field = "expires_in_version"
        expiration = definition.get(field)

        if not expiration:
            return

        # We forbid new probes from using "expires_in_version" : "default" field/value pair.
        # Old ones that use this are added to the allowlist.
        if (
            expiration == "default"
            and allowlists is not None
            and name not in allowlists["expiry_default"]
        ):
            ParserError(
                'New histogram "%s" cannot have "default" %s value.' % (name, field)
            ).handle_later()

        # Historical editions of Histograms.json can have the deprecated
        # expiration format 'N.Na1'. Fortunately, those scripts set
        # self._strict_type_checks to false.
        if (
            expiration != "default"
            and not utils.validate_expiration_version(expiration)
            and self._strict_type_checks
        ):
            ParserError(
                (
                    "Error for histogram {} - invalid {}: {}."
                    "\nSee: {}#expires-in-version"
                ).format(name, field, expiration, HISTOGRAMS_DOC_URL)
            ).handle_later()

        expiration = utils.add_expiration_postfix(expiration)

        definition[field] = expiration

    def check_label_values(self, name, definition):
        labels = definition.get("labels")
        if not labels:
            return

        invalid = filter(lambda l: len(l) > MAX_LABEL_LENGTH, labels)
        if len(list(invalid)) > 0:
            ParserError(
                'Label values for "%s" exceed length limit of %d: %s'
                % (name, MAX_LABEL_LENGTH, ", ".join(invalid))
            ).handle_later()

        if len(labels) > MAX_LABEL_COUNT:
            ParserError(
                'Label count for "%s" exceeds limit of %d' % (name, MAX_LABEL_COUNT)
            ).handle_now()

        # To make it easier to generate C++ identifiers from this etc., we restrict
        # the label values to a strict pattern.
        invalid = filter(
            lambda l: not re.match(CPP_IDENTIFIER_PATTERN, l, re.IGNORECASE), labels
        )
        if len(list(invalid)) > 0:
            ParserError(
                'Label values for %s are not matching pattern "%s": %s'
                % (name, CPP_IDENTIFIER_PATTERN, ", ".join(invalid))
            ).handle_later()

    def check_record_in_processes(self, name, definition):
        if not self._strict_type_checks:
            return

        field = "record_in_processes"
        rip = definition.get(field)

        DOC_URL = HISTOGRAMS_DOC_URL + "#record-in-processes"

        if not rip:
            ParserError(
                'Histogram "%s" must have a "%s" field:\n%s' % (name, field, DOC_URL)
            ).handle_later()

        for process in rip:
            if not utils.is_valid_process_name(process):
                ParserError(
                    'Histogram "%s" has unknown process "%s" in %s.\n%s'
                    % (name, process, field, DOC_URL)
                ).handle_later()

    def check_products(self, name, definition):
        if not self._strict_type_checks:
            return

        field = "products"
        products = definition.get(field)

        DOC_URL = HISTOGRAMS_DOC_URL + "#products"

        if not products:
            ParserError(
                'Histogram "%s" must have a "%s" field:\n%s' % (name, field, DOC_URL)
            ).handle_now()

        for product in products:
            if not utils.is_valid_product(product):
                ParserError(
                    'Histogram "%s" has unknown product "%s" in %s.\n%s'
                    % (name, product, field, DOC_URL)
                ).handle_later()
            if utils.is_geckoview_streaming_product(product):
                kind = definition.get("kind")
                if kind not in GECKOVIEW_STREAMING_SUPPORTED_KINDS:
                    ParserError(
                        (
                            'Histogram "%s" is of kind "%s" which is unsupported for '
                            'product "%s".'
                        )
                        % (name, kind, product)
                    ).handle_later()
                keyed = definition.get("keyed")
                if keyed:
                    ParserError(
                        'Keyed histograms like "%s" are unsupported for product "%s"'
                        % (name, product)
                    ).handle_later()

    def check_operating_systems(self, name, definition):
        if not self._strict_type_checks:
            return

        field = "operating_systems"
        operating_systems = definition.get(field)

        DOC_URL = HISTOGRAMS_DOC_URL + "#operating-systems"

        if not operating_systems:
            # operating_systems is optional
            return

        for operating_system in operating_systems:
            if not utils.is_valid_os(operating_system):
                ParserError(
                    'Histogram "%s" has unknown operating system "%s" in %s.\n%s'
                    % (name, operating_system, field, DOC_URL)
                ).handle_later()

    def check_record_into_store(self, name, definition):
        if not self._strict_type_checks:
            return

        field = "record_into_store"
        DOC_URL = HISTOGRAMS_DOC_URL + "#record-into-store"

        if field not in definition:
            # record_into_store is optional
            return

        record_into_store = definition.get(field)
        # record_into_store should not be empty
        if not record_into_store:
            ParserError(
                'Histogram "%s" has empty list of stores, which is not allowed.\n%s'
                % (name, DOC_URL)
            ).handle_later()

    def check_keys_field(self, name, definition):
        keys = definition.get("keys")
        if not self._strict_type_checks or keys is None:
            return

        if not definition.get("keyed", False):
            raise ValueError(
                "'keys' field is not valid for %s; only allowed for keyed histograms."
                % (name)
            )

        if len(keys) == 0:
            raise ValueError("The key list for %s cannot be empty" % (name))

        if len(keys) > MAX_KEY_COUNT:
            raise ValueError(
                "Label count for %s exceeds limit of %d" % (name, MAX_KEY_COUNT)
            )

        invalid = filter(lambda k: len(k) > MAX_KEY_LENGTH, keys)
        if len(list(invalid)) > 0:
            raise ValueError(
                '"keys" values for %s are exceeding length "%d": %s'
                % (name, MAX_KEY_LENGTH, ", ".join(invalid))
            )

    def check_allowlisted_kind(self, name, definition):
        # We don't need to run any of these checks on the server.
        if not self._strict_type_checks or allowlists is None:
            return

        # Disallow "flag" and "count" histograms on desktop, suggest to use
        # scalars instead. Allow using these histograms on Android, as we
        # don't support scalars there yet.
        hist_kind = definition.get("kind")
        android_target = "android" in definition.get("operating_systems", [])

        if (
            not android_target
            and hist_kind in ["flag", "count"]
            and name not in allowlists["kind"]
        ):
            ParserError(
                (
                    'Unsupported kind "%s" for histogram "%s":\n'
                    'New "%s" histograms are not supported on Desktop, you should'
                    " use scalars instead:\n"
                    "%s\n"
                    "Are you trying to add a histogram on Android?"
                    ' Add "operating_systems": ["android"] to your histogram definition.'
                )
                % (hist_kind, name, hist_kind, SCALARS_DOC_URL)
            ).handle_now()

    # Check for the presence of fields that old histograms are allowlisted for.
    def check_allowlistable_fields(self, name, definition):
        # Use counters don't have any mechanism to add the fields checked here,
        # so skip the check for them.
        # We also don't need to run any of these checks on the server.
        if self._is_use_counter or not self._strict_type_checks:
            return

        # In the pipeline we don't have allowlists available.
        if allowlists is None:
            return

        for field in ["alert_emails", "bug_numbers"]:
            if field not in definition and name not in allowlists[field]:
                ParserError(
                    'New histogram "%s" must have a "%s" field.' % (name, field)
                ).handle_later()
            if field in definition and name in allowlists[field]:
                msg = (
                    'Histogram "%s" should be removed from the allowlist for "%s" in '
                    "histogram-allowlists.json."
                )
                ParserError(msg % (name, field)).handle_later()

    def check_field_types(self, name, definition):
        # Define expected types for the histogram properties.
        type_checked_fields = {
            "n_buckets": int,
            "n_values": int,
            "low": int,
            "high": int,
            "keyed": bool,
            "expires_in_version": str,
            "kind": str,
            "description": str,
            "releaseChannelCollection": str,
        }

        # For list fields we check the items types.
        type_checked_list_fields = {
            "bug_numbers": int,
            "alert_emails": str,
            "labels": str,
            "record_in_processes": str,
            "keys": str,
            "products": str,
            "operating_systems": str,
            "record_into_store": str,
        }

        # For the server-side, where _strict_type_checks==False, we want to
        # skip the stricter type checks for these fields for dealing with
        # historical data.
        coerce_fields = ["low", "high", "n_values", "n_buckets"]
        if not self._strict_type_checks:
            # This handles some old non-numeric expressions.
            EXPRESSIONS = {
                "JS::GCReason::NUM_TELEMETRY_REASONS": 101,
                "mozilla::StartupTimeline::MAX_EVENT_ID": 12,
            }

            def try_to_coerce_to_number(v):
                if v in EXPRESSIONS:
                    return EXPRESSIONS[v]
                try:
                    return eval(v, {})
                except Exception:
                    return v

            for key in [k for k in coerce_fields if k in definition]:
                definition[key] = try_to_coerce_to_number(definition[key])
            # This handles old "keyed":"true" definitions (bug 1271986).
            if definition.get("keyed", None) == "true":
                definition["keyed"] = True

        def nice_type_name(t):
            if t is str:
                return "string"
            return t.__name__

        for key, key_type in type_checked_fields.items():
            if key not in definition:
                continue
            if not isinstance(definition[key], key_type):
                ParserError(
                    'Value for key "{0}" in histogram "{1}" should be {2}.'.format(
                        key, name, nice_type_name(key_type)
                    )
                ).handle_later()

        # Make sure the max range is lower than or equal to INT_MAX
        if "high" in definition and not c_int(definition["high"]).value > 0:
            ParserError(
                'Value for high in histogram "{0}" should be lower or equal to INT_MAX.'.format(
                    nice_type_name(c_int)
                )
            ).handle_later()

        for key, key_type in type_checked_list_fields.items():
            if key not in definition:
                continue
            if not all(isinstance(x, key_type) for x in definition[key]):
                ParserError(
                    'All values for list "{0}" in histogram "{1}" should be of type'
                    " {2}.".format(key, name, nice_type_name(key_type))
                ).handle_later()

    def check_keys(self, name, definition, allowed_keys):
        if not self._strict_type_checks:
            return
        for key in iter(definition.keys()):
            if key not in allowed_keys:
                ParserError(
                    'Key "%s" is not allowed for histogram "%s".' % (key, name)
                ).handle_later()

    def set_bucket_parameters(self, low, high, n_buckets):
        self._low = low
        self._high = high
        self._n_buckets = n_buckets
        max_n_buckets = 101 if self._kind in ["enumerated", "categorical"] else 100
        if (
            allowlists is not None
            and self._n_buckets > max_n_buckets
            and type(self._n_buckets) is int
        ):
            if self._name not in allowlists["n_buckets"]:
                ParserError(
                    'New histogram "%s" is not permitted to have more than 100 buckets.\n'
                    "Histograms with large numbers of buckets use disproportionately high"
                    " amounts of resources. Contact a Telemetry peer (e.g. in #telemetry)"
                    " if you think an exception ought to be made:\n"
                    "https://wiki.mozilla.org/Modules/Toolkit#Telemetry" % self._name
                ).handle_later()

    @staticmethod
    def boolean_flag_bucket_parameters(definition):
        return (1, 2, 3)

    @staticmethod
    def linear_bucket_parameters(definition):
        return (definition.get("low", 1), definition["high"], definition["n_buckets"])

    @staticmethod
    def enumerated_bucket_parameters(definition):
        n_values = definition["n_values"]
        return (1, n_values, n_values + 1)

    @staticmethod
    def categorical_bucket_parameters(definition):
        # Categorical histograms default to 50 buckets to make working with them easier.
        # Otherwise when adding labels later we run into problems with the pipeline not
        # supporting bucket changes.
        # This can be overridden using the n_values field.
        n_values = max(
            len(definition["labels"]),
            definition.get("n_values", 0),
            MIN_CATEGORICAL_BUCKET_COUNT,
        )
        return (1, n_values, n_values + 1)

    @staticmethod
    def exponential_bucket_parameters(definition):
        return (definition.get("low", 1), definition["high"], definition["n_buckets"])

    def set_nsITelemetry_kind(self):
        # Pick a Telemetry implementation type.
        types = {
            "boolean": "BOOLEAN",
            "flag": "FLAG",
            "count": "COUNT",
            "enumerated": "LINEAR",
            "categorical": "CATEGORICAL",
            "linear": "LINEAR",
            "exponential": "EXPONENTIAL",
        }

        if self._kind not in types:
            ParserError(
                'Unknown kind "%s" for histogram "%s".' % (self._kind, self._name)
            ).handle_later()

        self._nsITelemetry_kind = "nsITelemetry::HISTOGRAM_%s" % types[self._kind]

    def set_dataset(self, definition):
        datasets = {
            "opt-in": "DATASET_PRERELEASE_CHANNELS",
            "opt-out": "DATASET_ALL_CHANNELS",
        }

        value = definition.get("releaseChannelCollection", "opt-in")
        if value not in datasets:
            ParserError(
                "Unknown value for releaseChannelCollection"
                ' policy for histogram "%s".' % self._name
            ).handle_later()

        self._dataset = "nsITelemetry::" + datasets[value]

# This hook function loads the histograms into an OrderedDict.
# It will raise a ParserError if duplicate keys are found.
def load_histograms_into_dict(ordered_pairs, strict_type_checks):
    d = collections.OrderedDict()
    for key, value in ordered_pairs:
        if strict_type_checks and key in d:
            ParserError(
                "Found duplicate key in Histograms file: %s" % key
            ).handle_later()
        d[key] = value
    return d

# We support generating histograms from multiple different input files, not
# just Histograms.json.  For each file's basename, we have a specific
# routine to parse that file, and return a dictionary mapping histogram
# names to histogram parameters.
def from_json(filename, strict_type_checks):
    with open(filename, "r") as f:
        try:

            def hook(ps):
                return load_histograms_into_dict(ps, strict_type_checks)

            histograms = json.load(f, object_pairs_hook=hook)
        except ValueError as e:
            ParserError(
                "error parsing histograms in %s: %s" % (filename, e)
            ).handle_now()
    return histograms

def from_UseCounters_conf(filename, strict_type_checks):
    return usecounters.generate_histograms(filename)

def from_UseCountersWorker_conf(filename, strict_type_checks):
    return usecounters.generate_histograms(filename, True)

def from_nsDeprecatedOperationList(filename, strict_type_checks):
    operation_regex = re.compile("^DEPRECATED_OPERATION\\(([^)]+)\\)")
    histograms = collections.OrderedDict()

    with open(filename, "r") as f:
        for line in f:
            match = operation_regex.search(line)
            if not match:
                continue

            op = match.group(1)

            def add_counter(context):
                name = "USE_COUNTER2_DEPRECATED_%s_%s" % (op, context.upper())
                histograms[name] = {
                    "expires_in_version": "never",
                    "kind": "boolean",
                    "description": "Whether a %s used %s" % (context, op),
                }

            add_counter("document")
            add_counter("page")

    return histograms

def to_camel_case(property_name):
    return re.sub(
        "(^|_|-)([a-z0-9])",
        lambda m: m.group(2).upper(),
        property_name.strip("_").strip("-"),
    )

def add_css_property_counters(histograms, property_name):
    def add_counter(context):
        name = "USE_COUNTER2_CSS_PROPERTY_%s_%s" % (
            to_camel_case(property_name),
            context.upper(),
        )
        histograms[name] = {
            "expires_in_version": "never",
            "kind": "boolean",
            "description": "Whether a %s used the CSS property %s"
            % (context, property_name),
        }

    add_counter("document")
    add_counter("page")

def from_ServoCSSPropList(filename, strict_type_checks):
    histograms = collections.OrderedDict()
    properties = runpy.run_path(filename)["data"]
    for prop in properties:
        add_css_property_counters(histograms, prop.name)
    return histograms

def from_counted_unknown_properties(filename, strict_type_checks):
    histograms = collections.OrderedDict()
    properties = runpy.run_path(filename)["COUNTED_UNKNOWN_PROPERTIES"]

    # NOTE(emilio): Unlike ServoCSSProperties, `prop` here is just the property
    # name.
    #
    # We use the same naming as CSS properties so that we don't get
    # discontinuity when we implement or prototype them.
    for prop in properties:
        add_css_property_counters(histograms, prop)
    return histograms

# This is only used for probe-scraper.
def from_properties_db(filename, strict_type_checks):
    histograms = collections.OrderedDict()
    with open(filename, "r") as f:
        in_css_properties = False

        for line in f:
            if not in_css_properties:
                if line.startswith("exports.CSS_PROPERTIES = {"):
                    in_css_properties = True
                continue

            if line.startswith("};"):
                break

            if not line.startswith('  "'):
                continue

            name = line.split('"')[1]
            add_css_property_counters(histograms, name)
    return histograms

FILENAME_PARSERS = [
    (lambda x: from_json if x.endswith(".json") else None),
    (
        lambda x: from_nsDeprecatedOperationList
        if x == "nsDeprecatedOperationList.h"
        else None
    ),
    (lambda x: from_ServoCSSPropList if x == "ServoCSSPropList.py" else None),
    (
        lambda x: from_counted_unknown_properties
        if x == "counted_unknown_properties.py"
        else None
    ),
    (lambda x: from_properties_db if x == "properties-db.js" else None),
]

# Similarly to the dance above with buildconfig, usecounters may not be
# available, so handle that gracefully.
try:
    import usecounters

    FILENAME_PARSERS.append(
        lambda x: from_UseCounters_conf if x == "UseCounters.conf" else None
    )
    FILENAME_PARSERS.append(
        lambda x: from_UseCountersWorker_conf if x == "UseCountersWorker.conf" else None
    )
except ImportError:
    pass

def from_files(filenames, strict_type_checks=True):
    """Return an iterator that provides a sequence of Histograms for
    the histograms defined in filenames.
    """
    if strict_type_checks:
        load_allowlist()

    all_histograms = OrderedDict()
    for filename in filenames:
        parser = None
        for checkFn in FILENAME_PARSERS:
            parser = checkFn(os.path.basename(filename))
            if parser is not None:
                break

        if parser is None:
            ParserError("Don't know how to parse %s." % filename).handle_now()

        histograms = parser(filename, strict_type_checks)

        # OrderedDicts are important, because then the iteration order over
        # the parsed histograms is stable, which makes the insertion into
        # all_histograms stable, which makes ordering in generated files
        # stable, which makes builds more deterministic.
        if not isinstance(histograms, OrderedDict):
            ParserError("Histogram parser did not provide an OrderedDict.").handle_now()

        for (name, definition) in histograms.items():
            if name in all_histograms:
                ParserError('Duplicate histogram name "%s".' % name).handle_later()
            all_histograms[name] = definition

    def check_continuity(iterable, filter_function, name):
        indices = list(filter(filter_function, enumerate(iter(iterable.keys()))))
        if indices:
            lower_bound = indices[0][0]
            upper_bound = indices[-1][0]
            n_counters = upper_bound - lower_bound + 1
            if n_counters != len(indices):
                ParserError(
                    "Histograms %s must be defined in a contiguous block." % name
                ).handle_later()

    # We require that all USE_COUNTER2_*_WORKER histograms be defined in a contiguous
    # block.
    check_continuity(
        all_histograms,
        lambda x: x[1].startswith("USE_COUNTER2_") and x[1].endswith("_WORKER"),
        "use counter worker",
    )
    # And all other USE_COUNTER2_* histograms be defined in a contiguous
    # block.
    check_continuity(
        all_histograms,
        lambda x: x[1].startswith("USE_COUNTER2_") and not x[1].endswith("_WORKER"),
        "use counter",
    )

    # Check that histograms that were removed from Histograms.json etc.
    # are also removed from the allowlists.
    if allowlists is not None:
        all_allowlist_entries = itertools.chain.from_iterable(iter(allowlists.values()))
        orphaned = set(all_allowlist_entries) - set(all_histograms.keys())
        if len(orphaned) > 0:
            msg = (
                "The following entries are orphaned and should be removed from "
                "histogram-allowlists.json:\n%s"
            )
            ParserError(msg % (", ".join(sorted(orphaned)))).handle_later()

    for (name, definition) in all_histograms.items():
        yield Histogram(name, definition, strict_type_checks=strict_type_checks)


"""Functions to create loggers"""

import logging
from typing import Text
import sys

def get_console_handler() -> logging.StreamHandler:
    """Get console handler

    Returns:
        logging.StreamHandler which logs into stdout

    """
    console_handler = logging.StreamHandler(sys.stdout)
    formatter = logging.Formatter("%(asctime)s — %(name)s — %(levelname)s — %(message)s")
    console_handler.setFormatter(formatter)

    return console_handler

def get_logger(name: Text = __name__,
               log_level: Text or int = logging.DEBUG) -> logging.Logger:
    """Get logger

    Params:
        name {Text}: logger name
        log_level {Text or int}: logging level; can be string name or integer value

    Returns:
        logging.Logger instance

    """
    logger = logging.getLogger(name)
    logger.setLevel(log_level)

    # Prevent duplicate outputs in Jupyter Notebook
    if logger.hasHandlers():
        logger.handlers.clear()

    logger.addHandler(get_console_handler())
    logger.propagate = False

    return logger