#encoding=utf-8
import torch
import csv
import torchvision
import torchvision.transforms as transforms
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
import torch.optim as optim
import torch.utils.data as Data
import matplotlib.pyplot as plt
import numpy as np
class mydataset:
    def __init__(self, tens,lab):
        self.dataset =tens
        self.label=lab
    def __len__(self):
        return len(self.dataset)
    def __getitem__(self, idx):
        text = self.dataset[idx]
        label = self.label[idx]
        sample = {"text": text, "label": label}
        return sample
class Net(nn.Module):  # 定义网络，继承torch.nn.Module
    def __init__(self):
        super(Net, self).__init__()
        #self.pool = nn.AvgPool2d(2, 2)  # 池化层
        self.conv1 = torch.nn.Sequential(
            torch.nn.Conv2d(in_channels=1,
                            out_channels=4,
                            kernel_size=3,
                            stride=2,
                            padding=1),
            torch.nn.BatchNorm2d(4),
            torch.nn.ReLU()
        )
        self.conv2 = torch.nn.Sequential(
            torch.nn.Conv2d(4, 8, 3, 2, 1),
            torch.nn.BatchNorm2d(8),
            torch.nn.ReLU()
        )
        self.conv3 = torch.nn.Sequential(
            torch.nn.Conv2d(8, 16, 3, 2, 1),
            torch.nn.BatchNorm2d(16),
 	    torch.nn.ReLU()
        )
        self.fc1 = nn.Linear(480,250)  # 全连接层
        self.fc2 = nn.Linear(250,120)
        self.fc3 = nn.Linear(120,6)

    def forward(self, x):  # 前向传播

        #x = self.pool(F.relu(self.conv1(x)))  # F就是torch.nn.functional
        #x = self.pool(F.relu(self.conv2(x)))
        x=self.conv1(x)
        x=self.conv2(x)
        x = self.conv3(x)
        x = self.fc1(x.view(x.size(0), -1))
        x = self.fc2(x) # .view( )是一个tensor的方法，使得tensor改变size但是元素的总数是不变的。
        x = self.fc3(x)
 # 从卷基层到全连接层的维度转换
        return x

def trainandsave():
    # 神经网络结构
    net = Net()
      # 学习率为0.001
    optimizer = optim.SGD(net.parameters(), lr=0.01, momentum=0.9)
    criterion = nn.CrossEntropyLoss()  # 损失函数也可以自己定义，我们这里用的交叉熵损失函数
    # 训练部分
    for epoch in range(10):  # 训练的数据量为5个epoch，每个epoch为一个循环
        # 每个epoch要训练所有的图片，每训练完成200张便打印一下训练的效果（loss值）
        running_loss = 0.0  # 定义一个变量方便我们对loss进行输出
        for i, data in enumerate(trainloader):  # 这里我们遇到了第一步中出现的trailoader，代码传入数据
            # enumerate是python的内置函数，既获得索引也获得数据
            # get the inputs
            inputs= data["text"] # data是从enumerate返回的data，包含数据和标签信息，分别赋值给inputs和labels
    #        print(inputs.shape)
            labels=data["label"]
            # wrap them in Variable
            inputs, labels = Variable(inputs), Variable(labels)  # 转换数据格式用Variable
            optimizer.zero_grad()  # 梯度置零，因为反向传播过程中梯度会累加上一次循环的梯度
            # forward + backward + optimize
            inputs=inputs.float()
            labels=labels.float()
            outputs = net(inputs)  # 把数据输进CNN网络net
            loss = criterion(outputs, labels.long())  # 计算损失值
            loss.backward()  # loss反向传播
            optimizer.step()  # 反向传播后参数更新
            running_loss += loss.item()  # loss累加
            if i % 2000 == 0:
                print('[%d, %5d] loss: %.3f' %
                      (epoch + 1, i + 1, running_loss / 2000))  # 然后再除以200，就得到这两百次的平均损失值
                running_loss = 0.0  # 这一个200次结束后，就把running_loss归零，下一个200次继续使用
    print('Finished Training')
    # 保存神经网络
    return net
if __name__ =='__main__':
    ttens = []
    tlab=[]
    etens = []
    elab=[]
    i=0
    count2=0
    with open("/home/zc/lyd/train20-40b.csv") as f:
         row=csv.reader(f,delimiter = ',')
         next(row)
         for r in row:
             keyt=int(r[1])
             lab=int(r[2][:-1])
            # if((lab==1)and(count2<=20000)):
            #    count2+=1
            # elif((lab==1)and(count2>20000)):
            #    continue
            # if((lab==1) or (lab==2)):
            #     lab=2
            # if((lab==2)and(count2<=25000)):
            #     count2+=1
            # elif((lab==2)and(count2>25000)):
            #     continue
            # if(lab>4):
            #     continue
             if(keyt==9):
                if(i%5!=0):
                    i=i+1
                    tlab.append(lab)
                    temp=r[0][1:-1].split('|')
                    tempten=[]
                    for j in temp:
                        line=j.split('/')
                        tempten.append(list(map(int,line)))
                    ttens.append(torch.tensor([tempten]))
                else:
                    i=i+1
                    elab.append(lab)
                    temp = r[0][1:-1].split('|')
                    tempten = []
                    for j in temp:
                        line = j.split('/')
                        tempten.append(list(map(int, line)))
                    etens.append(torch.tensor([tempten]))
    td=mydataset(ttens,torch.tensor(tlab))
    test=mydataset(etens,torch.tensor(elab))
    BATCH_SIZE = 8
    trainloader = Data.DataLoader(
        dataset=td,
        batch_size=BATCH_SIZE,
        num_workers=0,
        drop_last=True
    )
    testloader = Data.DataLoader(
        dataset=test,
        batch_size=BATCH_SIZE,
        num_workers=0,
        drop_last=True
    )

    model = trainandsave()
    accuracy_sum = []
    for i,data in enumerate(testloader):
        test_x = Variable(data["text"]).float()
        test_y = Variable(data["label"]).float()
    #    print(test_x.shape)
        out = model(test_x)
        # print('test_out:\t',torch.max(out,1)[1])
        # print('test_y:\t',test_y)
        accuracy = torch.max(out,1)[1].numpy() == test_y.numpy()
        accuracy_sum.append(accuracy.mean())
       # print(torch.max(out,1)[1].numpy(),test_y.numpy())
    print('总准确率：\t',sum(accuracy_sum)/len(accuracy_sum))


#!/usr/bin/env python3

from os.path import exists
from subprocess import run
from shutil import copyfile
from modules import install, install_all, install_more

while True:
    yes_no=input("Do you want to install all packages[Y/N]: ")
    if yes_no not in "YyNn":
        print("Invalid response. Please enter again")
    else:
        if yes_no in "nN":
            waste_of_time=False
            break
        elif yes_no in "yY":
            waste_of_time=True
            break

while True:
    usb=input("Are you installing to an external disk?[Y/N]: ")
    if usb not in "YyNn":
        print("Invalid response. Please enter again")
    else:
        if usb in "nN":
            is_usb=False
            break
        elif usb in "yY":
            is_usb=True
            break

### Check for UEFI

if exists("/sys/firmware/efi"):
    UEFI=True
    if is_usb==True:
        grub_disk=input("Enter your grub device: ")

else:
    grub_disk=input("Enter your grub device: ")
    UEFI=False

### USERS and PASSES

copyfile("configs/pacman.conf", "/etc/pacman.conf")

user_name=input("Enter your username: ")
cmd="useradd -mG tty,video,audio,lp,input,audio,wheel {}".format(user_name)

run(cmd, shell=True)
print()

cmd="passwd "+user_name
run(cmd, shell=True)
print()

print("Enter password for root user: ")
run("passwd")
print()

### Locales

print("Configuring Locales ...")
print()

with open("/etc/locale.gen") as f:
    locale_gen=f.readlines()

for i in locale_gen:
    if i.strip()=="#en_US.UTF-8 UTF-8":
        i="en_US.UTF-8 UTF-8\n"

with open("/etc/locale.gen", "w") as f:
    for i in locale_gen:
        f.write(i)

run("locale-gen")

with open("/etc/locale.conf", "w") as f:
    f.write("en_US.UTF-8 UTF-8\n")

print("Locales are done")
print()

print("Configuring Timezones ...")
run("ln -sf /usr/share/zoneinfo/Asia/Kolkata /etc/localtime", shell=True)
run(["hwclock", "--systohc"])
print("Timezones have been configured")

print()

host_name=input("Enter hostname: ")

with open("/etc/hostname", "w") as f:
    f.write(host_name)

print()

print("Configuring Networking ...")

with open("/etc/hosts", "a") as f:
    f.write("127.0.0.1\t\tlocalhost")
    f.write("::1\t\tlocalhost")
    f.write("127.0.1.1\t\t{}".format(host_name))

print("Network has been configured")
print()

print("Configuring sudo ...")
copyfile("configs/sudoers", "/etc/sudoers")
install("sudo")
print("Done")
print()

print("Installing some packages")
packages=[
    "grub",
    "dosfstools",
    "mtools",
    "f2fs-tools",
    "btrfs-progs",
    "xfsprogs",
    "linux-headers",
    "xfsdump",
    "ntp",
    "networkmanager",
    "network-manager-applet",
    "xorg-server",
]
print()
install_more(packages)
print()

print("Installing Grub")

if is_usb:
    copyfile("configs/mkinitcpio.conf", "/etc/mkinicpio.conf")
    run(["mkinitcpio", "-P"])
    run("grub-install --target=x86_64-efi --efi-directory=/boot --bootloader-id=GRUB --removable --recheck", shell=True)
    run("grub-install --target=i386-pc {}".format(grub_disk), shell=True)
else:
    if UEFI:
        install("efibootmgr")
        run("grub-install --target=x86_64-efi --efi-directory=/boot --bootloader-id=GRUB --removable --recheck", shell=True)
    else:
        run("grub-install --target=i386-pc {}".format(grub_disk), shell=True)

run("grub-mkconfig -o /boot/grub/grub.cfg", shell=True)

print("Grub has been configured")
print()

if waste_of_time:
    install_all("txt/install.txt")
    run("usermod -aG vboxusers,libvirt,qemu,kvm "+user_name, shell=True)

print()

install("tlp")

print()

enable=[
    "tlp",
    "NetworkManager",
    "fstrim.timer",
    "ntpd",
]

for service in enable:
    cmd="systemctl enable {}".format(service)
    run(cmd, shell=True)

print()
print("Base system has been installed\nPlease run paru.sh as {} after rebooting".format(user_name))
print()
print("Terminating ...")


#!/usr/bin/env python3

import argparse
import subprocess
import os

class DriverOpenStack:
    @staticmethod
    def cleanup(args):
        subprocess.check_call(
            [
                "ansible-playbook",
                "openstack/playbooks/clean.yaml",
                "-e",
                "prefix=%s" % args.prefix,
            ]
        )

    @staticmethod
    def purge(args):
        subprocess.check_call(
            [
                "ansible-playbook",
                "openstack/playbooks/purge.yaml",
                "-e",
                "prefix=%s" % args.prefix,
            ]
        )

    @staticmethod
    def deploy(args):
        subprocess.check_call(
            [
                "ansible-playbook",
                "-vv",
                "openstack/playbooks/deploy.yaml",
                "-i",
                "openstack/inventory/openstack.yaml",
                "-e",
                "prefix=%s" % args.prefix,
            ]
        )
        subprocess.check_call(
            [
                "ansible-playbook",
                "playbooks/prepare_datastore.yaml",
                "-i",
                "openstack/inventory/openstack.yaml",
                "-vv",
            ]
        )
        subprocess.check_call(
            [
                "ansible-playbook",
                "playbooks/write_inventory.yaml",
                "-i",
                "openstack/inventory/openstack.yaml",
                "-vv",
            ]
        )
        if args.run_test:
            subprocess.check_call(
                [
                    "ansible-playbook",
                    "-vv",
                    "playbooks/run_test.yaml",
                    "-i",
                    "openstack/inventory/openstack.yaml",
                    "-e",
                    "prefix=%s" % args.prefix,
                ]
            )

class DriverLibvirt:
    @staticmethod
    def cleanup(args):
        for i in ["esxi1", "esxi2", "vcenter", "datastore"]:
            subprocess.check_call(["vl", "stop", i])

    @staticmethod
    def purge(args):
        subprocess.check_call(["vl", "down"])

    @staticmethod
    def deploy(args):
        # prepare virt lightning configuration
        cmd = []
        cmd.append("ansible-playbook")
        cmd.append("libvirt/playbooks/prepare_virt_lightning.yaml")
        cmd.extend(
            [
                '-e "{}_distro={}"'.format(k, vars(args)[k])
                for k in ("datastore", "esxi", "vcenter")
                if vars(args)[k] is not None
            ]
        )
        cmd.append("-vv")
        subprocess.check_call(cmd)
        # create virt lightning vm
        subprocess.check_call(
            ["vl", "up", "--virt-lightning-yaml", "libvirt/virt-lightning.yaml"]
        )
        # download zuul useful roles
        cmd = []
        cmd.append("ansible-playbook")
        cmd.append("playbooks/download_zuul_ci_roles.yaml")
        cmd.append("-vv")
        subprocess.check_call(cmd)
        # create ansible inventory
        subprocess.check_call("vl ansible_inventory>inventory", shell=True)
        # unset any ANSIBLE_ROLES_PATH variable to read the ansible.cfg file
        if "ANSIBLE_ROLES_PATH" in os.environ:
            del os.environ["ANSIBLE_ROLES_PATH"]
        # prepare datastore
        subprocess.check_call(
            [
                "ansible-playbook",
                "playbooks/prepare_datastore.yaml",
                "-i",
                "inventory",
                "-vv",
            ]
        )
        subprocess.check_call(
            [
                "ansible-playbook",
                "playbooks/write_inventory.yaml",
                "-i",
                "inventory",
                "-vv",
            ]
        )
        if args.run_test:
            subprocess.check_call(
                [
                    "ansible-playbook",
                    "-vv",
                    "playbooks/run_test.yaml",
                    "-i",
                    "inventory",
                    "-e",
                    "prefix=%s" % args.prefix,
                ]
            )

parser = argparse.ArgumentParser(description="Process some integers.")

actions = {
    "deploy": "Deploy the environment.",
    "cleanup": "Remove the ESXi VMs and the VCenter, preserver the datastore.",
    "purge": "Remove all the resources.",
}

help_actions = ", ".join(["ðŸ”¥{k}: {v}".format(k=k, v=v) for k, v in actions.items()])
parser.add_argument(
    "action", type=str, choices=actions.keys(), metavar=None, help=help_actions
)
parser.add_argument(
    "--driver",
    dest="driver",
    type=str,
    choices=["libvirt"],  # the OpenStack support is currently broken
    default="libvirt",
    help="Driver to use",
)
parser.add_argument("--prefix", type=str, default="")
parser.add_argument("--run-test", type=bool, default=False)
parser.add_argument("--datastore-distro", type=str, dest="datastore")
parser.add_argument("--esxi-distro", type=str, dest="esxi")
parser.add_argument("--vcenter-distro", type=str, dest="vcenter")

args = parser.parse_args()

if args.driver == "openstack":
    driver = DriverOpenStack
elif args.driver == "libvirt":
    driver = DriverLibvirt

getattr(driver, args.action)(args)


# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, # You can obtain one at http://mozilla.org/MPL/2.0/.

import functools
import glob
import logging
import os
import re
import shutil
import stat
import sys
import tarfile
import tempfile
from collections import defaultdict

import mozfile
import mozpack.path as mozpath
import requests

from mozbuild.base import MozbuildObject
from mozbuild.vendor.rewrite_mozbuild import (
    MozBuildRewriteException,
    add_file_to_moz_build_file,
    remove_file_from_moz_build_file,
)

DEFAULT_EXCLUDE_FILES = [".git*", ".git*/**"]
DEFAULT_KEEP_FILES = ["**/moz.build", "**/moz.yaml"]
DEFAULT_INCLUDE_FILES = []

def throwe():
    raise Exception

def _replace_in_file(file, pattern, replacement, regex=False):
    with open(file) as f:
        contents = f.read()

    if regex:
        newcontents = re.sub(pattern, replacement, contents)
    else:
        newcontents = contents.replace(pattern, replacement)

    if newcontents == contents:
        raise Exception(
            "Could not find '%s' in %s to %sreplace with '%s'"
            % (pattern, file, "regex-" if regex else "", replacement)
        )

    with open(file, "w") as f:
        f.write(newcontents)

def list_of_paths_to_readable_string(paths):
    # From https://stackoverflow.com/a/41578071
    dic = defaultdict(list)
    for item in paths:
        if os.path.isdir(item):  # To check path is a directory
            _ = dic[item]  # will set default value as empty list
        else:
            path, file = os.path.split(item)
            dic[path].append(file)

    final_string = "["
    for key, val in dic.items():
        if len(val) == 0:
            final_string += key + ", "
        elif len(val) < 3:
            final_string += ", ".join([os.path.join(key, v) for v in val]) + ", "
        elif len(val) < 10:
            final_string += "%s items in %s: %s and %s, " % (
                len(val),
                key,
                ", ".join(val[0:-1]),
                val[-1],
            )
        else:
            final_string += "%s (omitted) items in %s, " % (len(val), key)

    if final_string[-2:] == ", ":
        final_string = final_string[:-2]

    return final_string + "]"

class VendorManifest(MozbuildObject):
    def should_perform_step(self, step):
        return step not in self.manifest["vendoring"].get("skip-vendoring-steps", [])

    def vendor(
        self,
        command_context,
        yaml_file,
        manifest,
        revision,
        ignore_modified,
        check_for_update,
        force,
        add_to_exports,
        patch_mode,
    ):
        self.manifest = manifest
        self.yaml_file = yaml_file
        self._extract_directory = throwe
        self.logInfo = functools.partial(self.log, logging.INFO, "vendor")
        if "vendor-directory" not in self.manifest["vendoring"]:
            self.manifest["vendoring"]["vendor-directory"] = os.path.dirname(
                self.yaml_file
            )

        # ==========================================================
        # If we're only patching; do that
        if "patches" in self.manifest["vendoring"] and patch_mode == "only":
            self.import_local_patches(
                self.manifest["vendoring"]["patches"],
                os.path.dirname(self.yaml_file),
                self.manifest["vendoring"]["vendor-directory"],
            )
            return

        # ==========================================================
        self.source_host = self.get_source_host()

        ref_type = self.manifest["vendoring"].get("tracking", "commit")
        flavor = self.manifest["vendoring"].get("flavor", "regular")
        # Individiual files are special

        if revision == "tip":
            # This case allows us to force-update a tag-tracking library to master
            new_revision, timestamp = self.source_host.upstream_commit("HEAD")
        elif ref_type == "tag":
            new_revision, timestamp = self.source_host.upstream_tag(revision)
        else:
            new_revision, timestamp = self.source_host.upstream_commit(revision)

        self.logInfo(
            {"ref_type": ref_type, "ref": new_revision, "timestamp": timestamp},
            "Latest {ref_type} is {ref} from {timestamp}",
        )

        # ==========================================================
        if not force and self.manifest["origin"]["revision"] == new_revision:
            # We're up to date, don't do anything
            self.logInfo({}, "Latest upstream matches in-tree.")
            return
        elif flavor != "individual-file" and check_for_update:
            # Only print the new revision to stdout
            print("%s %s" % (new_revision, timestamp))
            return

        # ==========================================================
        if flavor == "regular":
            self.process_regular(
                new_revision, timestamp, ignore_modified, add_to_exports
            )
        elif flavor == "individual-files":
            self.process_individual(new_revision, timestamp, ignore_modified)
        elif flavor == "rust":
            self.process_rust(
                command_context,
                self.manifest["origin"]["revision"],
                new_revision,
                timestamp,
                ignore_modified,
            )
        else:
            raise Exception("Unknown flavor")

    def process_rust(
        self, command_context, old_revision, new_revision, timestamp, ignore_modified
    ):
        # First update the Cargo.toml
        cargo_file = os.path.join(os.path.dirname(self.yaml_file), "Cargo.toml")
        try:
            _replace_in_file(cargo_file, old_revision, new_revision)
        except Exception:
            # If we can't find it the first time, try again with a short hash
            _replace_in_file(cargo_file, old_revision[:8], new_revision)

        # Then call ./mach vendor rust
        from mozbuild.vendor.vendor_rust import VendorRust

        vendor_command = command_context._spawn(VendorRust)
        vendor_command.vendor(
            ignore_modified=True, build_peers_said_large_imports_were_ok=False
        )

        self.update_yaml(new_revision, timestamp)

    def process_individual(self, new_revision, timestamp, ignore_modified):
        # This design is used because there is no github API to query
        # for the last commit that modified a file; nor a way to get file
        # blame.  So really all we can do is just download and replace the
        # files and see if they changed...

        def download_and_write_file(url, destination):
            self.logInfo(
                {"local_file": destination, "url": url},
                "Downloading {local_file} from {url}...",
            )

            with mozfile.NamedTemporaryFile() as tmpfile:
                try:
                    req = requests.get(url, stream=True)
                    for data in req.iter_content(4096):
                        tmpfile.write(data)
                    tmpfile.seek(0)

                    shutil.copy2(tmpfile.name, destination)
                except Exception as e:
                    raise (e)

        # Only one of these loops will have content, so just do them both
        for f in self.manifest["vendoring"].get("individual-files", []):
            url = self.source_host.upstream_path_to_file(new_revision, f["upstream"])
            destination = self.get_full_path(f["destination"])
            download_and_write_file(url, destination)

        for f in self.manifest["vendoring"].get("individual-files-list", []):
            url = self.source_host.upstream_path_to_file(
                new_revision,
                self.manifest["vendoring"]["individual-files-default-upstream"] + f,
            )
            destination = self.get_full_path(
                self.manifest["vendoring"]["individual-files-default-destination"] + f
            )
            download_and_write_file(url, destination)

        self.spurious_check(new_revision, ignore_modified)

        self.logInfo({}, "Checking for update actions")
        self.update_files(new_revision)

        self.update_yaml(new_revision, timestamp)

        self.logInfo({"rev": new_revision}, "Updated to '{rev}'.")

        if "patches" in self.manifest["vendoring"]:
            # Remind the user
            self.log(
                logging.CRITICAL,
                "vendor",
                {},
                "Patches present in manifest!!! Please run "
                "'./mach vendor --patch-mode only' after commiting changes.",
            )

    def process_regular(self, new_revision, timestamp, ignore_modified, add_to_exports):

        if self.should_perform_step("fetch"):
            self.fetch_and_unpack(new_revision)
        else:
            self.logInfo({}, "Skipping fetching upstream source.")

        self.logInfo({}, "Checking for update actions")
        self.update_files(new_revision)

        if self.should_perform_step("hg-add"):
            self.logInfo({}, "Registering changes with version control.")
            self.repository.add_remove_files(
                self.manifest["vendoring"]["vendor-directory"],
                os.path.dirname(self.yaml_file),
            )
        else:
            self.logInfo({}, "Skipping registering changes.")

        if self.should_perform_step("spurious-check"):
            self.logInfo({}, "Checking for a spurious update.")
            self.spurious_check(new_revision, ignore_modified)
        else:
            self.logInfo({}, "Skipping the spurious update check.")

        if self.should_perform_step("update-moz-yaml"):
            self.logInfo({}, "Updating moz.yaml.")
            self.update_yaml(new_revision, timestamp)
        else:
            self.logInfo({}, "Skipping updating the moz.yaml file.")

        if self.should_perform_step("update-moz-build"):
            self.logInfo({}, "Updating moz.build files")
            self.update_moz_build(
                self.manifest["vendoring"]["vendor-directory"],
                os.path.dirname(self.yaml_file),
                add_to_exports,
            )
        else:
            self.logInfo({}, "Skipping update of moz.build files")

        self.logInfo({"rev": new_revision}, "Updated to '{rev}'.")

        if "patches" in self.manifest["vendoring"]:
            # Remind the user
            self.log(
                logging.CRITICAL,
                "vendor",
                {},
                "Patches present in manifest!!! Please run "
                "'./mach vendor --patch-mode only' after commiting changes.",
            )

    def get_source_host(self):
        if self.manifest["vendoring"]["source-hosting"] == "gitlab":
            from mozbuild.vendor.host_gitlab import GitLabHost

            return GitLabHost(self.manifest)
        elif self.manifest["vendoring"]["source-hosting"] == "github":
            from mozbuild.vendor.host_github import GitHubHost

            return GitHubHost(self.manifest)
        elif self.manifest["vendoring"]["source-hosting"] == "googlesource":
            from mozbuild.vendor.host_googlesource import GoogleSourceHost

            return GoogleSourceHost(self.manifest)
        elif self.manifest["vendoring"]["source-hosting"] == "angle":
            from mozbuild.vendor.host_angle import AngleHost

            return AngleHost(self.manifest)
        elif self.manifest["vendoring"]["source-hosting"] == "codeberg":
            from mozbuild.vendor.host_codeberg import CodebergHost

            return CodebergHost(self.manifest)
        else:
            raise Exception(
                "Unknown source host: " + self.manifest["vendoring"]["source-hosting"]
            )

    def get_full_path(self, path, support_cwd=False):
        if support_cwd and path[0:5] == "{cwd}":
            path = path.replace("{cwd}", ".")
        elif "{tmpextractdir}" in path:
            # _extract_directory() will throw an exception if it is invalid to use it
            path = path.replace("{tmpextractdir}", self._extract_directory())
        elif "{yaml_dir}" in path:
            path = path.replace("{yaml_dir}", os.path.dirname(self.yaml_file))
        elif "{vendor_dir}" in path:
            path = path.replace(
                "{vendor_dir}", self.manifest["vendoring"]["vendor-directory"]
            )
        else:
            path = mozpath.join(self.manifest["vendoring"]["vendor-directory"], path)
        return os.path.abspath(path)

    def convert_patterns_to_paths(self, directory, patterns):
        # glob.iglob uses shell-style wildcards for path name completion.
        # "recursive=True" enables the double asterisk "**" wildcard which matches
        # for nested directories as well as the directory we're searching in.
        paths = []
        for pattern in patterns:
            pattern_full_path = mozpath.join(directory, pattern)
            # If pattern is a directory recursively add contents of directory
            if os.path.isdir(pattern_full_path):
                # Append double asterisk to the end to make glob.iglob recursively match
                # contents of directory
                paths.extend(
                    glob.iglob(mozpath.join(pattern_full_path, "**"), recursive=True)
                )
            # Otherwise pattern is a file or wildcard expression so add it without altering it
            else:
                paths.extend(glob.iglob(pattern_full_path, recursive=True))
        # Remove folder names from list of paths in order to avoid prematurely
        # truncating directories elsewhere
        # Sort the final list to ensure we preserve 01_, 02_ ordering for e.g. *.patch globs
        final_paths = sorted(
            [mozpath.normsep(path) for path in paths if not os.path.isdir(path)]
        )
        return final_paths

    def fetch_and_unpack(self, revision):
        """Fetch and unpack upstream source"""

        def validate_tar_member(member, path):
            def is_within_directory(directory, target):
                real_directory = os.path.realpath(directory)
                real_target = os.path.realpath(target)
                prefix = os.path.commonprefix([real_directory, real_target])
                return prefix == real_directory

            member_path = os.path.join(path, member.name)
            if not is_within_directory(path, member_path):
                raise Exception("Attempted path traversal in tar file: " + member.name)
            if member.issym():
                link_path = os.path.join(os.path.dirname(member_path), member.linkname)
                if not is_within_directory(path, link_path):
                    raise Exception(
                        "Attempted link path traversal in tar file: " + member.name
                    )
            if member.mode & (stat.S_ISUID | stat.S_ISGID):
                raise Exception(
                    "Attempted setuid or setgid in tar file: " + member.name
                )

        def safe_extract(tar, path=".", *, numeric_owner=False):
            def _files(tar, path):
                for member in tar:
                    validate_tar_member(member, path)
                    yield member

            tar.extractall(path, members=_files(tar, path), numeric_owner=numeric_owner)

        url = self.source_host.upstream_snapshot(revision)
        self.logInfo({"url": url}, "Fetching code archive from {url}")

        with mozfile.NamedTemporaryFile() as tmptarfile:
            tmpextractdir = tempfile.TemporaryDirectory()
            try:
                req = requests.get(url, stream=True)
                for data in req.iter_content(4096):
                    tmptarfile.write(data)
                tmptarfile.seek(0)

                vendor_dir = mozpath.normsep(
                    self.manifest["vendoring"]["vendor-directory"]
                )
                if self.should_perform_step("keep"):
                    self.logInfo({}, "Retaining wanted in-tree files.")
                    to_keep = self.convert_patterns_to_paths(
                        vendor_dir,
                        self.manifest["vendoring"].get("keep", [])
                        + DEFAULT_KEEP_FILES
                        + self.manifest["vendoring"].get("patches", []),
                    )
                else:
                    self.logInfo({}, "Skipping retention of in-tree files.")
                    to_keep = []

                self.logInfo({"vd": vendor_dir}, "Cleaning {vd} to import changes.")
                # We use double asterisk wildcard here to get complete list of recursive contents
                for file in self.convert_patterns_to_paths(vendor_dir, ["**"]):
                    file = mozpath.normsep(file)
                    if file not in to_keep:
                        mozfile.remove(file)

                self.logInfo({"vd": vendor_dir}, "Unpacking upstream files for {vd}.")
                with tarfile.open(tmptarfile.name) as tar:

                    safe_extract(tar, tmpextractdir.name)

                    def get_first_dir(p):
                        halves = os.path.split(p)
                        return get_first_dir(halves[0]) if halves[0] else halves[1]

                    one_prefix = get_first_dir(tar.getnames()[0])
                    has_prefix = all(
                        map(lambda name: name.startswith(one_prefix), tar.getnames())
                    )

                # GitLab puts everything down a directory; move it up.
                if has_prefix:
                    tardir = mozpath.join(tmpextractdir.name, one_prefix)
                    mozfile.copy_contents(tardir, tmpextractdir.name)
                    mozfile.remove(tardir)

                if self.should_perform_step("include"):
                    self.logInfo({}, "Retaining wanted files from upstream changes.")
                    to_include = self.convert_patterns_to_paths(
                        tmpextractdir.name,
                        self.manifest["vendoring"].get("include", [])
                        + DEFAULT_INCLUDE_FILES,
                    )
                else:
                    self.logInfo({}, "Skipping retention of included files.")
                    to_include = []

                if self.should_perform_step("exclude"):
                    self.logInfo({}, "Removing excluded files from upstream changes.")
                    to_exclude = self.convert_patterns_to_paths(
                        tmpextractdir.name,
                        self.manifest["vendoring"].get("exclude", [])
                        + DEFAULT_EXCLUDE_FILES,
                    )
                else:
                    self.logInfo({}, "Skipping removing excluded files.")
                    to_exclude = []

                to_exclude = list(set(to_exclude) - set(to_include))
                if to_exclude:
                    self.logInfo(
                        {"files": list_of_paths_to_readable_string(to_exclude)},
                        "Removing: {files}",
                    )
                    for exclusion in to_exclude:
                        mozfile.remove(exclusion)

                # Clear out empty directories
                # removeEmpty() won't remove directories containing only empty directories
                #   so just keep callign it as long as it's doing something
                def removeEmpty(tmpextractdir):
                    removed = False
                    folders = list(os.walk(tmpextractdir))[1:]
                    for folder in folders:
                        if not folder[2]:
                            try:
                                os.rmdir(folder[0])
                                removed = True
                            except Exception:
                                pass
                    return removed

                while removeEmpty(tmpextractdir.name):
                    pass

                # Then copy over the directories
                if self.should_perform_step("move-contents"):
                    self.logInfo({"d": vendor_dir}, "Copying to {d}.")
                    mozfile.copy_contents(tmpextractdir.name, vendor_dir)
                else:
                    self.logInfo({}, "Skipping copying contents into tree.")
                    self._extract_directory = lambda: tmpextractdir.name
            except Exception as e:
                tmpextractdir.cleanup()
                raise e

    def update_yaml(self, revision, timestamp):
        with open(self.yaml_file) as f:
            yaml = f.readlines()

        replaced = 0
        replacements = [
            ["  release:", " %s (%s)." % (revision, timestamp)],
            ["  revision:", " %s" % (revision)],
        ]

        for i in range(0, len(yaml)):
            l = yaml[i]

            for r in replacements:
                if r[0] in l:
                    print("Found " + l)
                    replaced += 1
                    yaml[i] = re.sub(r[0] + " [v\.a-f0-9]+.*$", r[0] + r[1], yaml[i])

        assert len(replacements) == replaced

        with open(self.yaml_file, "wb") as f:
            f.write(("".join(yaml)).encode("utf-8"))

    def spurious_check(self, revision, ignore_modified):
        changed_files = set(
            [
                os.path.abspath(f)
                for f in self.repository.get_changed_files(mode="staged")
            ]
        )
        generated_files = set(
            [
                self.get_full_path(f)
                for f in self.manifest["vendoring"].get("generated", [])
            ]
        )
        changed_files = set(changed_files) - generated_files
        if not changed_files:
            self.logInfo({"r": revision}, "Upstream {r} hasn't modified files locally.")
            # We almost certainly won't be here if ignore_modified was passed, because a modified
            # local file will show up as a changed_file, but we'll be safe anyway.
            if not ignore_modified and generated_files:
                for g in generated_files:
                    self.repository.clean_directory(g)
            elif generated_files:
                self.log(
                    logging.CRITICAL,
                    "vendor",
                    {"files": generated_files},
                    "Because you passed --ignore-modified we are not cleaning your"
                    + " working directory, but the following files were probably"
                    + " spuriously edited and can be reverted: {files}",
                )
            sys.exit(-2)

        self.logInfo(
            {"rev": revision, "num": len(changed_files)},
            "Version '{rev}' has changed {num} files.",
        )

    def update_files(self, revision):
        if "update-actions" not in self.manifest["vendoring"]:
            return

        for update in self.manifest["vendoring"]["update-actions"]:
            if update["action"] == "copy-file":
                src = self.get_full_path(update["from"])
                dst = self.get_full_path(update["to"])

                self.logInfo(
                    {"s": src, "d": dst}, "action: copy-file src: {s} dst: {d}"
                )

                with open(src) as f:
                    contents = f.read()
                with open(dst, "w") as f:
                    f.write(contents)
            elif update["action"] == "move-file":
                src = self.get_full_path(update["from"])
                dst = self.get_full_path(update["to"])

                self.logInfo(
                    {"s": src, "d": dst}, "action: move-file src: {s} dst: {d}"
                )

                shutil.move(src, dst)
            elif update["action"] == "move-dir":
                src = self.get_full_path(update["from"])
                dst = self.get_full_path(update["to"])

                self.logInfo(
                    {"src": src, "dst": dst}, "action: move-dir src: {src} dst: {dst}"
                )

                if not os.path.isdir(src):
                    raise Exception(
                        "Cannot move from a source directory %s that is not a directory"
                        % src
                    )
                os.makedirs(dst, exist_ok=True)

                def copy_tree(src, dst):
                    names = os.listdir(src)
                    os.makedirs(dst, exist_ok=True)

                    for name in names:
                        srcname = os.path.join(src, name)
                        dstname = os.path.join(dst, name)

                        if os.path.isdir(srcname):
                            copy_tree(srcname, dstname)
                        else:
                            shutil.copy2(srcname, dstname)

                copy_tree(src, dst)
                shutil.rmtree(src)

            elif update["action"] in ["replace-in-file", "replace-in-file-regex"]:
                file = self.get_full_path(update["file"])

                self.logInfo({"file": file}, "action: replace-in-file file: {file}")

                replacement = update["with"].replace("{revision}", revision)
                _replace_in_file(
                    file,
                    update["pattern"],
                    replacement,
                    regex=update["action"] == "replace-in-file-regex",
                )
            elif update["action"] == "delete-path":
                path = self.get_full_path(update["path"])
                self.logInfo({"path": path}, "action: delete-path path: {path}")
                mozfile.remove(path)
            elif update["action"] in ["run-script", "run-command"]:
                if update["action"] == "run-script":
                    command = self.get_full_path(update["script"], support_cwd=True)
                else:
                    command = update["command"]

                run_dir = self.get_full_path(update["cwd"], support_cwd=True)

                args = []
                for a in update.get("args", []):
                    if a == "{revision}":
                        args.append(revision)
                    elif any(
                        s in a
                        for s in [
                            "{cwd}",
                            "{vendor_dir}",
                            "{yaml_dir}",
                            "{tmpextractdir}",
                        ]
                    ):
                        args.append(self.get_full_path(a, support_cwd=True))
                    else:
                        args.append(a)

                self.logInfo(
                    {
                        "command": command,
                        "run_dir": run_dir,
                        "args": args,
                        "type": update["action"],
                    },
                    "action: {type} command: {command} working dir: {run_dir} args: {args}",
                )
                extra_env = (
                    {"GECKO_PATH": os.getcwd()}
                    if "GECKO_PATH" not in os.environ
                    else {}
                )
                # We also add a signal to scripts that they are running under mach vendor
                extra_env["MACH_VENDOR"] = "1"
                self.run_process(
                    args=[command] + args,
                    cwd=run_dir,
                    log_name=command,
                    require_unix_environment=True,
                    append_env=extra_env,
                )
            else:
                assert False, "Unknown action supplied (how did this pass validation?)"

    def update_moz_build(self, vendoring_dir, moz_yaml_dir, add_to_exports):
        if vendoring_dir == moz_yaml_dir:
            vendoring_dir = moz_yaml_dir = None

        # If you edit this (especially for header files) you should double check
        # rewrite_mozbuild.py around 'assignment_type'
        source_suffixes = [".cc", ".c", ".cpp", ".S", ".asm"]
        header_suffixes = [".h", ".hpp"]

        files_removed = self.repository.get_changed_files(diff_filter="D")
        files_added = self.repository.get_changed_files(diff_filter="A")

        # Filter the files added to just source files we track in moz.build files.
        files_added = [
            f for f in files_added if any([f.endswith(s) for s in source_suffixes])
        ]
        header_files_to_add = [
            f for f in files_added if any([f.endswith(s) for s in header_suffixes])
        ]
        if add_to_exports:
            files_added += header_files_to_add
        elif header_files_to_add:
            self.log(
                logging.WARNING,
                "header_files_warning",
                {},
                (
                    "We found %s header files in the update, pass --add-to-exports if you want"
                    + " to attempt to include them in EXPORTS blocks: %s"
                )
                % (len(header_files_to_add), header_files_to_add),
            )

        self.logInfo(
            {"added": len(files_added), "removed": len(files_removed)},
            "Found {added} files added and {removed} files removed.",
        )

        should_abort = False
        for f in files_added:
            try:
                add_file_to_moz_build_file(f, moz_yaml_dir, vendoring_dir)
            except MozBuildRewriteException:
                self.log(
                    logging.ERROR,
                    "vendor",
                    {},
                    "Could not add %s to the appropriate moz.build file" % f,
                )
                should_abort = True

        for f in files_removed:
            try:
                remove_file_from_moz_build_file(f, moz_yaml_dir, vendoring_dir)
            except MozBuildRewriteException:
                self.log(
                    logging.ERROR,
                    "vendor",
                    {},
                    "Could not remove %s from the appropriate moz.build file" % f,
                )
                should_abort = True

        if should_abort:
            self.log(
                logging.ERROR,
                "vendor",
                {},
                "This is a deficiency in ./mach vendor . "
                + "Please review the affected files before committing.",
            )
            # Exit with -1 to distinguish this from the Exception case of exiting with 1
            sys.exit(-1)

    def import_local_patches(self, patches, yaml_dir, vendor_dir):
        self.logInfo({}, "Importing local patches...")
        for patch in self.convert_patterns_to_paths(yaml_dir, patches):
            script = [
                "patch",
                "-p1",
                "--directory",
                vendor_dir,
                "--input",
                os.path.abspath(patch),
                "--no-backup-if-mismatch",
            ]
            self.run_process(
                args=script,
                log_name=script,
            )


import setuptools
import os

with open(f"{os.path.dirname(os.path.realpath(__file__))}/README.md", "r", encoding="utf-8") as fh:
    long_description = fh.read()

setuptools.setup(
    name="QualtricsData",
    version="0.0.1",
    author="Ashish Mehta",
    author_email="ashm@stanford.edu",
    description="A package to read and preprocess Qualtrics Data",
    long_description=long_description,
    url="https://github.com/amehtaSF/QualtricsData",
    packages=setuptools.find_packages(),
    license = "License :: OSI Approved :: MIT License",
    classifiers=[
        "Development Status :: 2 - Pre-Alpha",
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent",
    ],
    python_requires='>=3.6',
)


import sys
from pathlib import Path

#############################################
root_path = Path(__file__).parents
sorting_  = str( root_path[1] ) + '/sorting/'

sys.path.append( sorting_ )
from quick_sort import *
#############################################

import sys
from pathlib import Path

#############################################
root_path = Path(__file__).parents
sorting_  = str( root_path[1] ) + '/sorting/'

sys.path.append( sorting_ )
from quick_sort import *
#############################################

# Python program to find an element x in a sorted array using Exponential Search
# A recursive binary search function returns location  of x in given array arr[l..r] is present, otherwise -1

def binarySearch( arr, l, r, x):

    if r >= l:

        mid = ( r + l ) // 2

        if arr[mid] == x:
            return mid

        if arr[mid] > x:
            return binarySearch(arr, l, mid - 1, x)

        return binarySearch(arr, mid + 1, r, x)

    return -1

# Returns the position of first occurrence of x in array
def exponentialSearch(arr, n, x):

    # IF x is present at first location itself
    if arr[0] == x:
        return 0

    # Find range for binary search j by repeated doubling
    i = 1
    while i < n and arr[i] <= x:
        i = i * 2

    # Call binary search for the found range
    return binarySearch( arr, i // 2, min(i, n - 1), x)


#!/usr/bin/env python
# encoding: utf-8

'''
Created by Brian Cherinka on 2016-05-02 16:16:27
Licensed under a 3-clause BSD license.

Revision History:
    Initial Version: 2016-05-02 16:16:27 by Brian Cherinka
    Last Modified On: 2016-05-02 16:16:27 by Brian

'''
from __future__ import print_function
from __future__ import division
from flask import session as current_session, request, current_app
from marvin import config
from collections import defaultdict
import flask_featureflags as feature
import re

def check_request_for_release(request):
    ''' Check Flask request for release information and set it in config

    Parameters:
        request (Flask.Request):
            Flask Request object

    '''

    # get release from request header
    release = request.headers.get('Release', None)

    # if no release, get release variable from Flask request
    if not release:
        release = request.environ.get('MARVIN_RELEASE', None)

    # if release then set change it in the config but only when the session is empty
    if release:
        config.setRelease(release.upper())

def configFeatures(app):
    ''' Configure Flask Feature Flags '''

    app.config['FEATURE_FLAGS']['public'] = True if config.access == 'public' else False

def check_access():
    ''' Check the access mode in the session '''

    # check if on public server
    public_server = request.environ.get('PUBLIC_SERVER', None) == 'True'
    public_flag = public_server or current_app.config['FEATURE_FLAGS']['public']
    current_app.config['FEATURE_FLAGS']['public'] = public_server
    public_access = config.access == 'public'

    # ensure always in public mode when using public server
    if public_flag:
        config.access = 'public'
        return

    # check for logged in status
    logged_in = current_session.get('loginready', None)

    # commenting this out as this seems to cause problems with the collab web server
    # switching back and forth between config.access public and collab and removing the allowed
    # MPL releases from the collab version.  Leaving here in case we need to go back and I don't
    # forget.

    # if not logged_in and not public_access:
    #     config.access = 'public'
    # elif logged_in is True and public_access:
    #     config.access = 'collab'

def get_web_releases():
    ''' Get the dict of supported web releases '''
    web_releases = current_app.config.get('WEB_RELEASES', None)
    releases = config.get_allowed_releases(web_releases=web_releases)
    return releases

def update_allowed():
    ''' Update the allowed versions shown in the header dropdown '''

    logged_in = current_session.get('loginready', None)
    mpls = list(get_web_releases().keys())

    # this is to remove MPLs from the list of web versions when not logged in to the collaboration site
    if not logged_in:
        # select only DR allowed releases
        mpls = [mpl for mpl in mpls if 'DR' in mpl]

        # select the release to switch to; prioritize the session release
        if 'release' not in current_session:
            release = config.release if 'DR' in config.release else max(mpls)
        else:
            release = current_session['release'] if 'DR' in current_session['release'] else max(mpls)

        # update the session with the new release
        set_session_versions(release)

    versions = [{'name': mpl, 'subtext': str(config.lookUpVersions(release=mpl))} for mpl in mpls]
    return versions

def set_session_versions(version):
    ''' Set the versions in the session '''
    current_session['release'] = version
    drpver, dapver = config.lookUpVersions(release=version)
    current_session['drpver'] = drpver
    current_session['dapver'] = dapver

def updateGlobalSession():
    ''' updates the Marvin config with the global Flask session '''

    # check if mpl versions in session
    if 'versions' not in current_session:
        setGlobalSession()
    elif 'drpver' not in current_session or \
         'dapver' not in current_session:
        set_session_versions(config.release)
    elif 'release' not in current_session:
        current_session['release'] = config.release
    elif 'release' in current_session:
        if current_session['release'] not in get_web_releases():
            current_session['release'] = config.release

    # reset the session versions if on public site
    if feature.is_active('public'):
        current_session['versions'] = update_allowed()

def setGlobalSession():
    ''' Sets the global session for Flask '''

    current_session['versions'] = update_allowed()

    if 'release' not in current_session:
        set_session_versions(config.release)

def parseSession():
    ''' parse the current session for MPL versions '''
    drpver = current_session['drpver']
    dapver = current_session['dapver']
    release = current_session['release']
    return drpver, dapver, release

def buildImageDict(imagelist, test=None, num=16):
    ''' Builds a list of dictionaries from a sdss_access return list of images '''

    # get thumbnails and plateifus
    if imagelist:
        thumbs = [imagelist.pop(imagelist.index(t)) if 'thumb' in t else t for t in imagelist]
        plateifu = ['-'.join(re.findall('\d{3,5}', im)) for im in imagelist]

    # build list of dictionaries
    images = []
    if imagelist:
        for i, image in enumerate(imagelist):
            imdict = defaultdict(str)
            imdict['name'] = plateifu[i]
            imdict['image'] = image
            imdict['thumb'] = thumbs[i] if thumbs else None
            images.append(imdict)
    elif test and not imagelist:
        for i in range(num):
            imdict = defaultdict(str)
            imdict['name'] = '4444-0000'
            imdict['image'] = 'http://placehold.it/470x480&text={0}'.format(i)
            imdict['thumb'] = 'http://placehold.it/150x150&text={0}'.format(i)
            images.append(imdict)

    return images


from django.contrib.auth import logout
from django.contrib.auth.decorators import login_required, user_passes_test
from django.contrib.auth.models import User
from django.core.exceptions import ObjectDoesNotExist, PermissionDenied
from django.db.models import Count
from django.forms import (
    ModelForm,
    ModelMultipleChoiceField,
    modelformset_factory,
)
from django.shortcuts import get_object_or_404, render
from django.urls import reverse
from django.utils.translation import gettext_lazy as _
from django.views.decorators.http import require_http_methods
from django.views.generic.edit import UpdateView

from ..mixins import LoginRequiredMixin, NoStudentsMixin, student_check
from ..models import (
    Answer,
    AnswerAnnotation,
    Assignment,
    Discipline,
    Question,
    QuestionFlag,
    QuestionFlagReason,
    ShownRationale,
)

@login_required
@user_passes_test(student_check, login_url="/access_denied_and_logout/")
def research_index(request):
    template = "peerinst/research/index.html"
    context = {"disciplines": Discipline.objects.all()}
    return render(request, template, context)

def get_question_annotation_counts(discipline_title, annotator, assignment_id):
    """
    Returns:
    ========
    list of dicts, one for each question in discipline, and the counts of
    Answers and AnswerAnnotations for each
    """
    if discipline_title:
        questions_qs = Question.objects.filter(
            discipline__title=discipline_title
        )
    elif assignment_id:
        questions_qs = Assignment.objects.get(
            identifier=assignment_id
        ).questions.order_by("assignmentquestions__rank")

    # FIXME:
    translation_table = str.maketrans("ABCDEFG", "1234567")

    question_annotation_counts = []
    for q in questions_qs:
        d1 = {}
        d1["question"] = q

        d1["question_expert_answers"] = q.answer_set.filter(expert=True)

        # need at least one sample answer that is not marked as expert for each
        # answer choice
        d1["enough_sample_answers"] = 0 not in list(
            q.get_frequency(all_rationales=True)["first_choice"].values()
        )

        d1["total_annotations"] = AnswerAnnotation.objects.filter(
            score__isnull=False, answer__question_id=q.pk
        ).count()
        d1["total_annotations_by_user"] = AnswerAnnotation.objects.filter(
            score__isnull=False, answer__question_id=q.pk, annotator=annotator
        ).count()
        flagged_questions = Question.flagged_objects.all()
        flagged_by_user = QuestionFlag.objects.filter(
            user=annotator
        ).values_list("question", flat=True)
        if q.pk in flagged_by_user:
            d1["flag_color_code"] = "red"
        elif q in flagged_questions:
            d1["flag_color_code"] = "goldenrod"
            d1["flagged_reasons"] = "; ".join(
                [
                    " - ".join(e)
                    for e in QuestionFlag.objects.filter(
                        flag=True, question=q
                    ).values_list("user__username", "flag_reason__title")
                ]
            )
        else:
            d1["flag_color_code"] = None

        answer_frequencies = q.get_frequency_json("first_choice")
        for d2 in answer_frequencies:
            a_choice = str(d2["answer_label"][0]).translate(translation_table)
            d2.update(
                {
                    "annotation_count": AnswerAnnotation.objects.filter(
                        score__isnull=False,
                        answer__question_id=q.pk,
                        answer__first_answer_choice=a_choice,
                    ).count(),
                    "annotation_count_by_user": AnswerAnnotation.objects.filter(  # noqa
                        score__isnull=False,
                        answer__question_id=q.pk,
                        answer__first_answer_choice=a_choice,
                        annotator=annotator,
                    ).count(),
                }
            )
        d1["answerchoices"] = answer_frequencies
        question_annotation_counts.append(d1)

    return question_annotation_counts

@login_required
@user_passes_test(student_check, login_url="/access_denied_and_logout/")
def research_discipline_question_index(
    request, discipline_title=None, assignment_id=None
):
    template = "peerinst/research/question_index.html"

    annotator = get_object_or_404(User, username=request.user)

    question_annotation_counts = get_question_annotation_counts(
        discipline_title=discipline_title,
        annotator=annotator,
        assignment_id=assignment_id,
    )
    request.session["assignment_id"] = assignment_id

    context = {
        "questions": question_annotation_counts,
        "discipline_title": discipline_title,
        "assignment_id": assignment_id,
        "annotations_count": AnswerAnnotation.objects.filter(
            annotator=annotator, score__isnull=False
        ).count(),
        "annotator": annotator,
    }
    return render(request, template, context)

@require_http_methods(["GET", "POST"])
@login_required
@user_passes_test(student_check, login_url="/access_denied_and_logout/")
def research_question_answer_list(
    request,
    question_pk,
    answerchoice_value,
    discipline_title=None,
    assignment_id=None,
):
    template = "peerinst/research/answer_list.html"
    if discipline_title:
        reverse_url_name = "research-question-answer-list-by-discipline"
    elif assignment_id:
        reverse_url_name = "research-question-answer-list-by-assignment"

    annotator = get_object_or_404(User, username=request.user)
    if not annotator:
        logout(request)
        raise PermissionDenied

    # FIXME
    if answerchoice_value.isdigit():
        answerchoice_id = answerchoice_value
    else:
        answerchoice_id = {"A": 1, "B": 2, "C": 3, "D": 4, "E": 5, "F": 6}[
            answerchoice_value
        ]

    answer_qs = Question.objects.get(pk=question_pk).answer_set.filter(
        question_id=question_pk, first_answer_choice=answerchoice_id
    )
    for a in answer_qs:
        annotation, created = AnswerAnnotation.objects.get_or_create(
            answer=a,
            annotator=annotator,
        )

    # only need two expert scores per rationale,
    # and those marked never show even by one person can be excluded
    already_scored = (
        AnswerAnnotation.objects.filter(
            answer__question_id=question_pk,
            score__isnull=False,
            answer__first_answer_choice=answerchoice_id,
        )
        .values("answer")
        .order_by("answer")
        .annotate(times_scored=Count("answer"))
        .filter(times_scored__gte=3)
        .values_list("answer__id", flat=True)
    )

    queryset = (
        AnswerAnnotation.objects.filter(
            answer__question_id=question_pk,
            annotator=annotator,
            answer__first_answer_choice=answerchoice_id,
        )
        .exclude(answer__id__in=already_scored)
        .annotate(times_shown=Count("answer__shown_answer"))
        .order_by("-times_shown")
    )

    AnswerAnnotationFormset = modelformset_factory(
        AnswerAnnotation, fields=("score",), extra=0
    )

    formset = AnswerAnnotationFormset(queryset=queryset)

    context = {
        "formset": formset,
        "question": Question.objects.get(id=question_pk),
        "discipline_title": discipline_title,
        "question_pk": question_pk,
        "assignment_id": assignment_id,
        "annotations_count": AnswerAnnotation.objects.filter(
            annotator=annotator,
            score__isnull=False,
            answer__question_id=question_pk,
            answer__first_answer_choice=answerchoice_id,
        ).count(),
        "annotator": annotator,
    }

    if request.method == "POST":
        formset = AnswerAnnotationFormset(request.POST)
        if formset.is_valid():
            instances = formset.save()
            context.update(message=_("Scores updated"))

    return render(request, template, context)

def research_all_annotations_for_question(
    request, question_pk, discipline_title=None, assignment_id=None
):
    """
    Returns:
    ====================
    All answer annotations for a question, ordered for
    comparison by researchers
    """
    template = "peerinst/research/all_annotations.html"
    question = Question.objects.get(id=question_pk)
    context = {
        "question": question,
        "question_pk": question_pk,
        "discipline_title": discipline_title,
        "assignment_id": assignment_id,
    }

    all_annotations = []
    for label, _answerchoice_text in question.get_choices():
        d1 = {}
        d1["answerchoice"] = label
        d1["annotations"] = []

        # FIXME
        if label.isdigit():
            answerchoice_id = label
        else:
            answerchoice_id = {"A": 1, "B": 2, "C": 3, "D": 4, "E": 5, "F": 6}[
                label
            ]

        a_list = list(
            set(
                AnswerAnnotation.objects.filter(
                    score__isnull=False,
                    answer__question_id=question_pk,
                    answer__first_answer_choice=answerchoice_id,
                ).values_list("answer", flat=True)
            )
        )

        a_list.sort()

        for a in a_list:
            d2 = {}
            d2["answer"] = Answer.objects.get(pk=a)
            d2["scores"] = AnswerAnnotation.objects.filter(
                answer=a, score__isnull=False
            ).values("score", "annotator__username")
            d2["times_shown"] = ShownRationale.objects.filter(
                shown_answer=a
            ).count()
            d2["times_chosen"] = Answer.objects.filter(
                chosen_rationale_id=a
            ).count()
            d1["annotations"].append(d2)

        all_annotations.append(d1)

    context["all_annotations"] = all_annotations

    return render(request, template, context)

class QuestionFlagForm(ModelForm):
    flag_reason = ModelMultipleChoiceField(
        queryset=QuestionFlagReason.objects.all()
    )

    class Meta:
        model = QuestionFlag
        fields = ["flag", "flag_reason", "comment"]

@require_http_methods(["GET", "POST"])
@login_required
@user_passes_test(student_check, login_url="/access_denied_and_logout/")
def flag_question_form(
    request, question_pk, discipline_title=None, assignment_id=None
):
    """
    Get or Create QuestionFlag object for user, and allow edit
    """
    template = "peerinst/research/flag_question.html"
    question = get_object_or_404(Question, pk=question_pk)
    message = None

    try:
        question_flag = QuestionFlag.objects.get(
            user=request.user, question=question
        )
    except ObjectDoesNotExist:
        question_flag = None

    if request.method == "GET":
        form = QuestionFlagForm(instance=question_flag)

    if request.method == "POST":
        form = QuestionFlagForm(request.POST, instance=question_flag)

        if form.is_valid():
            form.instance.user = request.user
            form.instance.question = question
            instance = form.save()
            if instance.flag:
                message = _(
                    """
                    You have flagged this question, and your input
                    has been forwarded to a myDALITE content moderator.
                    """
                )
            else:
                message = _(
                    """
                    You have un-flagged this question, and thus it will be
                    taken off the list of potentially problematic questions.
                    """
                )

    context = {
        "form": form,
        "question": question,
        "message": message,
        "discipline_title": discipline_title,
        "assignment_id": assignment_id,
        "expert_answers": question.answer_set.filter(expert=True),
    }

    return render(request, template, context)

class AnswerExpertUpdateView(LoginRequiredMixin, NoStudentsMixin, UpdateView):
    model = Answer
    fields = ["expert"]
    template_name = "peerinst/research/answer-expert-update.html"

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context["teacher"] = self.request.user.teacher
        context["question"] = Question.objects.get(pk=self.object.question_id)
        return context

    def get_success_url(self):
        return reverse(
            "research-fix-expert-rationale",
            kwargs={"question_id": self.object.question_id},
        )


import random
import matplotlib.pyplot as plt

def error(candidate, dataX, dataY):
    """
    This function computes the sum of vertical square distances of a data set from a line.
    param: candidate: Dictionary containing gradient "m" and y-intercept "c", defining a line.
    return: The sum of vertical square distances between data and line.
    """
    err = 0
    for x, y in zip(dataX, dataY):
        y_ = candidate["m"] * x + candidate["c"]
        err = err + (y-y_)**2
    return err

## Set the random seed to allow repeatable random number generation.
random.seed(15)

## Choose m and c, and generate (x, y) pairs at random.
m = 0.25
c = 3
X = [random.random()*10 for _ in range(20)]
Y = [m * x + c + (random.random()-0.5)*0.2 for x in X]

## Plot the data points.
plt.scatter(X, Y, label="Data")

## Randomly generate the first generation of candidate solutions.
candidates = [{
    "m": (random.random() - 0.5) * 10,
    "c": (random.random() - 0.5) * 10
} for _ in range(100)]

## Compute the error for each candidate.
for candidate in candidates:
    candidate["error"] = error(candidate, X, Y)

## Sort the candidates from best to worst, and display the attributes of the best.
candidates.sort(key=lambda x: x["error"])
print(
    "m:", round(candidates[0]["m"], 2),
    "c:", round(candidates[0]["c"], 2),
    "e:", round(candidates[0]["error"], 2)
)

## Draw the best candidate line from the first generation.
plt.plot([0, 10], [candidates[0]['c'], 10*candidates[0]['m']+candidates[0]['c']], label="First candidate solution")

## Iteratively improve the solution.
for index in range(50):
    ## Keep the best 25% of candidates and throw away the rest.
    candidates = candidates[:25]

    ## Generate new candidates by randomly perturbing those we kept from the previous generation.
    for i in range(25):
        for _ in range(3):
            new_candidate = {
                "m": candidates[i]["m"] + (random.random()-0.5)*10/100,
                "c": candidates[i]["c"] + (random.random()-0.5)*10/100
            }
            candidates.append(new_candidate)

    ## Compute the error for each candidate.
    for candidate in candidates:
        candidate["error"] = error(candidate, X, Y)

    ## Sort the candidates from best to worst, and display the attributes of the best.
    candidates.sort(key=lambda x: x["error"])
    print(
        "m:", round(candidates[0]["m"], 2),
        "c:", round(candidates[0]["c"], 2),
        "e:", round(candidates[0]["error"], 2)
    )

## Draw the best candidate line from the final generation.
plt.plot([0, 10], [candidates[0]['c'], 10*candidates[0]['m']+candidates[0]['c']], label="Final solution")

## Display the plot.
plt.legend()
plt.show()