# Copyright (c) 2019, Apple Inc. All rights reserved.
#
# Use of this source code is governed by a BSD-3-clause license that can be
# found in the LICENSE.txt file or at https://opensource.org/licenses/BSD-3-Clause

import os
import shutil
from coremltools.models.utils import _is_macos

import unittest

from coremltools.models import MLModel
from coremltools.models.nearest_neighbors import KNearestNeighborsClassifierBuilder
from coremltools._deps import _HAS_SKLEARN

if _HAS_SKLEARN:
    from sklearn.datasets import load_iris

@unittest.skipIf(not _HAS_SKLEARN, "Missing sklearn. Skipping tests.")
class NearestNeighborsBuilderTest(unittest.TestCase):
    """
    Unit tests for the nearest neighbors builder class.
    """

    def setUp(self):
        iris_samples = load_iris()
        self.iris_X = iris_samples.data
        self.iris_y = iris_samples.target
        self.training_X = self.iris_X[-30:]
        self.training_y = self.iris_y[-30:]

    def tearDown(self):
        # Do any cleanup here
        pass

    def create_builder(self, default_class_label="default_label"):
        builder = KNearestNeighborsClassifierBuilder(
            input_name="input",
            output_name="output",
            number_of_dimensions=4,
            default_class_label=default_class_label,
        )
        return builder

    def test_builder_output_types(self):
        builder = self.create_builder(default_class_label="default")
        self.assertIsNotNone(builder)
        self.assertTrue(
            builder.spec.kNearestNeighborsClassifier.HasField("stringClassLabels")
        )

        builder = self.create_builder(default_class_label=12)
        self.assertIsNotNone(builder)
        self.assertTrue(
            builder.spec.kNearestNeighborsClassifier.HasField("int64ClassLabels")
        )

        with self.assertRaises(TypeError):
            bad_default_label = float(21.32)
            self.create_builder(default_class_label=bad_default_label)

    def test_builder_training_input(self):
        builder = self.create_builder(default_class_label="default")
        self.assertIsNotNone(builder)
        self.assertTrue(
            builder.spec.kNearestNeighborsClassifier.HasField("stringClassLabels")
        )

        self.assertEqual(builder.spec.description.trainingInput[0].name, "input")
        self.assertEqual(
            builder.spec.description.trainingInput[0].type.WhichOneof("Type"),
            "multiArrayType",
        )
        self.assertEqual(builder.spec.description.trainingInput[1].name, "output")
        self.assertEqual(
            builder.spec.description.trainingInput[1].type.WhichOneof("Type"),
            "stringType",
        )

    def test_make_updatable(self):
        builder = self.create_builder()
        self.assertIsNotNone(builder)

        self.assertTrue(builder.spec.isUpdatable)
        builder.is_updatable = False
        self.assertFalse(builder.spec.isUpdatable)
        builder.is_updatable = True
        self.assertTrue(builder.spec.isUpdatable)

    def test_author(self):
        builder = self.create_builder()
        self.assertIsNotNone(builder)

        self.assertEqual(builder.spec.description.metadata.author, "")
        builder.author = "John Doe"
        self.assertEqual(builder.author, "John Doe")
        self.assertEqual(builder.spec.description.metadata.author, "John Doe")

    def test_description(self):
        builder = self.create_builder()
        self.assertIsNotNone(builder)

        self.assertEqual(builder.spec.description.metadata.shortDescription, "")
        builder.description = "This is a description"
        self.assertEqual(builder.description, "This is a description")
        self.assertEqual(
            builder.spec.description.metadata.shortDescription, "This is a description"
        )

    def test_weighting_scheme(self):
        builder = self.create_builder()
        self.assertIsNotNone(builder)

        builder.weighting_scheme = "uniform"
        self.assertEqual(builder.weighting_scheme, "uniform")

        builder.weighting_scheme = "inverse_distance"
        self.assertEqual(builder.weighting_scheme, "inverse_distance")

        builder.weighting_scheme = "unIfOrM"
        self.assertEqual(builder.weighting_scheme, "uniform")

        builder.weighting_scheme = "InVerSE_DISTance"
        self.assertEqual(builder.weighting_scheme, "inverse_distance")

        with self.assertRaises(TypeError):
            builder.weighting_scheme = "test"

    def test_index_type(self):
        builder = self.create_builder()
        self.assertIsNotNone(builder)

        self.assertEqual(builder.index_type, "linear")
        self.assertEqual(builder.leaf_size, 0)

        builder.set_index_type("kd_tree")
        self.assertEqual(builder.index_type, "kd_tree")  # test default value
        self.assertEqual(builder.leaf_size, 30)

        builder.set_index_type("linear")
        self.assertEqual(builder.index_type, "linear")
        self.assertEqual(builder.leaf_size, 0)

        builder.set_index_type("kd_tree", leaf_size=45)  # test user-defined value
        self.assertEqual(builder.index_type, "kd_tree")
        self.assertEqual(builder.leaf_size, 45)

        builder.set_index_type("linear", leaf_size=37)
        self.assertEqual(builder.index_type, "linear")
        self.assertEqual(builder.leaf_size, 0)

        builder.set_index_type("KD_TrEe", leaf_size=22)  # test user-defined value
        self.assertEqual(builder.index_type, "kd_tree")
        self.assertEqual(builder.leaf_size, 22)

        builder.set_index_type("linEAR")
        self.assertEqual(builder.index_type, "linear")
        self.assertEqual(builder.leaf_size, 0)

        with self.assertRaises(TypeError):
            builder.set_index_type("unsupported_index")

        with self.assertRaises(TypeError):
            builder.set_index_type("kd_tree", -10)

        with self.assertRaises(TypeError):
            builder.set_index_type("kd_tree", 0)

    def test_leaf_size(self):
        builder = self.create_builder()
        self.assertIsNotNone(builder)

        builder.set_index_type("kd_tree", leaf_size=45)  # test user-defined value
        self.assertEqual(builder.index_type, "kd_tree")
        self.assertEqual(builder.leaf_size, 45)

        builder.leaf_size = 12
        self.assertEqual(builder.index_type, "kd_tree")
        self.assertEqual(builder.leaf_size, 12)

    def test_set_number_of_neighbors_with_bounds(self):
        builder = self.create_builder()
        self.assertIsNotNone(builder)

        self.assertEqual(builder.number_of_neighbors, 5)
        (min_value, max_value) = builder.number_of_neighbors_allowed_range()
        self.assertEqual(min_value, 1)
        self.assertEqual(max_value, 1000)

        builder.set_number_of_neighbors_with_bounds(12, allowed_range=(2, 24))
        (min_value, max_value) = builder.number_of_neighbors_allowed_range()
        self.assertEqual(builder.number_of_neighbors, 12)
        self.assertEqual(min_value, 2)
        self.assertEqual(max_value, 24)
        allowed_values = builder.number_of_neighbors_allowed_set()
        self.assertIsNone(allowed_values)

        test_set = {3, 5, 7, 9}
        builder.set_number_of_neighbors_with_bounds(7, allowed_set=test_set)
        self.assertEqual(builder.number_of_neighbors, 7)
        allowed_values = builder.number_of_neighbors_allowed_set()
        self.assertIsNotNone(allowed_values)
        self.assertEqual(allowed_values, test_set)

    def test_set_number_of_neighbors_with_bounds_error_conditions(self):
        builder = self.create_builder()
        self.assertIsNotNone(builder)

        with self.assertRaises(ValueError):
            builder.set_number_of_neighbors_with_bounds(3)

        test_range = (3, 15)
        test_set = {1, 3, 5}
        with self.assertRaises(ValueError):
            builder.set_number_of_neighbors_with_bounds(
                3, allowed_range=test_range, allowed_set=test_set
            )

        with self.assertRaises(ValueError):
            builder.set_number_of_neighbors_with_bounds(3, allowed_range=(-5, 5))

        with self.assertRaises(ValueError):
            builder.set_number_of_neighbors_with_bounds(3, allowed_range=(5, 1))

        with self.assertRaises(ValueError):
            builder.set_number_of_neighbors_with_bounds(
                3, allowed_range=test_range, allowed_set=test_set
            )

        with self.assertRaises(ValueError):
            builder.set_number_of_neighbors_with_bounds(2, allowed_range=test_range)

        with self.assertRaises(TypeError):
            builder.set_number_of_neighbors_with_bounds(5, allowed_set={5, -3, 7})

        with self.assertRaises(ValueError):
            builder.set_number_of_neighbors_with_bounds(4, allowed_set=test_set)

        with self.assertRaises(ValueError):
            builder.set_number_of_neighbors_with_bounds(4, allowed_set=test_set)

        with self.assertRaises(TypeError):
            builder.set_number_of_neighbors_with_bounds(2, allowed_set=[1, 2, 3])

        with self.assertRaises(TypeError):
            builder.set_number_of_neighbors_with_bounds(4, allowed_range={2, 200})

        with self.assertRaises(TypeError):
            builder.set_number_of_neighbors_with_bounds(4, allowed_range=(2, 10, 20))

        with self.assertRaises(TypeError):
            builder.set_number_of_neighbors_with_bounds(4, allowed_set=set())

        with self.assertRaises(TypeError):
            builder.set_number_of_neighbors_with_bounds(4, allowed_range=[])

    def test_set_number_of_neighbors(self):
        builder = self.create_builder()
        self.assertIsNotNone(builder)

        builder.set_number_of_neighbors_with_bounds(12, allowed_range=(2, 24))
        self.assertEqual(builder.number_of_neighbors, 12)

        with self.assertRaises(ValueError):
            builder.set_number_of_neighbors_with_bounds(1, allowed_range=(2, 24))
        builder.set_number_of_neighbors_with_bounds(4, allowed_range=(2, 24))
        self.assertEqual(builder.number_of_neighbors, 4)

        test_set = {3, 5, 7, 9}
        builder.set_number_of_neighbors_with_bounds(7, allowed_set=test_set)

        with self.assertRaises(ValueError):
            builder.set_number_of_neighbors_with_bounds(4, allowed_set=test_set)
        builder.set_number_of_neighbors_with_bounds(5, allowed_set=test_set)
        self.assertEqual(builder.number_of_neighbors, 5)

    def test_add_samples_invalid_data(self):
        builder = self.create_builder()
        self.assertIsNotNone(builder)

        invalid_X = [[1.0, 2.4]]
        with self.assertRaises(TypeError):
            builder.add_samples(invalid_X, self.training_y)

        with self.assertRaises(TypeError):
            builder.add_samples(self.training_X, self.training_y[:3])

        with self.assertRaises(TypeError):
            builder.add_samples([], self.training_y)

        with self.assertRaises(TypeError):
            builder.add_samples(self.training_X, [])

    def test_add_samples_int_labels(self):
        builder = self.create_builder(default_class_label=12)
        self.assertIsNotNone(builder)

        some_X = self.training_X[:10]
        some_y = self.training_y[:10]
        builder.add_samples(some_X, some_y)
        self._validate_samples(builder.spec, some_X, some_y)

        addl_X = self.training_X[10:20]
        addl_y = self.training_y[10:20]
        builder.add_samples(addl_X, addl_y)
        self._validate_samples(builder.spec, self.training_X[:20], self.training_y[:20])

    def test_add_samples_string_labels(self):
        builder = self.create_builder(default_class_label="default")
        self.assertIsNotNone(builder)

        some_X = self.training_X[:3]
        some_y = ["one", "two", "three"]
        builder.add_samples(some_X, some_y)
        self._validate_samples(builder.spec, some_X, some_y)

        addl_X = self.training_X[3:6]
        addl_y = ["four", "five", "six"]
        builder.add_samples(addl_X, addl_y)
        self._validate_samples(builder.spec, self.training_X[0:6], some_y + addl_y)

    def test_add_samples_invalid_label_types(self):
        builder_int_labels = self.create_builder(default_class_label=42)
        self.assertIsNotNone(builder_int_labels)

        some_X = self.training_X[:3]
        invalid_int_y = [0, "one", 2]
        with self.assertRaises(TypeError):
            builder_int_labels.add_samples(some_X, invalid_int_y)

        builder_string_labels = self.create_builder(default_class_label="default")
        self.assertIsNotNone(builder_string_labels)

        invalid_string_y = ["zero", "one", 2]
        with self.assertRaises(TypeError):
            builder_string_labels.add_samples(some_X, invalid_string_y)

    @unittest.skipUnless(_is_macos(), "Only supported on MacOS platform.")
    def test_can_init_and_save_model_from_builder_with_updated_spec(self):
        builder = KNearestNeighborsClassifierBuilder(
            input_name="input",
            output_name="output",
            number_of_dimensions=10,
            default_class_label="defaultLabel",
            k=3,
            weighting_scheme="inverse_distance",
            index_type="kd_tree",
            leaf_size=50,
        )
        builder.author = "CoreML Team"
        builder.license = "MIT"
        builder.description = "test_builder_with_validation"

        # Save the updated spec
        coreml_model = MLModel(builder.spec)
        self.assertIsNotNone(coreml_model)
        coreml_model_path = "/tmp/__test_builder_with_validation.mlmodel"

        try:
            coreml_model.save(coreml_model_path)
            self.assertTrue(os.path.isfile(coreml_model_path))
        finally:
            self._delete_mlmodel_and_mlmodelc(coreml_model_path)

    @unittest.skipUnless(_is_macos(), "Only supported on MacOS platform.")
    def test_can_init_and_save_model_from_builder_default_parameters(self):
        builder = KNearestNeighborsClassifierBuilder(
            input_name="input",
            output_name="output",
            number_of_dimensions=4,
            default_class_label="defaultLabel",
        )

        # Save the updated spec
        coreml_model = MLModel(builder.spec)
        self.assertIsNotNone(coreml_model)
        coreml_model_path = "/tmp/__test_builder_with_validation.mlmodel"

        try:
            coreml_model.save(coreml_model_path)
            self.assertTrue(os.path.isfile(coreml_model_path))
        finally:
            self._delete_mlmodel_and_mlmodelc(coreml_model_path)

    def _validate_samples(self, spec, expected_X, expected_y):
        """Validate the float samples returned from the converted scikit KNeighborsClassifier"""
        num_dimensions = (
            spec.kNearestNeighborsClassifier.nearestNeighborsIndex.numberOfDimensions
        )
        for index, sample in enumerate(
            spec.kNearestNeighborsClassifier.nearestNeighborsIndex.floatSamples
        ):
            for dim in range(0, num_dimensions):
                self.assertAlmostEqual(
                    sample.vector[dim], expected_X[index][dim], places=6
                )

        if spec.kNearestNeighborsClassifier.HasField("int64ClassLabels"):
            for index, label in enumerate(
                spec.kNearestNeighborsClassifier.int64ClassLabels.vector
            ):
                self.assertEqual(label, expected_y[index])

        elif spec.kNearestNeighborsClassifier.HasField("stringClassLabels"):
            for index, label in enumerate(
                spec.kNearestNeighborsClassifier.stringClassLabels.vector
            ):
                self.assertEqual(label, expected_y[index])

    @staticmethod
    def _delete_mlmodel_and_mlmodelc(path_to_mlmodel):
        """Delete the .mlmodel and .mlmodelc for the given .mlmodel."""
        if os.path.exists(path_to_mlmodel):
            os.remove(path_to_mlmodel)
        path_to_mlmodelc = "{}c".format(path_to_mlmodel)
        if os.path.exists(path_to_mlmodelc):
            shutil.rmtree(path_to_mlmodelc)

{'repo_name': 'apple/coremltools', 'stars': '1786', 'repo_language': 'Python', 'file_name': 'coreml-utils.cmake', 'mime_type': 'text/plain', 'hash': -1470743550062126222, 'source_dataset': 'data'}


################################################################################
#
# mkpasswd
#
################################################################################

# source included in buildroot, taken from
# https://github.com/rfc1036/whois/blob/master/
# at revision 5a0f08500fa51608b6d3b73ee338be38c692eadb
HOST_MKPASSWD_SOURCE =
HOST_MKPASSWD_LICENSE = GPLv2+

HOST_MKPASSWD_VERSION = buildroot-$(BR2_VERSION)

define HOST_MKPASSWD_BUILD_CMDS
	$(HOSTCC) $(HOST_CFLAGS) $(HOST_LDFLAGS) \
		package/mkpasswd/mkpasswd.c package/mkpasswd/utils.c \
		-o $(@D)/mkpasswd -lcrypt
endef

define HOST_MKPASSWD_INSTALL_CMDS
	$(INSTALL) -D -m 755 $(@D)/mkpasswd $(HOST_DIR)/usr/bin/mkpasswd
endef

$(eval $(host-generic-package))

MKPASSWD = $(HOST_DIR)/usr/bin/mkpasswd

{'repo_name': 'eq-3/occu', 'stars': '150', 'repo_language': 'Tcl', 'file_name': 'mxs_raw_auart.c', 'mime_type': 'text/x-c', 'hash': -6389429662003529659, 'source_dataset': 'data'}


<?php
/*
 * Copyright 2014 Google Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

class Google_Service_Books_DictlayerdataDictWordsSensesSource extends Google_Model
{
  public $attribution;
  public $url;

  public function setAttribution($attribution)
  {
    $this->attribution = $attribution;
  }
  public function getAttribution()
  {
    return $this->attribution;
  }
  public function setUrl($url)
  {
    $this->url = $url;
  }
  public function getUrl()
  {
    return $this->url;
  }
}

{'repo_name': 'GibbonEdu/core', 'stars': '232', 'repo_language': 'PHP', 'file_name': 'installerFunctions.php', 'mime_type': 'text/x-php', 'hash': 6010722839510470205, 'source_dataset': 'data'}


# !/usr/bin/env python3
# -*- coding: utf-8 -*-
"""

@author: jshliu

Scripts of multi-event encounter level models
"""

import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
from torch.autograd import Variable
from sklearn import metrics
import pdb
import pickle
import json
# import matplotlib.pyplot as plt
# from matplotlib.backends.backend_pdf import PdfPages
# import matplotlib.gridspec as gridspec

import util
from torch.utils.data import Dataset, DataLoader
import torch.optim as optim

## ====== Data Loaders ============
class encDataset(Dataset):
    # Each record is: [[text_enc1, text_enc2, ...], [num_enc1, num_enc2, ...], disease, mask, age, gender, race, eth]
    def __init__(self, root_dir, dsName, nClassGender, nClassRace, nClassEthnic, transform=None):
        self.root_dir = root_dir
        self.ds = json.load(open(root_dir + dsName, 'r'))
        print('Loaded: ', dsName)
        self.transform = transform
        self.nClass = [nClassGender, nClassRace, nClassEthnic]

    def __len__(self):
        return len(self.ds)

    def __getitem__(self, idx):
        """
        Shape of the inputs:
            - Note: 1 x n_enc x noteLength
            - Num: 1 x n_enc x dimNum
            - Disease: 1 x 3
            - Mask: 1 x 3
            - Age: 1
            - Demo ([Gender, race, eth]): 1 x 3
        """
        Note, Num, Disease, Mask, Age, gender, race, eth = self.ds[idx]

        Num = np.asarray(Num, dtype='float32')
        Disease = np.asarray(Disease, dtype='int')
        Mask = np.asarray(Mask, dtype='int')
        Age = np.asarray(Age, dtype='float32')

        gender2 = self._idx2onehot(gender, self.nClass[0])
        race2 = self._idx2onehot(race, self.nClass[1])
        eth2 = self._idx2onehot(eth, self.nClass[2])

        Demo = np.concatenate([gender2, race2, eth2])
        sample = {'Note': Note, 'Num': Num, 'Disease': Disease, 'Mask': Mask, 'Age': Age, 'Demo': Demo}

        if self.transform:
            sample = self.transform(sample)

        return sample

    def _idx2onehot(self, value_idx, max_idx):
        temp = np.zeros(max_idx)
        if value_idx > 0:
            temp[value_idx - 1] = 1
        return temp

class concatNote(object):
    """
    concatenate all notes regardless of encounter, but still maintain numerics by encounter
    """

    def __init__(self, enc_len, doc_len):
        self.enc_len = enc_len
        self.doc_len = doc_len

    def __call__(self, sample):
        Note, Num, Disease, Mask, Age, Demo = sample['Note'], sample['Num'], sample['Disease'], sample['Mask'], sample[
            'Age'], sample['Demo']
        return self._run(Note, Num, Disease, Mask, Age, Demo)

    def _run(self, Note, Num, Disease, Mask, Age, Demo):
        # Pad / truncate notes
        if len(Note) > self.enc_len:
            Num = Num[-self.enc_len:, :]
        padded_Num = torch.zeros(self.enc_len, Num.shape[1])
        padded_Num[-Num.shape[0]:, :] = torch.from_numpy(Num).float()

        note_all = [w for note in Note for w in note]
        if len(note_all) > self.doc_len:
            note_all = note_all[-self.doc_len:]
        padded_Note = torch.zeros(1, self.doc_len)
        if len(note_all) > 0:
            padded_Note[:, -len(note_all):] = torch.from_numpy(np.asarray(note_all)).long()

        return {'Note': padded_Note,
                'Num': padded_Num,
                'Disease': torch.from_numpy(Disease.reshape(1, -1)).float(),
                'Mask': torch.from_numpy(Mask.reshape(1, -1)).float(),
                'Age': torch.from_numpy(Age.reshape(1, -1)).float(),
                'Demo': torch.from_numpy(Demo.reshape(1, -1)).float()
                }

class staticDataset(Dataset):

        # Each record is: [[text_enc1, text_enc2, ...], [num_enc1, num_enc2, ...], disease, mask, age, gender, race, eth]
    def __init__(self, root_dir, dsName, nClassGender, nClassRace, nClassEthnic, transform=None):
        self.root_dir = root_dir
        self.ds = json.load(open(root_dir + dsName, 'r'))
        print('Loaded: ', dsName)
        self.transform = transform
        self.nClass = [nClassGender, nClassRace, nClassEthnic]
        self.max_len = 3000

    def __len__(self):
        return len(self.ds)

    def __getitem__(self, idx):

        """
        Shape of the inputs:
            - Note: 1 x n_enc x noteLength
            - Num: 1 x n_enc x dimNum
            - Disease: 1 x 3
            - Mask: 1 x 3
            - Age: 1
            - Demo ([Gender, race, eth]): 1 x 3
        """

        Note, Num, Disease, Mask, Age, gender, race, eth = self.ds[idx]

        Note = np.asarray([item for sublist in Note for item in sublist])
        Num = np.asarray(Num, dtype='float32').mean(axis=0)
        Disease = np.asarray(Disease, dtype='int')
        Mask = np.asarray(Mask, dtype='int')
        Age = np.asarray(Age, dtype='float32')

        if len(Note) > self.max_len:
            Note = Note[:self.max_len]

        else:
            Note = np.concatenate([ np.zeros( self.max_len - Note.shape[0] ) , Note ])

        Note = torch.from_numpy(Note).long()

        gender2 = self._idx2onehot(gender, self.nClass[0])
        race2 = self._idx2onehot(race, self.nClass[1])
        eth2 = self._idx2onehot(eth, self.nClass[2])

        Demo = np.concatenate([gender2, race2, eth2])
        sample = {'Note': Note, 'Num': Num, 'Disease': Disease, 'Mask': Mask, 'Age': Age, 'Demo': Demo}

        return {'Note': Note,
                'Num': Num,
                'Disease': torch.from_numpy(Disease.reshape(1, -1)).float(),
                'Mask': torch.from_numpy(Mask.reshape(1, -1)).float(),
                'Age': torch.from_numpy(Age.reshape(1, -1)).float(),
                'Demo': torch.from_numpy(Demo.reshape(1, -1)).float()
                }

    def _idx2onehot(self, value_idx, max_idx):

        temp = np.zeros(max_idx)
        if value_idx > 0:
            temp[value_idx - 1] = 1
        return temp

class padOrTruncateToTensor(object):
    """
    pad or truncate sample to make it as long as the given enc_len, doc_len
    """

    def __init__(self, enc_len, doc_len):
        self.enc_len = enc_len
        self.doc_len = doc_len

    def __call__(self, sample):
        Note, Num, Disease, Mask, Age, Demo = sample['Note'], sample['Num'], sample['Disease'], sample['Mask'], sample[
            'Age'], sample['Demo']
        return self._run(Note, Num, Disease, Mask, Age, Demo)

    def _run(self, Note, Num, Disease, Mask, Age, Demo):
        # Pad / truncate notes
        if len(Note) > self.enc_len:
            Note = Note[-self.enc_len:]
            Num = Num[-self.enc_len:, :]
        padded_Note = self._pad_doc(Note, self.doc_len, self.enc_len)
        padded_Num = torch.zeros(self.enc_len, Num.shape[1])
        padded_Num[-Num.shape[0]:, :] = torch.from_numpy(Num).float()

        return {'Note': padded_Note,
                'Num': padded_Num,
                'Disease': torch.from_numpy(Disease.reshape(1, -1)).float(),
                'Mask': torch.from_numpy(Mask.reshape(1, -1)).float(),
                'Age': torch.from_numpy(Age.reshape(1, -1)).float(),
                'Demo': torch.from_numpy(Demo.reshape(1, -1)).float()
                }

    def _pad_doc(self, seq, max_len, n):
        padded_seq = torch.zeros(n, max_len)
        start = 0 if len(seq) >= n else n - len(seq)
        for i, s in enumerate(seq):
            if len(s) > max_len:
                padded_seq[start + i] = torch.from_numpy(np.asarray(s[:max_len])).long()
            else:
                if len(s) == 0:
                    continue
                padded_seq[start + i, -len(s):] = torch.from_numpy(np.asarray(s)).long()
        return padded_seq

## ======= Models =========

class Enc_SumLSTM(nn.Module):

    def __init__(self, model_paras, embedding = None):
        """
        model_paras:
            - enc_len: max number of encounters in each sample
            - doc_len: max number of words in each enc, input dimension should be [batchSize, enc_len, doc_len]
            - flg_updateEmb: whether to train embeddings or not
            - batchSize: batch size
            - rnnType: 'GRU' or 'LSTM'
            - bidir: whether to train bi-directional RNN or not
            - p_dropOut: dropOut percentage
            - lsDim: dimensions of [hidden_state, linear dimension 1, linear dimension 2...]
            - flg_cuda: use GPU or not
            - emb_dim: embedding dimension, do not need if provide embedding
            - n_words: vocabulary size,  do not need if provide embedding
        To-do:
            expand to handle more than 1 linear layers
        """

        super(Enc_SumLSTM,self).__init__()

        #self.enc_len = model_paras.get('enc_len', 15)
        self.doc_len = model_paras.get('doc_len', 20)
        flg_updateEmb = model_paras.get('flg_updateEmb', False)
        self.model_paras = model_paras
        self.rnnType = model_paras.get('rnnType', 'GRU')
        self.bidir = model_paras.get('bidir', False)
        self.p_dropOut = model_paras.get('p_dropOut', 0.8)
        self.lsDim = model_paras.get('lsDim', [128, 1])
        self.flg_cuda = model_paras.get('flg_cuda', True)

        if embedding is not None:
            self.n_words = embedding.size()[0]
            self.emb_dim = embedding.size()[1]
            self.embed = nn.Embedding(self.n_words, self.emb_dim)
            self.embed.weight = nn.Parameter(embedding,requires_grad=flg_updateEmb)
        else:
            self.n_words = model_paras.get('n_words', 20000)
            self.emb_dim = model_paras.get('emb_dim', 300)
            self.embed = nn.Embedding(self.n_words, self.emb_dim)

        self.lstm = getattr(nn, self.rnnType)(self.emb_dim, self.lsDim[0], 1, batch_first=True, bidirectional = self.bidir, dropout= self.p_dropOut)

        if self.bidir:
            self.fc = nn.Linear(2*self.lsDim[0], self.lsDim[1])
        else:
            self.fc = nn.Linear(self.lsDim[0], self.lsDim[1])

        self.params = list(self.lstm.parameters()) + list(self.fc.parameters())
        if flg_updateEmb:
            self.params += list(self.embed.parameters())

    def init_hidden(self, batchSize, nlayer = 1):
        # Before we've done anything, we dont have any hidden state.
        # Refer to the Pytorch documentation to see exactly
        # why they have this dimensionality.
        # The axes semantics are (num_layers, minibatch_size, hidden_dim)
        if self.rnnType == 'LSTM':
            if self.flg_cuda:
                return  (Variable(torch.zeros(nlayer, batchSize, self.lsDim[0])).cuda(), Variable(torch.zeros(nlayer, batchSize, self.lsDim[0])).cuda() )
            else:
                return (Variable(torch.zeros(nlayer, batchSize, self.lsDim[0])), Variable(torch.zeros(nlayer, batchSize, self.lsDim[0])))
        else:
            if self.flg_cuda:
                return Variable(torch.zeros(nlayer, batchSize, self.lsDim[0])).cuda()
            else:
                return Variable(torch.zeros(nlayer, batchSize, self.lsDim[0]))

    def forward(self,x):
        batchSize = x.size()[0]
        x= x.view(-1,self.doc_len)

        E= self.embed(x)

        E = torch.chunk(E, batchSize, 0)
        E = torch.stack(E)
        E = E.sum(2)
        #x = x.transpose(1,2)

        if self.bidir:
            h0 = self.init_hidden(batchSize = batchSize, nlayer = 2)
            z = self.lstm(E, h0)[1]
            z = torch.cat([ z[0].squeeze() , z[1].squeeze()] , 1)

        else:
            h0 = self.init_hidden(batchSize = batchSize, nlayer = 1)
            z = self.lstm(E, h0)[0][:, -1, :]

        y_hat = self.fc(z)
        return F.sigmoid(y_hat)

class Enc_CNN_LSTM(nn.Module):

    def __init__(self, model_paras, embedding=None):
        """
        CNN on word, then LSTM with enc
        model_paras:
        - enc_len: max number of encounters in each sample
        - doc_len: max number of words in each enc, input dimension should be [batchSize, enc_len, doc_len]
        - flg_updateEmb: whether to train embeddings or not
        - batchSize: batch size
        - rnnType: 'GRU' or 'LSTM'
        - bidir: whether to train bi-directional RNN or not
        - p_dropOut: dropOut percentage
        - lsDim: dimensions of [hidden_state, multi-event dimension, 1]
        - flg_cuda: use GPU or not
        - emb_dim: embedding dimension, do not need if provide embedding
        - n_words: vocabulary size,  do not need if provide embedding
        - filters: dimension of CNN output
        - Ks: kernels
        - randn_std: std of random noise on embedding
        """

        super(Enc_CNN_LSTM, self).__init__()

        # self.enc_len = model_paras.get('enc_len', 30)
        self.doc_len = model_paras.get('doc_len', 800)
        flg_updateEmb = model_paras.get('flg_updateEmb', False)
        self.model_paras = model_paras
        self.rnnType = model_paras.get('rnnType', 'GRU')
        self.dimLSTM = model_paras.get('dimLSTM', 128)  # LSTM hidden layer dimension
        self.bidir = model_paras.get('bidir', False)
        self.p_dropOut = model_paras.get('p_dropOut', 0.8)
        self.lsDim = model_paras.get('lsDim')
        self.flg_cuda = model_paras.get('flg_cuda', True)
        self.filters = model_paras.get('filters', 128)
        self.Ks = model_paras.get('Ks', [1, 2])
        self.randn_std = model_paras.get('randn_std', None)
        self.lastRelu = model_paras.get('lastRelu', False)
        self.isViz = model_paras.get('isViz', False)
        self.flgBias = model_paras.get('flgBias', True)
        self.flg_AllLSTM = model_paras.get('flg_AllLSTM', False)

        if embedding is not None:
            self.n_words = embedding.size()[0]
            self.emb_dim = embedding.size()[1]
            self.embed = nn.Embedding(self.n_words, self.emb_dim)
            self.embed.weight = nn.Parameter(embedding, requires_grad=flg_updateEmb)
        else:
            self.n_words = model_paras.get('n_words', 20000)
            self.emb_dim = model_paras.get('emb_dim', 300)
            self.embed = nn.Embedding(self.n_words, self.emb_dim)

        self.convs = nn.ModuleList([nn.Conv1d(self.emb_dim, self.filters, K) for K in self.Ks])
        self.bn_conv = nn.ModuleList([nn.BatchNorm1d(self.filters) for K in self.Ks])

        self.lstm = getattr(nn, self.rnnType)(self.filters * len(self.Ks), self.dimLSTM, 1, batch_first=True,
                                              bidirectional=self.bidir,
                                              dropout=self.p_dropOut, bias=self.flgBias)

        self.FCs = nn.ModuleList([nn.Linear(self.lsDim[i], self.lsDim[i + 1]) for i in range(len(self.lsDim) - 1)])
        self.bns = nn.ModuleList([nn.BatchNorm1d(self.lsDim[i + 1]) for i in range(len(self.lsDim) - 2)])

        self.params = list(self.lstm.parameters())
        # self.lstm.weight.data.normal_(0.0, 0.02)
        # self.lstm.bias.data.normal_(0.0, 0.01)
        # self.reg_params = list(self.lstm.)
        for c in self.convs:
            self.params += list(c.parameters())
            # n = c.kernel_size[0] * c.out_channels
            # c.weight.data.normal_(0, np.sqrt(2. / n))

        for b in self.bn_conv:
            self.params += list(b.parameters())

        for fc in self.FCs:
            self.params += list(fc.parameters())
            # fc.weight.data.normal_(0.0, 0.02)

        for bn in self.bns:
            self.params += list(bn.parameters())

        if flg_updateEmb:
            self.params += list(self.embed.parameters())

    def fc_layer(self, x, layer, bn=None):
        flg_bn = self.model_paras.get('flg_bn', True)
        p_dropOut = self.model_paras.get('p_dropOut', 0.5)

        x = layer(x)
        if flg_bn:
            x = F.relu(bn(x))
        else:
            x = F.relu(x)
        x = F.dropout(x, p_dropOut)
        return x

    def init_hidden(self, batchSize, nlayer=1):
        if self.rnnType == 'LSTM':
            if self.flg_cuda:
                return (Variable(torch.zeros(nlayer, batchSize, self.dimLSTM)).cuda(),
                        Variable(torch.zeros(nlayer, batchSize, self.dimLSTM)).cuda())
            else:
                return (Variable(torch.zeros(nlayer, batchSize, self.dimLSTM)),
                        Variable(torch.zeros(nlayer, batchSize, self.dimLSTM)))
        else:
            if self.flg_cuda:
                return Variable(torch.zeros(nlayer, batchSize, self.dimLSTM)).cuda()
            else:
                return Variable(torch.zeros(nlayer, batchSize, self.dimLSTM))

    def forward(self, Note, Num, Disease, Mask, Age, Demo):

        batchSize = Note.size()[0]
        self.x = Note.view(-1, self.doc_len)

        self.E = self.embed(self.x)  # Size: [batch * enc, doc_len, emb_dim]
        E = self.E.transpose(1, 2)

        if (self.randn_std is not None) & (self.training == True):
            if self.flg_cuda:
                noise = Variable(torch.randn(E.size()) * self.randn_std).cuda()
            else:
                noise = Variable(torch.randn(E.size()) * self.randn_std)
            E = E + noise

        # CNN on words
        h_CNN = [self.fc_layer(E, self.convs[i], self.bn_conv[i]) for i in range(len(self.convs))]
        h_CNN = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in h_CNN]
        h_CNN = torch.cat(h_CNN, 1)

        E2 = torch.chunk(h_CNN, batchSize, 0)
        E2 = torch.stack(E2)

        if self.bidir:
            h0 = self.init_hidden(batchSize=batchSize, nlayer=2)
            z = self.lstm(E2, h0)[1]
            z = torch.cat([z[0].squeeze(), z[1].squeeze()], 1)
        else:
            h0 = self.init_hidden(batchSize=batchSize, nlayer=1)
            if self.flg_AllLSTM:
                z = self.lstm(E2, h0)[0].contiguous().view(batchSize, -1)
            else:
                z = self.lstm(E2, h0)[0][:, -1, :]

        if len(self.lsDim) > 2:
            for i in range(len(self.lsDim) - 2):
                z = self.fc_layer(z, self.FCs[i], self.bns[i])

        self.y_hat = self.FCs[-1](z)

        if self.isViz:
            return self.y_hat
        else:
            return F.sigmoid(self.y_hat)

class DemoLab(nn.Module):
    def __init__(self, model_paras, embedding):
        """
        LSTM with lab and concatenate demographics
        model_paras:
        - enc_len: max number of encounters in each sample
        - flg_updateEmb: whether to train embeddings or not
        - batchSize: batch size
        - rnnType: 'GRU' or 'LSTM'
        - bidir: whether to train bi-directional RNN or not
        - p_dropOut: dropOut percentage
        - lsDim: dimensions of [hidden_state, multi-event dimension, 1]
        - flg_cuda: use GPU or not
        - emb_dim: embedding dimension, do not need if provide embedding
        - randn_std: std of random noise on embedding
        """

        super(DemoLab, self).__init__()

        self.enc_len = model_paras.get('enc_len', 30)
        flg_updateEmb = model_paras.get('flg_updateEmb', True) # Embedding for demographics
        self.model_paras = model_paras
        self.rnnType = model_paras.get('rnnType', 'GRU')
        self.dimLSTM = model_paras.get('dimLSTM', 128)  # LSTM hidden layer dimension
        self.bidir = model_paras.get('bidir', False)
        self.p_dropOut = model_paras.get('p_dropOut', 0.8)
        self.lsDim = model_paras.get('lsDim')
        self.flg_cuda = model_paras.get('flg_cuda', True)
        self.randn_std = model_paras.get('randn_std', None)
        self.lastRelu = model_paras.get('lastRelu', False)
        self.isViz = model_paras.get('isViz', False)
        self.flgBias = model_paras.get('flgBias', True)
        self.inSize = 151 + 56 + 1 # Dimension of input
        self.flg_AllLSTM = model_paras.get('flg_AllLSTM', False)

        #self.embed = nn.Embedding(self.n_words, self.emb_dim)

        self.lstm = getattr(nn, self.rnnType)(self.inSize, self.dimLSTM, 1, batch_first=True,
                                              bidirectional=self.bidir,
                                              dropout=self.p_dropOut, bias=self.flgBias)

        self.FCs = nn.ModuleList([nn.Linear(self.lsDim[i], self.lsDim[i + 1]) for i in range(len(self.lsDim) - 1)])
        self.bns = nn.ModuleList([nn.BatchNorm1d(self.lsDim[i + 1]) for i in range(len(self.lsDim) - 2)])

        self.params = list(self.lstm.parameters())

        for fc in self.FCs:
            self.params += list(fc.parameters())
            # fc.weight.data.normal_(0.0, 0.02)

        for bn in self.bns:
            self.params += list(bn.parameters())

    def fc_layer(self, x, layer, bn=None):
        flg_bn = self.model_paras.get('flg_bn', True)
        p_dropOut = self.model_paras.get('p_dropOut', 0.5)

        x = layer(x)
        if flg_bn:
            x = F.relu(bn(x))
        else:
            x = F.relu(x)
        x = F.dropout(x, p_dropOut)
        return x

    def init_hidden(self, batchSize, nlayer=1):
        if self.rnnType == 'LSTM':
            if self.flg_cuda:
                return (Variable(torch.zeros(nlayer, batchSize, self.dimLSTM)).cuda(),
                        Variable(torch.zeros(nlayer, batchSize, self.dimLSTM)).cuda())
            else:
                return (Variable(torch.zeros(nlayer, batchSize, self.dimLSTM)),
                        Variable(torch.zeros(nlayer, batchSize, self.dimLSTM)))
        else:
            if self.flg_cuda:
                return Variable(torch.zeros(nlayer, batchSize, self.dimLSTM)).cuda()
            else:
                return Variable(torch.zeros(nlayer, batchSize, self.dimLSTM))

    def forward(self, Note, Num, Disease, Mask, Age, Demo):

        batchSize = Note.size()[0]
        demo = torch.cat([Age, Demo], dim = 2)
        self.x = torch.cat([Num, demo.repeat(1, self.enc_len, 1)], dim = 2)

        if self.bidir:
            h0 = self.init_hidden(batchSize=batchSize, nlayer=2)
            z = self.lstm(self.x, h0)[1]
            z = torch.cat([z[0].squeeze(), z[1].squeeze()], 1)
        else:
            h0 = self.init_hidden(batchSize=batchSize, nlayer=1)
            if self.flg_AllLSTM:
                z = self.lstm(self.x, h0)[0].contiguous().view(batchSize, -1)
            else:
                z = self.lstm(self.x, h0)[0][:, -1, :]

        if len(self.lsDim) > 2:
            for i in range(len(self.lsDim) - 2):
                z = self.fc_layer(z, self.FCs[i], self.bns[i])

        self.y_hat = self.FCs[-1](z)

        if self.isViz:
            return self.y_hat
        else:
            return F.sigmoid(self.y_hat)

class Enc_CNN_LSTM_DemoLab(nn.Module):

    def __init__(self, model_paras, embedding=None):
        """
        CNN on word, then LSTM with enc
        model_paras:
        - enc_len: max number of encounters in each sample
        - doc_len: max number of words in each enc, input dimension should be [batchSize, enc_len, doc_len]
        - flg_updateEmb: whether to train embeddings or not
        - batchSize: batch size
        - rnnType: 'GRU' or 'LSTM'
        - bidir: whether to train bi-directional RNN or not
        - p_dropOut: dropOut percentage
        - lsDim: dimensions of [hidden_state, multi-event dimension, 1]
        - flg_cuda: use GPU or not
        - emb_dim: embedding dimension, do not need if provide embedding
        - n_words: vocabulary size,  do not need if provide embedding
        - filters: dimension of CNN output
        - Ks: kernels
        - randn_std: std of random noise on embedding
        """

        super(Enc_CNN_LSTM_DemoLab, self).__init__()

        # self.enc_len = model_paras.get('enc_len', 30)
        self.doc_len = model_paras.get('doc_len', 800)
        flg_updateEmb = model_paras.get('flg_updateEmb', False)
        self.model_paras = model_paras
        self.rnnType = model_paras.get('rnnType', 'GRU')
        self.dimLSTM = model_paras.get('dimLSTM', 128)  # LSTM hidden layer dimension
        self.bidir = model_paras.get('bidir', False)
        self.p_dropOut = model_paras.get('p_dropOut', 0.8)
        self.lsDim = model_paras.get('lsDim')
        self.flg_cuda = model_paras.get('flg_cuda', True)
        self.filters = model_paras.get('filters', 128)
        self.Ks = model_paras.get('Ks', [1, 2])
        self.randn_std = model_paras.get('randn_std', None)
        self.lastRelu = model_paras.get('lastRelu', False)
        self.isViz = model_paras.get('isViz', False)
        self.flgBias = model_paras.get('flgBias', True)
        self.flg_AllLSTM = model_paras.get('flg_AllLSTM', False)
        self.flg_useNum = model_paras.get('flg_useNum', False)
        if self.flg_useNum:
            self.inSize = 151 + 56 + 1  # Use lab + demo
        else:
            self.inSize = 56 + 1  # Demo only

        if embedding is not None:
            self.n_words = embedding.size()[0]
            self.emb_dim = embedding.size()[1]
            self.embed = nn.Embedding(self.n_words, self.emb_dim)
            self.embed.weight = nn.Parameter(embedding, requires_grad=flg_updateEmb)
        else:
            self.n_words = model_paras.get('n_words', 20000)
            self.emb_dim = model_paras.get('emb_dim', 300)
            self.embed = nn.Embedding(self.n_words, self.emb_dim)

        self.convs = nn.ModuleList([nn.Conv1d(self.emb_dim, self.filters, K) for K in self.Ks])
        self.bn_conv = nn.ModuleList([nn.BatchNorm1d(self.filters) for K in self.Ks])

        self.lstm = getattr(nn, self.rnnType)(self.filters * len(self.Ks) + self.inSize, self.dimLSTM, 1,
                                              batch_first=True,
                                              bidirectional=self.bidir,
                                              dropout=self.p_dropOut, bias=self.flgBias)

        self.FCs = nn.ModuleList([nn.Linear(self.lsDim[i], self.lsDim[i + 1]) for i in range(len(self.lsDim) - 1)])
        self.bns = nn.ModuleList([nn.BatchNorm1d(self.lsDim[i + 1]) for i in range(len(self.lsDim) - 2)])

        self.params = list(self.lstm.parameters())

        for c in self.convs:
            self.params += list(c.parameters())

        for b in self.bn_conv:
            self.params += list(b.parameters())

        for fc in self.FCs:
            self.params += list(fc.parameters())

        for bn in self.bns:
            self.params += list(bn.parameters())

        if flg_updateEmb:
            self.params += list(self.embed.parameters())

    def fc_layer(self, x, layer, bn=None):
        flg_bn = self.model_paras.get('flg_bn', True)
        p_dropOut = self.model_paras.get('p_dropOut', 0.5)

        x = layer(x)
        if flg_bn:
            x = F.relu(bn(x))
        else:
            x = F.relu(x)
        x = F.dropout(x, p_dropOut)
        return x

    def init_hidden(self, batchSize, nlayer=1):
        if self.rnnType == 'LSTM':
            if self.flg_cuda:
                return (Variable(torch.zeros(nlayer, batchSize, self.dimLSTM)).cuda(),
                        Variable(torch.zeros(nlayer, batchSize, self.dimLSTM)).cuda())
            else:
                return (Variable(torch.zeros(nlayer, batchSize, self.dimLSTM)),
                        Variable(torch.zeros(nlayer, batchSize, self.dimLSTM)))
        else:
            if self.flg_cuda:
                return Variable(torch.zeros(nlayer, batchSize, self.dimLSTM)).cuda()
            else:
                return Variable(torch.zeros(nlayer, batchSize, self.dimLSTM))

    def forward(self, Note, Num, Disease, Mask, Age, Demo):

        batchSize, enc_len = Note.size()[0], Note.size()[1]
        self.x = Note.view(-1, self.doc_len)

        self.E = self.embed(self.x)  # Size: [batch * enc, doc_len, emb_dim]
        E = self.E.transpose(1, 2)

        if (self.randn_std is not None) & (self.training == True):
            if self.flg_cuda:
                noise = Variable(torch.randn(E.size()) * self.randn_std).cuda()
            else:
                noise = Variable(torch.randn(E.size()) * self.randn_std)
            E = E + noise

        # CNN on words
        h_CNN = [self.fc_layer(E, self.convs[i], self.bn_conv[i]) for i in range(len(self.convs))]
        h_CNN = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in h_CNN]
        h_CNN = torch.cat(h_CNN, 1)

        E2 = torch.chunk(h_CNN, batchSize, 0)
        E2 = torch.stack(E2)

        demo = torch.cat([Age, Demo], dim=2)
        if self.flg_useNum:
            E2 = torch.cat([E2, Num, demo.repeat(1, enc_len, 1)], dim=2)
        else:
            E2 = torch.cat([E2, demo.repeat(1, enc_len, 1)], dim=2)

        if self.bidir:
            h0 = self.init_hidden(batchSize=batchSize, nlayer=2)
            z = self.lstm(E2, h0)[1]
            z = torch.cat([z[0].squeeze(), z[1].squeeze()], 1)
        else:
            h0 = self.init_hidden(batchSize=batchSize, nlayer=1)
            if self.flg_AllLSTM:
                z = self.lstm(E2, h0)[0].contiguous().view(batchSize, -1)
            else:
                z = self.lstm(E2, h0)[0][:, -1, :]

        if len(self.lsDim) > 2:
            for i in range(len(self.lsDim) - 2):
                z = self.fc_layer(z, self.FCs[i], self.bns[i])

        self.y_hat = self.FCs[-1](z)

        if self.isViz:
            return self.y_hat
        else:
            return F.sigmoid(self.y_hat)

# ============= Wrap training into a class ================
class trainModel(object):
    def __init__(self, train_paras, train_loader_pos, test_loader, model, optimizer, train_loader_neg=None):
        self.train_loader = train_loader_pos  # If no train_loader_neg, then pass the train_loader here; otherwise split _pos and _neg
        self.train_loader_neg = train_loader_neg
        self.test_loader = test_loader
        self.model = model
        self.optimizer = optimizer

        # self.train_paras = train_paras
        self.n_iter = train_paras.get('n_iter', 1)
        self.log_interval = train_paras.get('log_interval',
                                            1)  # list of two numbers: [log_per_n_epoch, log_per_n_batch]
        self.flg_cuda = train_paras.get('flg_cuda', False)
        # self.max_len = train_paras.get('max_len', 2000) # Max length of input
        self.lr_decay = train_paras.get('lr_decay',
                                        None)  # List of 4 numbers: [init_lr, lr_decay_rate, lr_decay_interval, min_lr]
        self.flgSave = train_paras.get('flgSave', False)  # Set to true if save model
        self.savePath = train_paras.get('savePath', './')
        self.posThres = train_paras.get('posThres', 0.5)
        self.alpha_L1 = train_paras.get('alpha_L1', 0.0)  # Regularization coefficient on fully connected weights
        self.flg_gradClip = train_paras.get('flg_gradClip', False)  # Whether to clip gradient or not

        if self.lr_decay:
            assert len(
                self.lr_decay) == 4  # Elements include: [starting_lr, decay_multiplier, decay_per_?_epoch, min_lr]
        self.criterion = torch.nn.BCELoss()
        # self.criterion = torch.nn.MultiLabelSoftMarginLoss() # For multi-class prediction
        self.cnt_iter = 0

    def run(self):
        lsTrainAccuracy = []
        lsTestAccuracy = []
        self.bestAccuracy = 0.0
        self.auc = 0.0

        for epoch in range(self.n_iter):
            self._train(epoch, lsTrainAccuracy)
            self._test(epoch, lsTestAccuracy)
            if self.auc > self.bestAccuracy:
                self.bestAccuracy = self.auc
                self._drawAUC(self.Y_multi, self.target_multi, self.mask)
                if self.flgSave:
                    self._saveModel()
        return self.model, lsTrainAccuracy, lsTestAccuracy

    def _train(self, epoch, lsTrainAccuracy):
        correct, train_loss = np.zeros(3), 0
        self.model.train()
        if self.lr_decay:
            lr = min(self.lr_decay[0] * (self.lr_decay[1] ** (epoch // self.lr_decay[2])), self.lr_decay[3])
            for param_group in self.optimizer.param_groups:
                param_group['lr'] = lr
        j, nRec = 0, np.zeros(3)

        self.Y_multi_train = []
        self.target_multi_train = []
        self.mask_train = []

        if epoch == 0:
            self.train_iter = self.train_loader.__iter__()
            if self.train_loader_neg is not None:
                self.train_iter_neg = self.train_loader_neg.__iter__()

        while (j <= 1000):
            try:
                sample = self.train_iter.__next__()
            except StopIteration:
                self.train_iter = self.train_loader.__iter__()
                sample = self.train_iter.__next__()

            Note, Num, Disease, Mask, Age, Demo = sample['Note'], sample['Num'], sample['Disease'], sample['Mask'], \
                                                  sample['Age'], sample['Demo']
            if self.train_loader_neg is not None:
                try:
                    sample_neg = self.train_iter_neg.__next__()
                except StopIteration:
                    self.train_iter_neg = self.train_loader_neg.__iter__()
                    sample_neg = self.train_iter_neg.__next__()

                Note_neg, Num_neg, Disease_neg, Mask_neg, Age_neg, Demo_neg = sample_neg['Note'], sample_neg['Num'], \
                                                                              sample_neg['Disease'], sample_neg['Mask'], \
                                                                              sample_neg['Age'], sample_neg['Demo']

                Note = torch.cat([Note, Note_neg], 0)
                Num = torch.cat([Num, Num_neg], 0)
                Disease = torch.cat([Disease, Disease_neg], 0)
                Mask = torch.cat([Mask, Mask_neg], 0)
                Age = torch.cat([Age, Age_neg], 0)
                Demo = torch.cat([Demo, Demo_neg], 0)

            nRec += Note.size()[0] - np.sum(Mask.numpy(), axis=0)[0]
            Note, Num, Disease, Mask, Age, Demo = Variable(Note).long(), Variable(Num).float(), Variable(
                Disease).float(), Variable(Mask).float(), Variable(Age).float(), Variable(Demo).float()

            self.cnt_iter += 1

            if self.flg_cuda:
                Note, Num, Disease, Mask, Age, Demo = Note.cuda(), Num.cuda(), Disease.cuda(), Mask.cuda(), Age.cuda(), Demo.cuda()

            self.optimizer.zero_grad()
            Disease = Disease.squeeze(1)
            Mask = Mask.squeeze(1)
            output = self.model(Note, Num, Disease, Mask, Age, Demo)
            output = output * (1.0 - Mask)
            Disease = Disease * (1.0 - Mask)
            loss = self.criterion(output, Disease)

            if self.alpha_L1 > 0:
                l1_crit = nn.L1Loss(size_average=False)
                for fc in self.model.FCs:
                    if self.flg_cuda:
                        target_reg = Variable(torch.zeros(fc.weight.size())).cuda()
                    else:
                        target_reg = Variable(torch.zeros(fc.weight.size()))
                    loss += l1_crit(fc.weight, target_reg) * self.alpha_L1

            loss.backward()
            if self.flg_gradClip:
                torch.nn.utils.clip_grad_norm(self.model.params, 0.5)
            self.optimizer.step()

            self.Y_multi_train.append(output.data.cpu().numpy())
            self.target_multi_train.append(Disease.data.cpu().numpy())
            self.mask_train.append(Mask.data.cpu().numpy())

            correct += self._getAccuracy(output, Disease, Mask)
            train_loss += loss.data[0]
            j += 1
            if (j % self.log_interval[1] == 0):
                train_loss_temp = train_loss / np.sum(nRec)
                print('Train Epoch: {}, Batch: {}, Loss: {:.4f}'.format(epoch, j, train_loss_temp))

        if (epoch == 0) | (epoch % self.log_interval[0] == 0) | (epoch == self.n_iter - 1):
            trainAccuracy = 100. * correct / nRec
            train_loss /= np.sum(nRec)
            lsTrainAccuracy.append(trainAccuracy)

            # print('\nTrain Epoch: {} Loss: {:.4f}, Accuracy: {}, {}, {}/{} ({:.2f}%, {:.2f}%, {:.2f}%)'.format(epoch, train_loss, correct[0], correct[1], correct[2], nRec, trainAccuracy[0], trainAccuracy[1], trainAccuracy[2]))
            print('\nTrain Epoch: {} Loss: {:.4f}'.format(epoch, train_loss))

            auc, f1, tp, p, tn, n = self._aucAll(self.Y_multi_train, self.target_multi_train, self.mask_train)
            for i in range(len(auc)):
                print(str(i) + ' Train set: Accuracy: pos_{}/{}, neg_{}/{} ({:.2f}%, AUC: {:.4f}, F1: {:.4f}%)'.format(
                    tp[i], p[i], tn[i], n[i], trainAccuracy[i], auc[i], f1[i] * 100))

    def _test(self, epoch, lsTestAccuracy):
        if (epoch == 0) | (epoch % self.log_interval[0] == 0) | (epoch == self.n_iter - 1):

            self.model.eval()
            test_loss = 0
            correct, nRec = np.zeros(3), np.zeros(3)

            self.Y_multi = []
            self.target_multi = []
            self.mask = []

            for batch_idx, sample in enumerate(self.test_loader):
                Note, Num, Disease, Mask, Age, Demo = sample['Note'], sample['Num'], sample['Disease'], sample['Mask'], \
                                                      sample['Age'], sample['Demo']
                nRec += Note.size()[0] - np.sum(Mask.numpy(), axis=0)[0]

                Note, Num, Disease, Mask, Age, Demo = Variable(Note, volatile=True).long(), Variable(Num,
                                                                                volatile=True).float(), Variable(
                    Disease, volatile=True).float(), \
                                                      Variable(Mask, volatile=True).float(), Variable(Age,
                                                                                volatile=True).float(), Variable(
                    Demo, volatile=True).float()
                if self.flg_cuda:
                    Note, Num, Disease, Mask, Age, Demo = Note.cuda(), Num.cuda(), Disease.cuda(), Mask.cuda(), Age.cuda(), Demo.cuda()
                with torch.no_grad():
                    output = self.model(Note, Num, Disease, Mask, Age, Demo)
                    Disease = Disease.squeeze(1)
                    Mask = Mask.squeeze(1)
                    output = output * (1.0 - Mask)
                    Disease = Disease * (1.0 - Mask)

                    test_loss += (self.criterion(output, Disease)).data[0]
                    correct += self._getAccuracy(output, Disease, Mask)

                self.Y_multi.append(output.data.cpu().numpy())
                self.target_multi.append(Disease.data.cpu().numpy())
                self.mask.append(Mask.data.cpu().numpy())

            testAccuracy = 100. * correct / nRec
            test_loss /= np.sum(nRec)  # loss function already averages over batch size
            auc, f1, tp, p, tn, n = self._aucAll(self.Y_multi, self.target_multi, self.mask)
            self.auc = np.mean(auc)
            print('Test set: Average loss: {:.4f}'.format(test_loss))
            for i in range(len(auc)):
                print(str(i) + ' Test set: Accuracy: pos_{}/{}, neg_{}/{} ({:.2f}%, AUC: {:.4f}, F1: {:.4f}%)'.format(
                    tp[i], p[i], tn[i], n[i], testAccuracy[i], auc[i], f1[i] * 100))
            lsTestAccuracy.append([testAccuracy, auc, f1])

    def _getAccuracy(self, output, target, mask):
        pred = (output.data > 0.5).float()
        accuracy = (pred.eq(target.data).cpu().float() * (1.0 - mask.data.cpu())).numpy()
        accuracy = np.sum(accuracy, axis=0)
        return accuracy

    def _aucAll(self, Y_hat, Y, mask):
        # Multi-task, inputs are multiple dimensions
        Y_hat = np.vstack(Y_hat)
        Y = np.vstack(Y)
        mask = np.vstack(mask)
        auc, f1, tp, p, tn, n = [], [], [], [], [], []
        for i in range(Y_hat.shape[1]):
            result = self._getAucF1(Y_hat[:, i], Y[:, i], mask[:, i])
            for j in range(len(result)):
                [auc, f1, tp, p, tn, n][j].append(result[j])
        return auc, f1, tp, p, tn, n

    def _getAucF1(self, Y_hat, Y, mask):
        # Single task
        Y_hat = Y_hat[mask == 0]
        Y = Y[mask == 0]
        tp = sum((Y == 1) & (Y_hat >= self.posThres))
        tn = sum((Y == 0) & (Y_hat < self.posThres))
        p = sum(Y == 1)
        n = sum(Y == 0)
        if len(np.unique(Y)) == 1:
            auc = 0
            f1 = 0
        else:
            auc = metrics.roc_auc_score(Y, Y_hat)
            f1 = metrics.f1_score(Y, 1 * (Y_hat > self.posThres))
        return auc, f1, tp, p, tn, n

    def _saveModel(self):
        torch.save(self.model, self.savePath + '_model.pt')

    def _drawAUC(self, Y_multi, target_multi, mask):
        Y_multi = np.vstack(Y_multi)
        target_multi = np.vstack(target_multi)
        mask = np.vstack(mask)
        pickle.dump([Y_multi, target_multi, mask], open(self.savePath + '_pred.p', 'wb'))
        """
        # Uncomment if matplotlib is available
        fpr, tpr, thresholds = metrics.roc_curve(Y, Y_hat, pos_label=1)
        prec, recall, thresholds = metrics.precision_recall_curve(Y, Y_hat, pos_label= 1)

        # Print ROC curve
        pp = PdfPages(self.savePath + '_plots.pdf')

        figure1 = plt.figure(figsize=(16, 6))
        gs1 = gridspec.GridSpec(1, 2)

        ax1 = figure1.add_subplot(gs1[0])
        ax1.plot(fpr, tpr, 'g-')
        ax1.plot([0, 1], [0, 1], ls="--")
        ax1.set_xlim([0, 1])
        ax1.set_ylim([0, 1])
        ax1.set_ylabel('True positive rate', fontsize=12)
        ax1.set_xlabel('False positive rate', fontsize=12)
        ax1.set_title('ROC curve', fontsize=16)

        ax2 = figure1.add_subplot(gs1[1])
        ax2.plot(prec, recall, 'b-')
        ax2.plot([0, 1], [0, 1], ls="--")
        ax2.set_xlim([0, 1])
        ax2.set_ylim([0, 1])
        ax2.set_ylabel('Recall', fontsize=12)
        ax2.set_xlabel('Precision', fontsize=12)
        ax2.set_title('Precision-recall curve', fontsize=16)

        pp.savefig(figure1)
        plt.close(figure1)
        pp.close()
        """

{'repo_name': 'NYUMedML/DeepEHR', 'stars': '195', 'repo_language': 'Python', 'file_name': 'file.py', 'mime_type': 'text/x-python', 'hash': -3761104089548515460, 'source_dataset': 'data'}


<?xml version="1.0" ?>
<sdf version="1.4">
  <world name="default">
    <include><uri>model://camera</uri></include>

    <model name="heightmap">
      <static>true</static>
      <link name="link">
        <collision name="collision">
          <geometry>
            <heightmap>
              <uri>file://media/materials/textures/heightmap_bowl.png</uri>
              <size>129 129 10</size>
              <pos>0 0 0</pos>
            </heightmap>
          </geometry>
        </collision>

        <visual name="visual">
          <geometry>
            <heightmap>
              <texture>
                <diffuse>file://media/materials/textures/dirt_diffusespecular.png</diffuse>
                <normal>file://media/materials/textures/flat_normal.png</normal>
                <size>50</size>
              </texture>
              <texture>
                <diffuse>file://media/materials/textures/grass_diffusespecular.png</diffuse>
                <normal>file://media/materials/textures/flat_normal.png</normal>
                <size>20</size>
              </texture>
              <texture>
                <diffuse>file://media/materials/textures/fungus_diffusespecular.png</diffuse>
                <normal>file://media/materials/textures/flat_normal.png</normal>
                <size>80</size>
              </texture>
              <blend>
                <min_height>2</min_height>
                <fade_dist>5</fade_dist>
              </blend>
              <blend>
                <min_height>4</min_height>
                <fade_dist>5</fade_dist>
              </blend>
              <uri>file://media/materials/textures/heightmap_bowl.png</uri>
              <size>129 129 10</size>
              <pos>0 0 0</pos>
            </heightmap>
          </geometry>
        </visual>
      </link>
    </model>
  </world>
</sdf>

{'repo_name': 'osrf/gazebo', 'stars': '144', 'repo_language': 'C++', 'file_name': 'RestUiWidget.cc', 'mime_type': 'text/x-c', 'hash': 5015869575374310969, 'source_dataset': 'data'}


/*
 * jvmti_agent.c: JVMTI agent interface
 *
 * Adapted from the Oprofile code in opagent.c:
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 *
 * Copyright 2007 OProfile authors
 * Jens Wilke
 * Daniel Hansel
 * Copyright IBM Corporation 2007
 */
#include <sys/types.h>
#include <sys/stat.h> /* for mkdir() */
#include <stdio.h>
#include <errno.h>
#include <string.h>
#include <stdlib.h>
#include <stdint.h>
#include <limits.h>
#include <fcntl.h>
#include <unistd.h>
#include <time.h>
#include <sys/mman.h>
#include <syscall.h> /* for gettid() */
#include <err.h>

#include "jvmti_agent.h"
#include "../util/jitdump.h"

#define JIT_LANG "java"

static char jit_path[PATH_MAX];
static void *marker_addr;

static inline pid_t gettid(void)
{
	return (pid_t)syscall(__NR_gettid);
}

static int get_e_machine(struct jitheader *hdr)
{
	ssize_t sret;
	char id[16];
	int fd, ret = -1;
	struct {
		uint16_t e_type;
		uint16_t e_machine;
	} info;

	fd = open("/proc/self/exe", O_RDONLY);
	if (fd == -1)
		return -1;

	sret = read(fd, id, sizeof(id));
	if (sret != sizeof(id))
		goto error;

	/* check ELF signature */
	if (id[0] != 0x7f || id[1] != 'E' || id[2] != 'L' || id[3] != 'F')
		goto error;

	sret = read(fd, &info, sizeof(info));
	if (sret != sizeof(info))
		goto error;

	hdr->elf_mach = info.e_machine;
	ret = 0;
error:
	close(fd);
	return ret;
}

static int use_arch_timestamp;

static inline uint64_t
get_arch_timestamp(void)
{
#if defined(__i386__) || defined(__x86_64__)
	unsigned int low, high;

	asm volatile("rdtsc" : "=a" (low), "=d" (high));

	return low | ((uint64_t)high) << 32;
#else
	return 0;
#endif
}

#define NSEC_PER_SEC	1000000000
static int perf_clk_id = CLOCK_MONOTONIC;

static inline uint64_t
timespec_to_ns(const struct timespec *ts)
{
        return ((uint64_t) ts->tv_sec * NSEC_PER_SEC) + ts->tv_nsec;
}

static inline uint64_t
perf_get_timestamp(void)
{
	struct timespec ts;
	int ret;

	if (use_arch_timestamp)
		return get_arch_timestamp();

	ret = clock_gettime(perf_clk_id, &ts);
	if (ret)
		return 0;

	return timespec_to_ns(&ts);
}

static int
debug_cache_init(void)
{
	char str[32];
	char *base, *p;
	struct tm tm;
	time_t t;
	int ret;

	time(&t);
	localtime_r(&t, &tm);

	base = getenv("JITDUMPDIR");
	if (!base)
		base = getenv("HOME");
	if (!base)
		base = ".";

	strftime(str, sizeof(str), JIT_LANG"-jit-%Y%m%d", &tm);

	snprintf(jit_path, PATH_MAX - 1, "%s/.debug/", base);

	ret = mkdir(jit_path, 0755);
	if (ret == -1) {
		if (errno != EEXIST) {
			warn("jvmti: cannot create jit cache dir %s", jit_path);
			return -1;
		}
	}

	snprintf(jit_path, PATH_MAX - 1, "%s/.debug/jit", base);
	ret = mkdir(jit_path, 0755);
	if (ret == -1) {
		if (errno != EEXIST) {
			warn("cannot create jit cache dir %s", jit_path);
			return -1;
		}
	}

	snprintf(jit_path, PATH_MAX - 1, "%s/.debug/jit/%s.XXXXXXXX", base, str);

	p = mkdtemp(jit_path);
	if (p != jit_path) {
		warn("cannot create jit cache dir %s", jit_path);
		return -1;
	}

	return 0;
}

static int
perf_open_marker_file(int fd)
{
	long pgsz;

	pgsz = sysconf(_SC_PAGESIZE);
	if (pgsz == -1)
		return -1;

	/*
	 * we mmap the jitdump to create an MMAP RECORD in perf.data file.
	 * The mmap is captured either live (perf record running when we mmap)
	 * or  in deferred mode, via /proc/PID/maps
	 * the MMAP record is used as a marker of a jitdump file for more meta
	 * data info about the jitted code. Perf report/annotate detect this
	 * special filename and process the jitdump file.
	 *
	 * mapping must be PROT_EXEC to ensure it is captured by perf record
	 * even when not using -d option
	 */
	marker_addr = mmap(NULL, pgsz, PROT_READ|PROT_EXEC, MAP_PRIVATE, fd, 0);
	return (marker_addr == MAP_FAILED) ? -1 : 0;
}

static void
perf_close_marker_file(void)
{
	long pgsz;

	if (!marker_addr)
		return;

	pgsz = sysconf(_SC_PAGESIZE);
	if (pgsz == -1)
		return;

	munmap(marker_addr, pgsz);
}

static void
init_arch_timestamp(void)
{
	char *str = getenv("JITDUMP_USE_ARCH_TIMESTAMP");

	if (!str || !*str || !strcmp(str, "0"))
		return;

	use_arch_timestamp = 1;
}

void *jvmti_open(void)
{
	char dump_path[PATH_MAX];
	struct jitheader header;
	int fd;
	FILE *fp;

	init_arch_timestamp();

	/*
	 * check if clockid is supported
	 */
	if (!perf_get_timestamp()) {
		if (use_arch_timestamp)
			warnx("jvmti: arch timestamp not supported");
		else
			warnx("jvmti: kernel does not support %d clock id", perf_clk_id);
	}

	memset(&header, 0, sizeof(header));

	debug_cache_init();

	/*
	 * jitdump file name
	 */
	snprintf(dump_path, PATH_MAX, "%s/jit-%i.dump", jit_path, getpid());

	fd = open(dump_path, O_CREAT|O_TRUNC|O_RDWR, 0666);
	if (fd == -1)
		return NULL;

	/*
	 * create perf.data maker for the jitdump file
	 */
	if (perf_open_marker_file(fd)) {
		warnx("jvmti: failed to create marker file");
		return NULL;
	}

	fp = fdopen(fd, "w+");
	if (!fp) {
		warn("jvmti: cannot create %s", dump_path);
		close(fd);
		goto error;
	}

	warnx("jvmti: jitdump in %s", dump_path);

	if (get_e_machine(&header)) {
		warn("get_e_machine failed\n");
		goto error;
	}

	header.magic      = JITHEADER_MAGIC;
	header.version    = JITHEADER_VERSION;
	header.total_size = sizeof(header);
	header.pid        = getpid();

	header.timestamp = perf_get_timestamp();

	if (use_arch_timestamp)
		header.flags |= JITDUMP_FLAGS_ARCH_TIMESTAMP;

	if (!fwrite(&header, sizeof(header), 1, fp)) {
		warn("jvmti: cannot write dumpfile header");
		goto error;
	}
	return fp;
error:
	fclose(fp);
	return NULL;
}

int
jvmti_close(void *agent)
{
	struct jr_code_close rec;
	FILE *fp = agent;

	if (!fp) {
		warnx("jvmti: incalid fd in close_agent");
		return -1;
	}

	rec.p.id = JIT_CODE_CLOSE;
	rec.p.total_size = sizeof(rec);

	rec.p.timestamp = perf_get_timestamp();

	if (!fwrite(&rec, sizeof(rec), 1, fp))
		return -1;

	fclose(fp);

	fp = NULL;

	perf_close_marker_file();

	return 0;
}

int
jvmti_write_code(void *agent, char const *sym,
	uint64_t vma, void const *code, unsigned int const size)
{
	static int code_generation = 1;
	struct jr_code_load rec;
	size_t sym_len;
	FILE *fp = agent;
	int ret = -1;

	/* don't care about 0 length function, no samples */
	if (size == 0)
		return 0;

	if (!fp) {
		warnx("jvmti: invalid fd in write_native_code");
		return -1;
	}

	sym_len = strlen(sym) + 1;

	rec.p.id           = JIT_CODE_LOAD;
	rec.p.total_size   = sizeof(rec) + sym_len;
	rec.p.timestamp    = perf_get_timestamp();

	rec.code_size  = size;
	rec.vma        = vma;
	rec.code_addr  = vma;
	rec.pid	       = getpid();
	rec.tid	       = gettid();

	if (code)
		rec.p.total_size += size;

	/*
	 * If JVM is multi-threaded, nultiple concurrent calls to agent
	 * may be possible, so protect file writes
	 */
	flockfile(fp);

	/*
	 * get code index inside lock to avoid race condition
	 */
	rec.code_index = code_generation++;

	ret = fwrite_unlocked(&rec, sizeof(rec), 1, fp);
	fwrite_unlocked(sym, sym_len, 1, fp);

	if (code)
		fwrite_unlocked(code, size, 1, fp);

	funlockfile(fp);

	ret = 0;

	return ret;
}

int
jvmti_write_debug_info(void *agent, uint64_t code, const char *file,
		       jvmti_line_info_t *li, int nr_lines)
{
	struct jr_code_debug_info rec;
	size_t sret, len, size, flen;
	uint64_t addr;
	const char *fn = file;
	FILE *fp = agent;
	int i;

	/*
	 * no entry to write
	 */
	if (!nr_lines)
		return 0;

	if (!fp) {
		warnx("jvmti: invalid fd in write_debug_info");
		return -1;
	}

	flen = strlen(file) + 1;

	rec.p.id        = JIT_CODE_DEBUG_INFO;
	size            = sizeof(rec);
	rec.p.timestamp = perf_get_timestamp();
	rec.code_addr   = (uint64_t)(uintptr_t)code;
	rec.nr_entry    = nr_lines;

	/*
	 * on disk source line info layout:
	 * uint64_t : addr
	 * int      : line number
	 * int      : column discriminator
	 * file[]   : source file name
	 */
	size += nr_lines * sizeof(struct debug_entry);
	size += flen * nr_lines;
	rec.p.total_size = size;

	/*
	 * If JVM is multi-threaded, nultiple concurrent calls to agent
	 * may be possible, so protect file writes
	 */
	flockfile(fp);

	sret = fwrite_unlocked(&rec, sizeof(rec), 1, fp);
	if (sret != 1)
		goto error;

	for (i = 0; i < nr_lines; i++) {

		addr = (uint64_t)li[i].pc;
		len  = sizeof(addr);
		sret = fwrite_unlocked(&addr, len, 1, fp);
		if (sret != 1)
			goto error;

		len  = sizeof(li[0].line_number);
		sret = fwrite_unlocked(&li[i].line_number, len, 1, fp);
		if (sret != 1)
			goto error;

		len  = sizeof(li[0].discrim);
		sret = fwrite_unlocked(&li[i].discrim, len, 1, fp);
		if (sret != 1)
			goto error;

		sret = fwrite_unlocked(fn, flen, 1, fp);
		if (sret != 1)
			goto error;
	}
	funlockfile(fp);
	return 0;
error:
	funlockfile(fp);
	return -1;
}

{'repo_name': 'ValveSoftware/steamos_kernel', 'stars': '292', 'repo_language': 'C', 'file_name': 'Kconfig', 'mime_type': 'text/plain', 'hash': -8873586715931933496, 'source_dataset': 'data'}


Python $B$N3+H/$O!"(B1990 $BG/$4$m$+$i3+;O$5$l$F$$$^$9!#(B
$B3+H/<T$N(B Guido van Rossum $B$O650iMQ$N%W%m%0%i%_%s%08@8l!V(BABC$B!W$N3+H/$K;22C$7$F$$$^$7$?$,!"(BABC $B$O<BMQ>e$NL\E*$K$O$"$^$jE,$7$F$$$^$;$s$G$7$?!#(B
$B$3$N$?$a!"(BGuido $B$O$h$j<BMQE*$J%W%m%0%i%_%s%08@8l$N3+H/$r3+;O$7!"1Q9q(B BBS $BJ|Aw$N%3%a%G%#HVAH!V%b%s%F%#(B $B%Q%$%=%s!W$N%U%!%s$G$"$k(B Guido $B$O$3$N8@8l$r!V(BPython$B!W$HL>$E$1$^$7$?!#(B
$B$3$N$h$&$JGX7J$+$i@8$^$l$?(B Python $B$N8@8l@_7W$O!"!V%7%s%W%k!W$G!V=,F@$,MF0W!W$H$$$&L\I8$K=EE@$,CV$+$l$F$$$^$9!#(B
$BB?$/$N%9%/%j%W%H7O8@8l$G$O%f!<%6$NL\@h$NMxJX@-$rM%@h$7$F?'!9$J5!G=$r8@8lMWAG$H$7$F<h$jF~$l$k>l9g$,B?$$$N$G$9$,!"(BPython $B$G$O$=$&$$$C$?>.:Y9)$,DI2C$5$l$k$3$H$O$"$^$j$"$j$^$;$s!#(B
$B8@8l<+BN$N5!G=$O:G>.8B$K2!$5$(!"I,MW$J5!G=$O3HD%%b%8%e!<%k$H$7$FDI2C$9$k!"$H$$$&$N$,(B Python $B$N%]%j%7!<$G$9!#(B

{'repo_name': 'Acmesec/CTFCrackTools', 'stars': '942', 'repo_language': 'Python', 'file_name': 'CODE_OF_CONDUCT.md', 'mime_type': 'text/plain', 'hash': 6365618424326550717, 'source_dataset': 'data'}


/*
* Copyright (c) 2018 naehrwert
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#ifndef _SDMMC_T210_H_
#define _SDMMC_T210_H_

#include "types.h"

#define TEGRA_MMC_PWRCTL_SD_BUS_POWER 0x1
#define TEGRA_MMC_PWRCTL_SD_BUS_VOLTAGE_V1_8 0xA
#define TEGRA_MMC_PWRCTL_SD_BUS_VOLTAGE_V3_0 0xC
#define TEGRA_MMC_PWRCTL_SD_BUS_VOLTAGE_V3_3 0xE
#define TEGRA_MMC_PWRCTL_SD_BUS_VOLTAGE_MASK 0xF1

#define TEGRA_MMC_HOSTCTL_1BIT 0x00
#define TEGRA_MMC_HOSTCTL_4BIT 0x02
#define TEGRA_MMC_HOSTCTL_8BIT 0x20

#define TEGRA_MMC_CLKCON_INTERNAL_CLOCK_ENABLE 0x1
#define TEGRA_MMC_CLKCON_INTERNAL_CLOCK_STABLE 0x2
#define TEGRA_MMC_CLKCON_SD_CLOCK_ENABLE 0x4
#define TEGRA_MMC_CLKCON_CLKGEN_SELECT 0x20

#define TEGRA_MMC_SWRST_SW_RESET_FOR_ALL 0x1
#define TEGRA_MMC_SWRST_SW_RESET_FOR_CMD_LINE 0x2
#define TEGRA_MMC_SWRST_SW_RESET_FOR_DAT_LINE 0x4

#define TEGRA_MMC_TRNMOD_DMA_ENABLE 0x1
#define TEGRA_MMC_TRNMOD_BLOCK_COUNT_ENABLE 0x2
#define TEGRA_MMC_TRNMOD_AUTO_CMD12 0x4
#define TEGRA_MMC_TRNMOD_DATA_XFER_DIR_SEL_WRITE 0x0
#define TEGRA_MMC_TRNMOD_DATA_XFER_DIR_SEL_READ 0x10
#define TEGRA_MMC_TRNMOD_MULTI_BLOCK_SELECT 0x20

#define TEGRA_MMC_TRNMOD_CMD_CRC_CHECK 0x8
#define TEGRA_MMC_TRNMOD_CMD_INDEX_CHECK 0x10
#define TEGRA_MMC_TRNMOD_DATA_PRESENT_SELECT_DATA_TRANSFER 0x20

#define TEGRA_MMC_CMDREG_RESP_TYPE_SELECT_MASK 0x3
#define TEGRA_MMC_CMDREG_RESP_TYPE_SELECT_NO_RESPONSE 0x0
#define TEGRA_MMC_CMDREG_RESP_TYPE_SELECT_LENGTH_136 0x1
#define TEGRA_MMC_CMDREG_RESP_TYPE_SELECT_LENGTH_48 0x2
#define TEGRA_MMC_CMDREG_RESP_TYPE_SELECT_LENGTH_48_BUSY 0x3

#define TEGRA_MMC_NORINTSTS_CMD_COMPLETE 0x1
#define TEGRA_MMC_NORINTSTS_XFER_COMPLETE 0x2
#define TEGRA_MMC_NORINTSTS_DMA_INTERRUPT 0x8
#define TEGRA_MMC_NORINTSTS_ERR_INTERRUPT 0x8000
#define TEGRA_MMC_NORINTSTS_CMD_TIMEOUT 0x10000

#define TEGRA_MMC_NORINTSTSEN_BUFFER_READ_READY 0x20

typedef struct _t210_sdmmc_t
{
	vu32 sysad;
	vu16 blksize;
	vu16 blkcnt;
	vu32 argument;
	vu16 trnmod;
	vu16 cmdreg;
	vu32 rspreg0;
	vu32 rspreg1;
	vu32 rspreg2;
	vu32 rspreg3;
	vu32 bdata;
	vu32 prnsts;
	vu8 hostctl;
	vu8 pwrcon;
	vu8 blkgap;
	vu8 wakcon;
	vu16 clkcon;
	vu8 timeoutcon;
	vu8 swrst;
	vu16 norintsts;
	vu16 errintsts;
	vu16 norintstsen;
	vu16 errintstsen;
	vu16 norintsigen;
	vu16 errintsigen;
	vu16 acmd12errsts;
	vu16 hostctl2;
	vu32 capareg;
	vu32 capareg_1;
	vu32 maxcurr;
	vu8 res3[4];
	vu16 setacmd12err;
	vu16 setinterr;
	vu8 admaerr;
	vu8 res4[3];
	vu32 admaaddr;
	vu32 admaaddr_hi;
	vu8 res5[156];
	vu16 slotintstatus;
	vu16 hcver;
	vu32 venclkctl;
	vu32 venspictl;
	vu32 venspiintsts;
	vu32 venceatactl;
	vu32 venbootctl;
	vu32 venbootacktout;
	vu32 venbootdattout;
	vu32 vendebouncecnt;
	vu32 venmiscctl;
	vu32 res6[34];
	vu32 field_1AC;
	vu32 field_1B0;
	vu8 res7[8];
	vu32 field_1BC;
	vu32 field_1C0;
	vu32 field_1C4;
	vu8 field_1C8[24];
	vu32 sdmemcmppadctl;
	vu32 autocalcfg;
	vu32 autocalintval;
	vu32 autocalsts;
	vu32 field_1F0;
} t210_sdmmc_t;

#endif

{'repo_name': 'Jan4V/hekate-ipl-50x', 'stars': '125', 'repo_language': 'C', 'file_name': 'hos.h', 'mime_type': 'text/x-c', 'hash': -3378772563399985508, 'source_dataset': 'data'}