<commit_before>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"

	"golang.org/x/net/context"
)

type (
	CommandConfig struct {
		Template []string            `json:"-"`
		Options  map[string][]string `json:"options,omitempty"`
		Dryrun   bool                `json:"dryrun,omitempty"`
	}

	Job struct {
		config *CommandConfig
		// https://godoc.org/google.golang.org/genproto/googleapis/pubsub/v1#ReceivedMessage
		message      *JobMessage
		notification *ProgressNotification
		storage      Storage

		// These are set at at setupWorkspace
		workspace     string
		downloads_dir string
		uploads_dir   string

		// These are set at setupDownloadFiles
		downloadFileMap     map[string]string
		remoteDownloadFiles interface{}
		localDownloadFiles  interface{}

		cmd *exec.Cmd
	}
)

func (job *Job) run(ctx context.Context) error {
	err := job.runImpl(ctx)
	switch err.(type) {
	case InvalidJobError:
		err := job.withNotify(CANCELLING, job.message.Ack)()
		if err != nil {
			return err
		}
	}
	return err
}

func (job *Job) runImpl(ctx context.Context) error {
	job.notification.notify(PROCESSING, job.message.MessageId(), "info")

	err := job.message.Validate()
	if err != nil {
		log.Printf("Invalid Message: MessageId: %v, Message: %v, error: %v\n", job.message.MessageId(), job.message.raw.Message, err)
		return nil
	}

	defer job.withNotify(CLEANUP, job.clearWorkspace)() // Call clearWorkspace even if setupWorkspace retuns error
	err = job.setupWorkspace()
	if err != nil {
		return err
	}

	err = job.withNotify(PREPARING, job.setupDownloadFiles)()
	if err != nil {
		return err
	}

	err = job.build()
	if err != nil {
		log.Fatalf("Command build Error template: %v msg: %v cause of %v\n", job.config.Template, job.message, err)
		return err
	}

	go job.message.sendMADPeriodically()
	defer job.message.Done()

	err = job.withNotify(DOWNLOADING, job.downloadFiles)()
	if err != nil {
		return err
	}

	err = job.withNotify(EXECUTING, job.execute)()
	if err != nil {
		return err
	}

	err = job.withNotify(UPLOADING, job.uploadFiles)()
	if err != nil {
		return err
	}

	err = job.withNotify(ACKSENDING, job.message.Ack)()
	if err != nil {
		return err
	}

	return err
}

func (job *Job) withNotify(progress int, f func() error) func() error {
	msg_id := job.message.MessageId()
	return func() error {
		job.notification.notify(progress, msg_id, "info")
		err := f()
		if err != nil {
			job.notification.notify(progress+2, msg_id, "error")
			return err
		}
		job.notification.notify(progress+1, msg_id, "info")
		return nil
	}
}

func (job *Job) setupWorkspace() error {
	dir, err := ioutil.TempDir("", "workspace")
	if err != nil {
		log.Fatal(err)
		return err
	}

	subdirs := []string{
		filepath.Join(dir, "downloads"),
		filepath.Join(dir, "uploads"),
	}
	for _, subdir := range subdirs {
		err := os.MkdirAll(subdir, 0700)
		if err != nil {
			return err
		}
	}
	job.workspace = dir
	job.downloads_dir = subdirs[0]
	job.uploads_dir = subdirs[1]
	return nil
}

func (job *Job) clearWorkspace() error {
	if job.workspace != "" {
		return os.RemoveAll(job.workspace)
	}
	return nil
}

func (job *Job) setupDownloadFiles() error {
	job.downloadFileMap = map[string]string{}
	job.remoteDownloadFiles = job.message.DownloadFiles()
	objects := job.flatten(job.remoteDownloadFiles)
	remoteUrls := []string{}
	for _, obj := range objects {
		switch obj.(type) {
		case string:
			remoteUrls = append(remoteUrls, obj.(string))
		default:
			log.Printf("Invalid download file URL: %v [%T]", obj, obj)
		}
	}
	for _, remote_url := range remoteUrls {
		url, err := url.Parse(remote_url)
		if err != nil {
			log.Fatalf("Invalid URL: %v because of %v\n", remote_url, err)
			return err
		}
		urlstr := fmt.Sprintf("gs://%v%v", url.Host, url.Path)
		destPath := filepath.Join(job.downloads_dir, url.Host, url.Path)
		job.downloadFileMap[urlstr] = destPath
	}
	job.localDownloadFiles = job.copyWithFileMap(job.remoteDownloadFiles)
	return nil
}

func (job *Job) copyWithFileMap(obj interface{}) interface{} {
	switch obj.(type) {
	case map[string]interface{}:
		result := map[string]interface{}{}
		for k, v := range obj.(map[string]interface{}) {
			result[k] = job.copyWithFileMap(v)
		}
		return result
	case []interface{}:
		result := []interface{}{}
		for _, v := range obj.([]interface{}) {
			result = append(result, job.copyWithFileMap(v))
		}
		return result
	case string:
		return job.downloadFileMap[obj.(string)]
	default:
		return obj
	}
}

func (job *Job) buildVariable() *Variable {
	return &Variable{
		data: map[string]interface{}{
			"workspace":             job.workspace,
			"downloads_dir":         job.downloads_dir,
			"uploads_dir":           job.uploads_dir,
			"download_files":        job.localDownloadFiles,
			"local_download_files":  job.localDownloadFiles,
			"remote_download_files": job.remoteDownloadFiles,
			"attrs":                 job.message.raw.Message.Attributes,
			"attributes":            job.message.raw.Message.Attributes,
			"data":                  job.message.raw.Message.Data,
		},
	}
}

func (job *Job) build() error {
	v := job.buildVariable()

	values, err := job.extract(v, job.config.Template)
	if err != nil {
		return err
	}
	if len(job.config.Options) > 0 {
		key := strings.Join(values, " ")
		t := job.config.Options[key]
		if t == nil {
			t = job.config.Options["default"]
		}
		if t != nil {
			values, err = job.extract(v, t)
			if err != nil {
				return err
			}
		}
	}
	job.cmd = exec.Command(values[0], values[1:]...)
	return nil
}

func (job *Job) extract(v *Variable, values []string) ([]string, error) {
	result := []string{}
	for _, src := range values {
		extracted, err := v.expand(src)
		if err != nil {
			return nil, &InvalidJobError{err.Error()}
		}
		vals := strings.Split(extracted, v.separator)
		for _, val := range vals {
			result = append(result, val)
		}
	}
	return result, nil
}

func (job *Job) downloadFiles() error {
	for remoteURL, destPath := range job.downloadFileMap {
		url, err := url.Parse(remoteURL)
		if err != nil {
			log.Fatalf("Invalid URL: %v because of %v\n", remoteURL, err)
			return err
		}

		dir := path.Dir(destPath)
		err = os.MkdirAll(dir, 0700)
		if err != nil {
			return err
		}

		err = job.storage.Download(url.Host, url.Path[1:], destPath)
		if err != nil {
			return err
		}
	}
	return nil
}

func (job *Job) execute() error {
	var out bytes.Buffer
	job.cmd.Stdout = &out
	job.cmd.Stderr = &out
	log.Printf("EXECUTE running: %v\n", job.cmd)
	err := job.cmd.Run()
	if err != nil {
		log.Printf("Command Error: cmd: %v cause of %v\n%v\n", job.cmd, err, out.String())
		return err
	}
	return nil
}

func (job *Job) uploadFiles() error {
	localPaths, err := job.listFiles(job.uploads_dir)
	if err != nil {
		return err
	}
	for _, localPath := range localPaths {
		relPath, err := filepath.Rel(job.uploads_dir, localPath)
		if err != nil {
			log.Fatalf("Error getting relative path of %v: %v\n", localPath, err)
			return err
		}
		sep := string([]rune{os.PathSeparator})
		parts := strings.Split(relPath, sep)
		bucket := parts[0]
		object := strings.Join(parts[1:], sep)
		err = job.storage.Upload(bucket, object, localPath)
		if err != nil {
			log.Fatalf("Error uploading %v to gs://%v/%v: %v\n", localPath, bucket, object, err)
			return err
		}
	}
	return nil
}

func (job *Job) listFiles(dir string) ([]string, error) {
	result := []string{}
	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			result = append(result, path)
		}
		return nil
	})
	if err != nil {
		log.Fatalf("Error listing upload files: %v\n", err)
		return nil, err
	}
	return result, nil
}

func (job *Job) flatten(obj interface{}) []interface{} {
	// Support only unmarshalled object from JSON
	// See https://golang.org/pkg/encoding/json/#Unmarshal also
	switch obj.(type) {
	case []interface{}:
		res := []interface{}{}
		for _, i := range obj.([]interface{}) {
			switch i.(type) {
			case bool, float64, string, nil:
				res = append(res, i)
			default:
				for _, j := range job.flatten(i) {
					res = append(res, j)
				}
			}
		}
		return res
	case map[string]interface{}:
		values := []interface{}{}
		for _, val := range obj.(map[string]interface{}) {
			values = append(values, val)
		}
		return job.flatten(values)
	default:
		return []interface{}{obj}
	}
}
<commit_msg>:+1: Always call clearWorkspace on runImpl<commit_after>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"

	"golang.org/x/net/context"
)

type (
	CommandConfig struct {
		Template []string            `json:"-"`
		Options  map[string][]string `json:"options,omitempty"`
		Dryrun   bool                `json:"dryrun,omitempty"`
	}

	Job struct {
		config *CommandConfig
		// https://godoc.org/google.golang.org/genproto/googleapis/pubsub/v1#ReceivedMessage
		message      *JobMessage
		notification *ProgressNotification
		storage      Storage

		// These are set at at setupWorkspace
		workspace     string
		downloads_dir string
		uploads_dir   string

		// These are set at setupDownloadFiles
		downloadFileMap     map[string]string
		remoteDownloadFiles interface{}
		localDownloadFiles  interface{}

		cmd *exec.Cmd
	}
)

func (job *Job) run(ctx context.Context) error {
	err := job.runImpl(ctx)
	switch err.(type) {
	case InvalidJobError:
		err := job.withNotify(CANCELLING, job.message.Ack)()
		if err != nil {
			return err
		}
	}
	return err
}

func (job *Job) runImpl(ctx context.Context) error {
	job.notification.notify(PROCESSING, job.message.MessageId(), "info")
	defer job.withNotify(CLEANUP, job.clearWorkspace)() // Call clearWorkspace even if setupWorkspace retuns error

	err := job.message.Validate()
	if err != nil {
		log.Printf("Invalid Message: MessageId: %v, Message: %v, error: %v\n", job.message.MessageId(), job.message.raw.Message, err)
		return nil
	}

	err = job.setupWorkspace()
	if err != nil {
		return err
	}

	err = job.withNotify(PREPARING, job.setupDownloadFiles)()
	if err != nil {
		return err
	}

	err = job.build()
	if err != nil {
		log.Fatalf("Command build Error template: %v msg: %v cause of %v\n", job.config.Template, job.message, err)
		return err
	}

	go job.message.sendMADPeriodically()
	defer job.message.Done()

	err = job.withNotify(DOWNLOADING, job.downloadFiles)()
	if err != nil {
		return err
	}

	err = job.withNotify(EXECUTING, job.execute)()
	if err != nil {
		return err
	}

	err = job.withNotify(UPLOADING, job.uploadFiles)()
	if err != nil {
		return err
	}

	err = job.withNotify(ACKSENDING, job.message.Ack)()
	if err != nil {
		return err
	}

	return err
}

func (job *Job) withNotify(progress int, f func() error) func() error {
	msg_id := job.message.MessageId()
	return func() error {
		job.notification.notify(progress, msg_id, "info")
		err := f()
		if err != nil {
			job.notification.notify(progress+2, msg_id, "error")
			return err
		}
		job.notification.notify(progress+1, msg_id, "info")
		return nil
	}
}

func (job *Job) setupWorkspace() error {
	dir, err := ioutil.TempDir("", "workspace")
	if err != nil {
		log.Fatal(err)
		return err
	}

	subdirs := []string{
		filepath.Join(dir, "downloads"),
		filepath.Join(dir, "uploads"),
	}
	for _, subdir := range subdirs {
		err := os.MkdirAll(subdir, 0700)
		if err != nil {
			return err
		}
	}
	job.workspace = dir
	job.downloads_dir = subdirs[0]
	job.uploads_dir = subdirs[1]
	return nil
}

func (job *Job) clearWorkspace() error {
	if job.workspace != "" {
		return os.RemoveAll(job.workspace)
	}
	return nil
}

func (job *Job) setupDownloadFiles() error {
	job.downloadFileMap = map[string]string{}
	job.remoteDownloadFiles = job.message.DownloadFiles()
	objects := job.flatten(job.remoteDownloadFiles)
	remoteUrls := []string{}
	for _, obj := range objects {
		switch obj.(type) {
		case string:
			remoteUrls = append(remoteUrls, obj.(string))
		default:
			log.Printf("Invalid download file URL: %v [%T]", obj, obj)
		}
	}
	for _, remote_url := range remoteUrls {
		url, err := url.Parse(remote_url)
		if err != nil {
			log.Fatalf("Invalid URL: %v because of %v\n", remote_url, err)
			return err
		}
		urlstr := fmt.Sprintf("gs://%v%v", url.Host, url.Path)
		destPath := filepath.Join(job.downloads_dir, url.Host, url.Path)
		job.downloadFileMap[urlstr] = destPath
	}
	job.localDownloadFiles = job.copyWithFileMap(job.remoteDownloadFiles)
	return nil
}

func (job *Job) copyWithFileMap(obj interface{}) interface{} {
	switch obj.(type) {
	case map[string]interface{}:
		result := map[string]interface{}{}
		for k, v := range obj.(map[string]interface{}) {
			result[k] = job.copyWithFileMap(v)
		}
		return result
	case []interface{}:
		result := []interface{}{}
		for _, v := range obj.([]interface{}) {
			result = append(result, job.copyWithFileMap(v))
		}
		return result
	case string:
		return job.downloadFileMap[obj.(string)]
	default:
		return obj
	}
}

func (job *Job) buildVariable() *Variable {
	return &Variable{
		data: map[string]interface{}{
			"workspace":             job.workspace,
			"downloads_dir":         job.downloads_dir,
			"uploads_dir":           job.uploads_dir,
			"download_files":        job.localDownloadFiles,
			"local_download_files":  job.localDownloadFiles,
			"remote_download_files": job.remoteDownloadFiles,
			"attrs":                 job.message.raw.Message.Attributes,
			"attributes":            job.message.raw.Message.Attributes,
			"data":                  job.message.raw.Message.Data,
		},
	}
}

func (job *Job) build() error {
	v := job.buildVariable()

	values, err := job.extract(v, job.config.Template)
	if err != nil {
		return err
	}
	if len(job.config.Options) > 0 {
		key := strings.Join(values, " ")
		t := job.config.Options[key]
		if t == nil {
			t = job.config.Options["default"]
		}
		if t != nil {
			values, err = job.extract(v, t)
			if err != nil {
				return err
			}
		}
	}
	job.cmd = exec.Command(values[0], values[1:]...)
	return nil
}

func (job *Job) extract(v *Variable, values []string) ([]string, error) {
	result := []string{}
	for _, src := range values {
		extracted, err := v.expand(src)
		if err != nil {
			return nil, &InvalidJobError{err.Error()}
		}
		vals := strings.Split(extracted, v.separator)
		for _, val := range vals {
			result = append(result, val)
		}
	}
	return result, nil
}

func (job *Job) downloadFiles() error {
	for remoteURL, destPath := range job.downloadFileMap {
		url, err := url.Parse(remoteURL)
		if err != nil {
			log.Fatalf("Invalid URL: %v because of %v\n", remoteURL, err)
			return err
		}

		dir := path.Dir(destPath)
		err = os.MkdirAll(dir, 0700)
		if err != nil {
			return err
		}

		err = job.storage.Download(url.Host, url.Path[1:], destPath)
		if err != nil {
			return err
		}
	}
	return nil
}

func (job *Job) execute() error {
	var out bytes.Buffer
	job.cmd.Stdout = &out
	job.cmd.Stderr = &out
	log.Printf("EXECUTE running: %v\n", job.cmd)
	err := job.cmd.Run()
	if err != nil {
		log.Printf("Command Error: cmd: %v cause of %v\n%v\n", job.cmd, err, out.String())
		return err
	}
	return nil
}

func (job *Job) uploadFiles() error {
	localPaths, err := job.listFiles(job.uploads_dir)
	if err != nil {
		return err
	}
	for _, localPath := range localPaths {
		relPath, err := filepath.Rel(job.uploads_dir, localPath)
		if err != nil {
			log.Fatalf("Error getting relative path of %v: %v\n", localPath, err)
			return err
		}
		sep := string([]rune{os.PathSeparator})
		parts := strings.Split(relPath, sep)
		bucket := parts[0]
		object := strings.Join(parts[1:], sep)
		err = job.storage.Upload(bucket, object, localPath)
		if err != nil {
			log.Fatalf("Error uploading %v to gs://%v/%v: %v\n", localPath, bucket, object, err)
			return err
		}
	}
	return nil
}

func (job *Job) listFiles(dir string) ([]string, error) {
	result := []string{}
	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			result = append(result, path)
		}
		return nil
	})
	if err != nil {
		log.Fatalf("Error listing upload files: %v\n", err)
		return nil, err
	}
	return result, nil
}

func (job *Job) flatten(obj interface{}) []interface{} {
	// Support only unmarshalled object from JSON
	// See https://golang.org/pkg/encoding/json/#Unmarshal also
	switch obj.(type) {
	case []interface{}:
		res := []interface{}{}
		for _, i := range obj.([]interface{}) {
			switch i.(type) {
			case bool, float64, string, nil:
				res = append(res, i)
			default:
				for _, j := range job.flatten(i) {
					res = append(res, j)
				}
			}
		}
		return res
	case map[string]interface{}:
		values := []interface{}{}
		for _, val := range obj.(map[string]interface{}) {
			values = append(values, val)
		}
		return job.flatten(values)
	default:
		return []interface{}{obj}
	}
}
<|endoftext|>


<commit_before>package common

import (
	"time"
)

type AccessType int

const (
	DB_BOTH AccessType = iota
	DB_PRIVATE
	DB_PUBLIC
)

type ForkType int

const (
	SPACE ForkType = iota
	ROOT
	STEM
	BRANCH
	END
)

type LicenseType int

const (
	// From http://opendefinition.org/licenses/
	CC0 LicenseType = iota
	PDDL
	CCBY
	ODCBY
	CCBYSA
	ODbL
	CCA
	CCSA
	DLDEBY
	DLDE0
	DSL
	FAL
	GNUFDL
	MIROSL
	OGLC
	OGLUK
	NONE
	OTHER
)

type ValType int

const (
	Binary ValType = iota
	Image
	Null
	Text
	Integer
	Float
)

// Stored cached data in memcache for 1/2 hour by default
const CacheTime = 1800

// ************************
// Configuration file types

// Configuration file
type TomlConfig struct {
	Admin AdminInfo
	Auth0 Auth0Info
	Cache CacheInfo
	DB4S  DB4SInfo
	Minio MinioInfo
	Pg    PGInfo
	Sign  SigningInfo
	Web   WebInfo
}

// Config info for the admin server
type AdminInfo struct {
	Server         string
	HTTPS          bool
	Certificate    string
	CertificateKey string `toml:"certificate_key"`
}

// Auth0 connection parameters
type Auth0Info struct {
	ClientID     string
	ClientSecret string
	Domain       string
}

// Memcached connection parameters
type CacheInfo struct {
	Server string
}

// Configuration info for the DB4S end point
type DB4SInfo struct {
	CAChain        string `toml:"ca_chain"`
	Certificate    string
	CertificateKey string `toml:"certificate_key"`
	Port           int
	Server         string
}

// Minio connection parameters
type MinioInfo struct {
	Server    string
	AccessKey string `toml:"access_key"`
	Secret    string
	HTTPS     bool
}

// PostgreSQL connection parameters
type PGInfo struct {
	Server   string
	Port     int
	Username string
	Password string
	Database string
}

// Used for signing DB4S client certificates
type SigningInfo struct {
	IntermediateCert string `toml:"intermediate_cert"`
	IntermediateKey  string `toml:"intermediate_key"`
}

type WebInfo struct {
	BindAddress    string `toml:"bind_address"`
	Certificate    string
	CertificateKey string `toml:"certificate_key"`
	RequestLog     string `toml:"request_log"`
	ServerName     string `toml:"server_name"`
}

// End of configuration file types
// *******************************

type Auth0Set struct {
	CallbackURL string
	ClientID    string
	Domain      string
}

type DataValue struct {
	Name  string
	Type  ValType
	Value interface{}
}
type DataRow []DataValue

type DBEntry struct {
	Owner     string
	Folder    string
	DBName    string
	DateEntry time.Time
}

type DBInfo struct {
	Database     string
	Tables       []string
	Watchers     int
	Stars        int
	Forks        int
	Discussions  int
	MRs          int
	Description  string
	Updates      int
	Branches     int
	Releases     int
	Contributors int
	Readme       string
	DateCreated  time.Time
	LastModified time.Time
	Public       bool
	Size         int
	Version      int
	Folder       string
	License      LicenseType
	DefaultTable string
}

type ForkEntry struct {
	Owner      string
	Folder     string
	DBName     string
	ID         int
	IconList   []ForkType
	ForkedFrom int
	Processed  bool
}

type MetaInfo struct {
	Protocol     string
	Server       string
	Title        string
	Owner        string
	Database     string
	LoggedInUser string
	ForkOwner    string
	ForkFolder   string
	ForkDatabase string
}

type SQLiteDBinfo struct {
	Info     DBInfo
	MaxRows  int
	MinioBkt string
	MinioId  string
}

type SQLiteRecordSet struct {
	Tablename string
	ColNames  []string
	ColCount  int
	RowCount  int
	TotalRows int
	Records   []DataRow
}

type WhereClause struct {
	Column string
	Type   string
	Value  string
}

type UserInfo struct {
	Username     string
	LastModified time.Time
}

type UserDetails struct {
	Username   string
	Email      string
	Password   string
	PVerify    string
	DateJoined time.Time
	ClientCert []byte
	PHash      []byte
}
<commit_msg>Change default memcache ttl to be 1 day<commit_after>package common

import (
	"time"
)

type AccessType int

const (
	DB_BOTH AccessType = iota
	DB_PRIVATE
	DB_PUBLIC
)

type ForkType int

const (
	SPACE ForkType = iota
	ROOT
	STEM
	BRANCH
	END
)

type LicenseType int

const (
	// From http://opendefinition.org/licenses/
	CC0 LicenseType = iota
	PDDL
	CCBY
	ODCBY
	CCBYSA
	ODbL
	CCA
	CCSA
	DLDEBY
	DLDE0
	DSL
	FAL
	GNUFDL
	MIROSL
	OGLC
	OGLUK
	NONE
	OTHER
)

type ValType int

const (
	Binary ValType = iota
	Image
	Null
	Text
	Integer
	Float
)

// Stored cached data in memcache for 1 full day (as a first guess, which will probably need training)
const CacheTime = 86400

// ************************
// Configuration file types

// Configuration file
type TomlConfig struct {
	Admin AdminInfo
	Auth0 Auth0Info
	Cache CacheInfo
	DB4S  DB4SInfo
	Minio MinioInfo
	Pg    PGInfo
	Sign  SigningInfo
	Web   WebInfo
}

// Config info for the admin server
type AdminInfo struct {
	Server         string
	HTTPS          bool
	Certificate    string
	CertificateKey string `toml:"certificate_key"`
}

// Auth0 connection parameters
type Auth0Info struct {
	ClientID     string
	ClientSecret string
	Domain       string
}

// Memcached connection parameters
type CacheInfo struct {
	Server string
}

// Configuration info for the DB4S end point
type DB4SInfo struct {
	CAChain        string `toml:"ca_chain"`
	Certificate    string
	CertificateKey string `toml:"certificate_key"`
	Port           int
	Server         string
}

// Minio connection parameters
type MinioInfo struct {
	Server    string
	AccessKey string `toml:"access_key"`
	Secret    string
	HTTPS     bool
}

// PostgreSQL connection parameters
type PGInfo struct {
	Server   string
	Port     int
	Username string
	Password string
	Database string
}

// Used for signing DB4S client certificates
type SigningInfo struct {
	IntermediateCert string `toml:"intermediate_cert"`
	IntermediateKey  string `toml:"intermediate_key"`
}

type WebInfo struct {
	BindAddress    string `toml:"bind_address"`
	Certificate    string
	CertificateKey string `toml:"certificate_key"`
	RequestLog     string `toml:"request_log"`
	ServerName     string `toml:"server_name"`
}

// End of configuration file types
// *******************************

type Auth0Set struct {
	CallbackURL string
	ClientID    string
	Domain      string
}

type DataValue struct {
	Name  string
	Type  ValType
	Value interface{}
}
type DataRow []DataValue

type DBEntry struct {
	Owner     string
	Folder    string
	DBName    string
	DateEntry time.Time
}

type DBInfo struct {
	Database     string
	Tables       []string
	Watchers     int
	Stars        int
	Forks        int
	Discussions  int
	MRs          int
	Description  string
	Updates      int
	Branches     int
	Releases     int
	Contributors int
	Readme       string
	DateCreated  time.Time
	LastModified time.Time
	Public       bool
	Size         int
	Version      int
	Folder       string
	License      LicenseType
	DefaultTable string
}

type ForkEntry struct {
	Owner      string
	Folder     string
	DBName     string
	ID         int
	IconList   []ForkType
	ForkedFrom int
	Processed  bool
}

type MetaInfo struct {
	Protocol     string
	Server       string
	Title        string
	Owner        string
	Database     string
	LoggedInUser string
	ForkOwner    string
	ForkFolder   string
	ForkDatabase string
}

type SQLiteDBinfo struct {
	Info     DBInfo
	MaxRows  int
	MinioBkt string
	MinioId  string
}

type SQLiteRecordSet struct {
	Tablename string
	ColNames  []string
	ColCount  int
	RowCount  int
	TotalRows int
	Records   []DataRow
}

type WhereClause struct {
	Column string
	Type   string
	Value  string
}

type UserInfo struct {
	Username     string
	LastModified time.Time
}

type UserDetails struct {
	Username   string
	Email      string
	Password   string
	PVerify    string
	DateJoined time.Time
	ClientCert []byte
	PHash      []byte
}
<|endoftext|>


<commit_before><commit_msg>Account responses renaming<commit_after><|endoftext|>


<commit_before>package ari

import "fmt"

const (
	// ApplicationKey is the key kind for ARI Application resources.
	ApplicationKey = "application"

	// BridgeKey is the key kind for the ARI Bridge resources.
	BridgeKey = "bridge"

	// ChannelKey is the key kind for the ARI Channel resource
	ChannelKey = "channel"

	// DeviceStateKey is the key kind for the ARI DeviceState resource
	DeviceStateKey = "devicestate"

	// EndpointKey is the key kind for the ARI Endpoint resource
	EndpointKey = "endpoint"

	// LiveRecordingKey is the key kind for the ARI LiveRecording resource
	LiveRecordingKey = "liverecording"

	// LoggingKey is the key kind for the ARI Logging resource
	LoggingKey = "logging"

	// MailboxKey is the key kind for the ARI Mailbox resource
	MailboxKey = "mailbox"

	// ModuleKey is the key kind for the ARI Module resource
	ModuleKey = "module"

	// PlaybackKey is the key kind for the ARI Playback resource
	PlaybackKey = "playback"

	// SoundKey is the key kind for the ARI Sound resource
	SoundKey = "sound"

	// StoredRecordingKey is the key kind for the ARI StoredRecording resource
	StoredRecordingKey = "storedrecording"

	// VariableKey is the key kind for the ARI Asterisk Variable resource
	VariableKey = "variable"
)

// Key identifies a unique resource in the system
type Key struct {
	// Kind indicates the type of resource the Key points to.  e.g., "channel",
	// "bridge", etc.
	Kind string `json:"kind"`

	// ID indicates the unique identifier of the resource
	ID string `json:"id"`

	// Node indicates the unique identifier of the Asterisk node on which the
	// resource exists or will be created
	Node string `json:"node,omitempty"`

	// Dialog indicates a named scope of the resource, for receiving events
	Dialog string `json:"dialog,omitempty"`

	// App indiciates the ARI application that this key is bound to.
	App string `json:"app,omitempty"`
}

// Keys is a list of keys
type Keys []*Key

// Filter filters the key list using the given key type match
func (kx Keys) Filter(mx ...Matcher) (ret Keys) {
	for _, m := range mx {
		for _, k := range kx {
			if m.Match(k) {
				ret = append(ret, k)
			}
		}
	}
	return
}

// Without removes keys that match the given matcher
func (kx Keys) Without(m Matcher) (ret Keys) {
	for _, k := range kx {
		if !m.Match(k) {
			ret = append(ret, k)
		}
	}
	return
}

// A Matcher provides the functionality for matching against a key.
type Matcher interface {
	Match(o *Key) bool
}

// MatchFunc is the functional type alias for providing functional `Matcher` implementations
type MatchFunc func(*Key) bool

// Match invokes the match function given the key
func (mf MatchFunc) Match(o *Key) bool {
	return mf(o)
}

// KeyOptionFunc is a functional argument alias for providing options for ARI keys
type KeyOptionFunc func(Key) Key

// WithDialog sets the given dialog identifier on the key.
func WithDialog(dialog string) KeyOptionFunc {
	return func(key Key) Key {
		key.Dialog = dialog
		return key
	}
}

// WithNode sets the given node identifier on the key.
func WithNode(node string) KeyOptionFunc {
	return func(key Key) Key {
		key.Node = node
		return key
	}
}

// WithApp sets the given node identifier on the key.
func WithApp(app string) KeyOptionFunc {
	return func(key Key) Key {
		key.App = app
		return key
	}
}

// WithLocationOf copies the partial key fields Node, Application, Dialog from the reference key
func WithLocationOf(ref *Key) KeyOptionFunc {
	return func(key Key) Key {
		if ref != nil {
			key.Node = ref.Node
			key.Dialog = ref.Dialog
			key.App = ref.App
		}
		return key
	}
}

// NewKey builds a new key given the kind, identifier, and any optional arguments.
func NewKey(kind string, id string, opts ...KeyOptionFunc) *Key {
	k := Key{
		Kind: kind,
		ID:   id,
	}
	for _, o := range opts {
		k = o(k)
	}

	return &k
}

// AppKey returns a key that is bound to the given application.
func AppKey(app string) *Key {
	return NewKey("", "", WithApp(app))
}

// ConfigID returns the configuration Key ID for the given configuration class, type/kind, and id.
func ConfigID(class, kind, id string) string {
	return fmt.Sprintf("%s/%s/%s", class, kind, id)
}

// EndpointID returns the endpoint Key ID for the given tech and resource
func EndpointID(tech, resource string) string {
	return fmt.Sprintf("%s/%s", tech, resource)
}

// DialogKey returns a key that is bound to the given dialog.
func DialogKey(dialog string) *Key {
	return NewKey("", "", WithDialog(dialog))
}

// NodeKey returns a key that is bound to the given application and node
func NodeKey(app, node string) *Key {
	return NewKey("", "", WithApp(app), WithNode(node))
}

// KindKey returns a key that is bound by a type only
func KindKey(kind string, opts ...KeyOptionFunc) *Key {
	return NewKey(kind, "", opts...)
}

// Match returns true if the given key matches the subject. Empty partial key fields are wildcards.
func (k *Key) Match(o *Key) bool {
	if k == o {
		return true
	}

	if k == nil || o == nil {
		return false
	}

	if k.App != "" && o.App != "" && k.App != o.App {
		return false
	}
	if k.Dialog != "" && o.Dialog != "" && k.Dialog != o.Dialog {
		return false
	}
	if k.Node != "" && o.Node != "" && k.Node != o.Node {
		return false
	}
	if k.Kind != "" && o.Kind != "" && k.Kind != o.Kind {
		return false
	}
	if k.ID != "" && o.ID != "" && k.ID != o.ID {
		return false
	}

	return true
}

// New returns a new key with the location information from the source key.
// This includes the App, the Node, and the Dialog.  the `kind` and `id`
// parameters are optional.  If kind is empty, the resulting key will not be
// typed.  If id is empty, the key will not be unique.
func (k *Key) New(kind, id string) *Key {
	n := NodeKey(k.App, k.Node)
	n.Dialog = k.Dialog
	n.Kind = kind
	n.ID = id

	return n
}

func (k *Key) String() string {
	if k.ID != "" {
		return k.ID
	}

	if k.Dialog != "" {
		return "[" + k.Dialog + "]"
	}

	if k.Node != "" {
		return k.App + "@" + k.Node
	}

	return "emptyKey"
}
<commit_msg>add keys.First()<commit_after>package ari

import "fmt"

const (
	// ApplicationKey is the key kind for ARI Application resources.
	ApplicationKey = "application"

	// BridgeKey is the key kind for the ARI Bridge resources.
	BridgeKey = "bridge"

	// ChannelKey is the key kind for the ARI Channel resource
	ChannelKey = "channel"

	// DeviceStateKey is the key kind for the ARI DeviceState resource
	DeviceStateKey = "devicestate"

	// EndpointKey is the key kind for the ARI Endpoint resource
	EndpointKey = "endpoint"

	// LiveRecordingKey is the key kind for the ARI LiveRecording resource
	LiveRecordingKey = "liverecording"

	// LoggingKey is the key kind for the ARI Logging resource
	LoggingKey = "logging"

	// MailboxKey is the key kind for the ARI Mailbox resource
	MailboxKey = "mailbox"

	// ModuleKey is the key kind for the ARI Module resource
	ModuleKey = "module"

	// PlaybackKey is the key kind for the ARI Playback resource
	PlaybackKey = "playback"

	// SoundKey is the key kind for the ARI Sound resource
	SoundKey = "sound"

	// StoredRecordingKey is the key kind for the ARI StoredRecording resource
	StoredRecordingKey = "storedrecording"

	// VariableKey is the key kind for the ARI Asterisk Variable resource
	VariableKey = "variable"
)

// Key identifies a unique resource in the system
type Key struct {
	// Kind indicates the type of resource the Key points to.  e.g., "channel",
	// "bridge", etc.
	Kind string `json:"kind"`

	// ID indicates the unique identifier of the resource
	ID string `json:"id"`

	// Node indicates the unique identifier of the Asterisk node on which the
	// resource exists or will be created
	Node string `json:"node,omitempty"`

	// Dialog indicates a named scope of the resource, for receiving events
	Dialog string `json:"dialog,omitempty"`

	// App indiciates the ARI application that this key is bound to.
	App string `json:"app,omitempty"`
}

// Keys is a list of keys
type Keys []*Key

// Filter filters the key list using the given key type match
func (kx Keys) Filter(mx ...Matcher) (ret Keys) {
	for _, m := range mx {
		for _, k := range kx {
			if m.Match(k) {
				ret = append(ret, k)
			}
		}
	}
	return
}

// Without removes keys that match the given matcher
func (kx Keys) Without(m Matcher) (ret Keys) {
	for _, k := range kx {
		if !m.Match(k) {
			ret = append(ret, k)
		}
	}
	return
}

// First returns the first key from a list of keys.  It is safe to use on empty lists, in which case, it will return nil.
func (kx *Keys) First() *Key {
	if len(kx) < 1 {
		return nil
	}
	return kx[0]
}

// A Matcher provides the functionality for matching against a key.
type Matcher interface {
	Match(o *Key) bool
}

// MatchFunc is the functional type alias for providing functional `Matcher` implementations
type MatchFunc func(*Key) bool

// Match invokes the match function given the key
func (mf MatchFunc) Match(o *Key) bool {
	return mf(o)
}

// KeyOptionFunc is a functional argument alias for providing options for ARI keys
type KeyOptionFunc func(Key) Key

// WithDialog sets the given dialog identifier on the key.
func WithDialog(dialog string) KeyOptionFunc {
	return func(key Key) Key {
		key.Dialog = dialog
		return key
	}
}

// WithNode sets the given node identifier on the key.
func WithNode(node string) KeyOptionFunc {
	return func(key Key) Key {
		key.Node = node
		return key
	}
}

// WithApp sets the given node identifier on the key.
func WithApp(app string) KeyOptionFunc {
	return func(key Key) Key {
		key.App = app
		return key
	}
}

// WithLocationOf copies the partial key fields Node, Application, Dialog from the reference key
func WithLocationOf(ref *Key) KeyOptionFunc {
	return func(key Key) Key {
		if ref != nil {
			key.Node = ref.Node
			key.Dialog = ref.Dialog
			key.App = ref.App
		}
		return key
	}
}

// NewKey builds a new key given the kind, identifier, and any optional arguments.
func NewKey(kind string, id string, opts ...KeyOptionFunc) *Key {
	k := Key{
		Kind: kind,
		ID:   id,
	}
	for _, o := range opts {
		k = o(k)
	}

	return &k
}

// AppKey returns a key that is bound to the given application.
func AppKey(app string) *Key {
	return NewKey("", "", WithApp(app))
}

// ConfigID returns the configuration Key ID for the given configuration class, type/kind, and id.
func ConfigID(class, kind, id string) string {
	return fmt.Sprintf("%s/%s/%s", class, kind, id)
}

// EndpointID returns the endpoint Key ID for the given tech and resource
func EndpointID(tech, resource string) string {
	return fmt.Sprintf("%s/%s", tech, resource)
}

// DialogKey returns a key that is bound to the given dialog.
func DialogKey(dialog string) *Key {
	return NewKey("", "", WithDialog(dialog))
}

// NodeKey returns a key that is bound to the given application and node
func NodeKey(app, node string) *Key {
	return NewKey("", "", WithApp(app), WithNode(node))
}

// KindKey returns a key that is bound by a type only
func KindKey(kind string, opts ...KeyOptionFunc) *Key {
	return NewKey(kind, "", opts...)
}

// Match returns true if the given key matches the subject. Empty partial key fields are wildcards.
func (k *Key) Match(o *Key) bool {
	if k == o {
		return true
	}

	if k == nil || o == nil {
		return false
	}

	if k.App != "" && o.App != "" && k.App != o.App {
		return false
	}
	if k.Dialog != "" && o.Dialog != "" && k.Dialog != o.Dialog {
		return false
	}
	if k.Node != "" && o.Node != "" && k.Node != o.Node {
		return false
	}
	if k.Kind != "" && o.Kind != "" && k.Kind != o.Kind {
		return false
	}
	if k.ID != "" && o.ID != "" && k.ID != o.ID {
		return false
	}

	return true
}

// New returns a new key with the location information from the source key.
// This includes the App, the Node, and the Dialog.  the `kind` and `id`
// parameters are optional.  If kind is empty, the resulting key will not be
// typed.  If id is empty, the key will not be unique.
func (k *Key) New(kind, id string) *Key {
	n := NodeKey(k.App, k.Node)
	n.Dialog = k.Dialog
	n.Kind = kind
	n.ID = id

	return n
}

func (k *Key) String() string {
	if k.ID != "" {
		return k.ID
	}

	if k.Dialog != "" {
		return "[" + k.Dialog + "]"
	}

	if k.Node != "" {
		return k.App + "@" + k.Node
	}

	return "emptyKey"
}
<|endoftext|>


<commit_before><commit_msg>Added channel for starting/stopping miner<commit_after><|endoftext|>


<commit_before>/*
SPDX-License-Identifier: MIT

Copyright (c) 2017 Thanh Ha

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/

// lhc is a checker to find code files missing license headers.
package main

import (
	"bufio"
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"unicode"
)

var VERSION = "0.1.0"

// check and exit if error.
func check(e error) {
	if e != nil {
		fmt.Println(e)
		os.Exit(1)
	}
}

// fetchLicense from file and return license text.
func fetchLicense(filename string) string {
	file, err := os.Open(filename)
	check(err)
	defer file.Close()

	comment, multilineComment := false, false
	licenseText := ""
	scanner := bufio.NewScanner(file)

	// Read the first 2 bytes to decide if it is a comment string
	b := make([]byte, 2)
	_, err = file.Read(b)
	check(err)
	if isComment(string(b)) {
		comment = true
	}
	file.Seek(0, 0) // Reset so we can read the full file next

	i := 0
	for scanner.Scan() {
		// Read only the first few lines to not read entire code file
		i++
		if i > 50 {
			break
		}

		s := scanner.Text()

		if ignoreComment(s) {
			continue
		}

		if comment == true {
			if strings.HasPrefix(s, "/*") {
				multilineComment = true
			} else if strings.Contains(s, "*/") {
				multilineComment = false
			}

			if !multilineComment && !isComment(s) {
				break
			}

			s = trimComment(s)
		}

		licenseText += s
	}

	return stripSpaces(licenseText)
}

// Check if a string is a comment line.
func isComment(str string) bool {
	if !strings.HasPrefix(str, "#") &&
		!strings.HasPrefix(str, "//") &&
		!strings.HasPrefix(str, "/*") {
		return false
	}

	return true
}

// Ignore certain lines containing key strings
func ignoreComment(str string) bool {
	s := strings.ToUpper(str)
	if strings.HasPrefix(s, "#!") ||
		strings.Contains(s, "COPYRIGHT") ||
		strings.Contains(s, "SPDX-LICENSE-IDENTIFIER") ||
		// License name in LICENSE file but not header
		strings.Contains(s, "MIT LICENSE") {
		return true
	}

	return false
}

// Strip whitespace from string.
func stripSpaces(str string) string {
	return strings.Map(func(r rune) rune {
		if unicode.IsSpace(r) {
			return -1
		}
		return r
	}, str)
}

// Trim the comment prefix from string.
func trimComment(str string) string {
	str = strings.TrimPrefix(str, "#")
	str = strings.TrimPrefix(str, "//")
	str = strings.TrimPrefix(str, "/*")
	str = strings.Split(str, "*/")[0]
	return str
}

// Usage prints a statement to explain how to use this command.
func usage() {
	fmt.Printf("Usage: %s [OPTIONS] [FILE]...\n", os.Args[0])
	fmt.Printf("Compare FILE with an expected license header.\n")
	fmt.Printf("\nOptions:\n")
	flag.PrintDefaults()
}

func main() {
	directoryPtr := flag.String("directory", ".", "Directory to search for files.")
	licensePtr := flag.String("license", "license.txt", "Comma-separated list of license files to compare against.")
	versionPtr := flag.Bool("version", false, "Print version")
	// extensions := flag.String("extensions", false, "Instead of a list of files list of extensions to search")

	flag.Usage = usage
	flag.Parse()

	if *versionPtr {
		fmt.Println("License Checker version", VERSION)
		os.Exit(0)
	}

	fmt.Println("Search Patterns:", flag.Args())

	licenseText := fetchLicense(*licensePtr)

	var checkFiles []string
	for _, p := range flag.Args() {
		f, _ := filepath.Glob(filepath.Join(*directoryPtr, p))
		checkFiles = append(checkFiles, f...)
	}

	for _, f := range checkFiles {
		headerText := fetchLicense(f)
		if licenseText != headerText {
			fmt.Println("✘", f)
		} else {
			fmt.Println("✔", f)
		}
	}
}
<commit_msg>Search directories recursively for files<commit_after>/*
SPDX-License-Identifier: MIT

Copyright (c) 2017 Thanh Ha

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/

// lhc is a checker to find code files missing license headers.
package main

import (
	"bufio"
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"unicode"
)

var VERSION = "0.1.0"

// check and exit if error.
func check(e error) {
	if e != nil {
		fmt.Println(e)
		os.Exit(1)
	}
}

func findFiles(directory string, patterns []string) []string {
	var files []string
	filepath.Walk(directory, func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			for _, p := range patterns {
				f, _ := filepath.Glob(filepath.Join(path, p))
				files = append(files, f...)
			}
		}
		return nil
	})
	return files
}

// fetchLicense from file and return license text.
func fetchLicense(filename string) string {
	file, err := os.Open(filename)
	check(err)
	defer file.Close()

	comment, multilineComment := false, false
	licenseText := ""
	scanner := bufio.NewScanner(file)

	// Read the first 2 bytes to decide if it is a comment string
	b := make([]byte, 2)
	_, err = file.Read(b)
	check(err)
	if isComment(string(b)) {
		comment = true
	}
	file.Seek(0, 0) // Reset so we can read the full file next

	i := 0
	for scanner.Scan() {
		// Read only the first few lines to not read entire code file
		i++
		if i > 50 {
			break
		}

		s := scanner.Text()

		if ignoreComment(s) {
			continue
		}

		if comment == true {
			if strings.HasPrefix(s, "/*") {
				multilineComment = true
			} else if strings.Contains(s, "*/") {
				multilineComment = false
			}

			if !multilineComment && !isComment(s) {
				break
			}

			s = trimComment(s)
		}

		licenseText += s
	}

	return stripSpaces(licenseText)
}

// Check if a string is a comment line.
func isComment(str string) bool {
	if !strings.HasPrefix(str, "#") &&
		!strings.HasPrefix(str, "//") &&
		!strings.HasPrefix(str, "/*") {
		return false
	}

	return true
}

// Ignore certain lines containing key strings
func ignoreComment(str string) bool {
	s := strings.ToUpper(str)
	if strings.HasPrefix(s, "#!") ||
		strings.Contains(s, "COPYRIGHT") ||
		strings.Contains(s, "SPDX-LICENSE-IDENTIFIER") ||
		// License name in LICENSE file but not header
		strings.Contains(s, "MIT LICENSE") {
		return true
	}

	return false
}

// Strip whitespace from string.
func stripSpaces(str string) string {
	return strings.Map(func(r rune) rune {
		if unicode.IsSpace(r) {
			return -1
		}
		return r
	}, str)
}

// Trim the comment prefix from string.
func trimComment(str string) string {
	str = strings.TrimPrefix(str, "#")
	str = strings.TrimPrefix(str, "//")
	str = strings.TrimPrefix(str, "/*")
	str = strings.Split(str, "*/")[0]
	return str
}

// Usage prints a statement to explain how to use this command.
func usage() {
	fmt.Printf("Usage: %s [OPTIONS] [FILE]...\n", os.Args[0])
	fmt.Printf("Compare FILE with an expected license header.\n")
	fmt.Printf("\nOptions:\n")
	flag.PrintDefaults()
}

func main() {
	directoryPtr := flag.String("directory", ".", "Directory to search for files.")
	licensePtr := flag.String("license", "license.txt", "Comma-separated list of license files to compare against.")
	versionPtr := flag.Bool("version", false, "Print version")
	// extensions := flag.String("extensions", false, "Instead of a list of files list of extensions to search")

	flag.Usage = usage
	flag.Parse()

	if *versionPtr {
		fmt.Println("License Checker version", VERSION)
		os.Exit(0)
	}

	fmt.Println("Search Patterns:", flag.Args())

	licenseText := fetchLicense(*licensePtr)
	checkFiles := findFiles(*directoryPtr, flag.Args())

	for _, f := range checkFiles {
		headerText := fetchLicense(f)
		if licenseText != headerText {
			fmt.Println("✘", f)
		} else {
			fmt.Println("✔", f)
		}
	}
}
<|endoftext|>


<commit_before>package parser

import (
	"fmt"
	"net/http"

	"github.com/go-playground/validator"
	"github.com/labstack/echo"
	"github.com/leebenson/conform"
	"github.com/microcosm-cc/bluemonday"
)

type (
	app struct {
		Key     string `json:"key" validate:"required,len=32,alphanum,excludesall= " conform:"trim,lower"`
		Version string `json:"version" validate:"required,gte=1,lte=15,excludesall= " conform:"trim,lower"`
	}

	user struct {
		Name   string `json:"name" validate:"omitempty,gte=3,lte=70" conform:"trim,name"`
		Email  string `json:"email" validate:"omitempty,email,gte=3,lte=100,excludesall= " conform:"trim,email"`
		MiBAID string `json:"mibaId" validate:"omitempty,uuid4,len=36,excludesall= " conform:"trim,lower"`
	}

	platform struct {
		Key     string `json:"key" validate:"required,len=32,alphanum,excludesall= " conform:"trim,lower"`
		Version string `json:"version" validate:"required,gte=1,lte=15,excludesall= " conform:"trim,lower"`
	}

	device struct {
		Name   string  `json:"name" validate:"required,gte=1,lte=30" conform:"trim"`
		Brand  *string `json:"brand" validate:"omitempty,gte=1,lte=30" conform:"trim"`
		Screen screen  `json:"screen" validate:"required"`
	}

	screen struct {
		Width  int  `json:"width" validate:"required,gt=0"`
		Height int  `json:"height" validate:"required,gt=0"`
		PPI    *int `json:"ppi" validate:"omitempty,gt=0"`
	}

	browser struct {
		Name    string `json:"name" validate:"required,gte=1,lte=15" conform:"trim"`
		Version string `json:"version" validate:"required,gte=1,lte=15,excludesall= " conform:"trim,lower"`
	}

	// Request holds the mapped fields from the request's JSON body
	Request struct {
		Rating      int8     `json:"rating" validate:"min=-127,max=127"`
		Description string   `json:"description" validate:"omitempty,gte=3,lte=30" conform:"trim,title"`
		Comment     string   `json:"comment" validate:"omitempty,gte=3,lte=1000" conform:"trim,ucfirst"`
		Range       string   `json:"range" validate:"required,len=32,alphanum,excludesall= " conform:"trim,lower"`
		App         app      `json:"app" validate:"required"`
		Platform    platform `json:"platform" validate:"required"`
		Device      device   `json:"device" validate:"required"`
		User        *user    `json:"user" validate:"omitempty"`
		Browser     *browser `json:"browser" validate:"omitempty"`
	}
)

// Parse parses, scrubs and escapes a request's JSON body and maps it to a struct
func Parse(context echo.Context) (*Request, error) {
	request := new(Request)

	conform.Strings(request)
	escape(request)

	if err := bind(request, context); err != nil {
		return request, err
	}

	if err := validate(request, context); err != nil {
		return request, err
	}

	return request, nil
}

func bind(request *Request, context echo.Context) error {
	if err := context.Bind(request); err != nil {
		errorDescription := err.Error()
		errorMessage := fmt.Sprintf("Error parsing request: %s", errorDescription)
		errorCode := http.StatusBadRequest

		if httpError, ok := err.(*echo.HTTPError); ok {
			if value, isString := httpError.Message.(string); isString {
				errorMessage = value
				errorCode = httpError.Code
			}
		}

		context.Logger().Error("Error binding request: ", errorDescription)

		return echo.NewHTTPError(errorCode, []string{errorMessage})
	}

	return nil
}

func validate(request *Request, context echo.Context) error {
	if errs := context.Validate(request); errs != nil {
		var errorList []string
		var errorMessage = "Error validating request: "

		if _, ok := errs.(*validator.InvalidValidationError); ok {
			context.Logger().Error(errorMessage, errs.Error())

			return echo.NewHTTPError(http.StatusUnprocessableEntity, []string{errs.Error()})
		}

		for _, err := range errs.(validator.ValidationErrors) {
			errorDescription := err.(error).Error()
			errorList = append(errorList, errorDescription)

			context.Logger().Error(errorMessage, errorDescription)
		}

		return echo.NewHTTPError(http.StatusUnprocessableEntity, errorList)
	}

	return nil
}

func escape(request *Request) {
	sanitizer := bluemonday.StrictPolicy()

	request.Comment = sanitizer.Sanitize(request.Comment)
	request.Description = sanitizer.Sanitize(request.Description)
}

func RegisterCustomValidators(validate *validator.Validate) {
	validate.RegisterStructValidation(UserCustomValidator, user{})
}

func UserCustomValidator(sl validator.StructLevel) {
	item := sl.Current().Interface().(user)

	if len(item.Email) == 0 && len(item.MiBAID) == 0 {
		sl.ReportError(item.Email, "Email", "email", "email/mibaid", "")
		sl.ReportError(item.MiBAID, "MiBAID", "baid", "email/mibaid", "")
	}
}
<commit_msg>Removed redundant validations<commit_after>package parser

import (
	"fmt"
	"net/http"

	"github.com/go-playground/validator"
	"github.com/labstack/echo"
	"github.com/leebenson/conform"
	"github.com/microcosm-cc/bluemonday"
)

type (
	app struct {
		Key     string `json:"key" validate:"required,len=32,alphanum" conform:"trim,lower"`
		Version string `json:"version" validate:"required,gte=1,lte=15,excludesall= " conform:"trim,lower"`
	}

	user struct {
		Name   string `json:"name" validate:"omitempty,gte=3,lte=70" conform:"trim,name"`
		Email  string `json:"email" validate:"omitempty,email,gte=3,lte=100,excludesall= " conform:"trim,email"`
		MiBAID string `json:"mibaId" validate:"omitempty,uuid4,len=36,excludesall= " conform:"trim,lower"`
	}

	platform struct {
		Key     string `json:"key" validate:"required,len=32,alphanum" conform:"trim,lower"`
		Version string `json:"version" validate:"required,gte=1,lte=15,excludesall= " conform:"trim,lower"`
	}

	device struct {
		Name   string  `json:"name" validate:"required,gte=1,lte=30" conform:"trim"`
		Brand  *string `json:"brand" validate:"omitempty,gte=1,lte=30" conform:"trim"`
		Screen screen  `json:"screen" validate:"required"`
	}

	screen struct {
		Width  int  `json:"width" validate:"required,gt=0"`
		Height int  `json:"height" validate:"required,gt=0"`
		PPI    *int `json:"ppi" validate:"omitempty,gt=0"`
	}

	browser struct {
		Name    string `json:"name" validate:"required,gte=1,lte=15" conform:"trim"`
		Version string `json:"version" validate:"required,gte=1,lte=15,excludesall= " conform:"trim,lower"`
	}

	// Request holds the mapped fields from the request's JSON body
	Request struct {
		Rating      int8     `json:"rating" validate:"min=-127,max=127"`
		Description string   `json:"description" validate:"omitempty,gte=3,lte=30" conform:"trim,title"`
		Comment     string   `json:"comment" validate:"omitempty,gte=3,lte=1000" conform:"trim,ucfirst"`
		Range       string   `json:"range" validate:"required,len=32,alphanum" conform:"trim,lower"`
		App         app      `json:"app" validate:"required"`
		Platform    platform `json:"platform" validate:"required"`
		Device      device   `json:"device" validate:"required"`
		User        *user    `json:"user" validate:"omitempty"`
		Browser     *browser `json:"browser" validate:"omitempty"`
	}
)

// Parse parses, scrubs and escapes a request's JSON body and maps it to a struct
func Parse(context echo.Context) (*Request, error) {
	request := new(Request)

	conform.Strings(request)
	escape(request)

	if err := bind(request, context); err != nil {
		return request, err
	}

	if err := validate(request, context); err != nil {
		return request, err
	}

	return request, nil
}

func bind(request *Request, context echo.Context) error {
	if err := context.Bind(request); err != nil {
		errorDescription := err.Error()
		errorMessage := fmt.Sprintf("Error parsing request: %s", errorDescription)
		errorCode := http.StatusBadRequest

		if httpError, ok := err.(*echo.HTTPError); ok {
			if value, isString := httpError.Message.(string); isString {
				errorMessage = value
				errorCode = httpError.Code
			}
		}

		context.Logger().Error("Error binding request: ", errorDescription)

		return echo.NewHTTPError(errorCode, []string{errorMessage})
	}

	return nil
}

func validate(request *Request, context echo.Context) error {
	if errs := context.Validate(request); errs != nil {
		var errorList []string
		var errorMessage = "Error validating request: "

		if _, ok := errs.(*validator.InvalidValidationError); ok {
			context.Logger().Error(errorMessage, errs.Error())

			return echo.NewHTTPError(http.StatusUnprocessableEntity, []string{errs.Error()})
		}

		for _, err := range errs.(validator.ValidationErrors) {
			errorDescription := err.(error).Error()
			errorList = append(errorList, errorDescription)

			context.Logger().Error(errorMessage, errorDescription)
		}

		return echo.NewHTTPError(http.StatusUnprocessableEntity, errorList)
	}

	return nil
}

func escape(request *Request) {
	sanitizer := bluemonday.StrictPolicy()

	request.Comment = sanitizer.Sanitize(request.Comment)
	request.Description = sanitizer.Sanitize(request.Description)
}

func RegisterCustomValidators(validate *validator.Validate) {
	validate.RegisterStructValidation(UserCustomValidator, user{})
}

func UserCustomValidator(sl validator.StructLevel) {
	item := sl.Current().Interface().(user)

	if len(item.Email) == 0 && len(item.MiBAID) == 0 {
		sl.ReportError(item.Email, "Email", "email", "email/mibaid", "")
		sl.ReportError(item.MiBAID, "MiBAID", "baid", "email/mibaid", "")
	}
}
<|endoftext|>


<commit_before>package handler

import (
	"net/http"
	"os"
	"strconv"
	"strings"

	"hermes/ratings/parser"
	"hermes/ratings/responses"

	"github.com/go-playground/validator"
	"github.com/labstack/echo"
	"github.com/labstack/echo/middleware"
	"github.com/labstack/gommon/log"
)

type RequestValidator struct {
	validator *validator.Validate
}

func (rv *RequestValidator) Validate(request interface{}) error {
	return rv.validator.Struct(request)
}

func badRequestMiddleware(next echo.HandlerFunc) echo.HandlerFunc {
	return func(context echo.Context) error {
		var message string

		if !hasAcceptHeader(context) {
			message = "Accept header is missing"

			return echo.NewHTTPError(http.StatusBadRequest, []string{message})
		}

		if context.Request().Method == echo.OPTIONS && hasContentTypeHeader(context) {
			message = "OPTIONS requests must have no body"

			return echo.NewHTTPError(http.StatusBadRequest, []string{message})
		}

		if context.Request().Method == echo.POST && !hasContentTypeHeader(context) {
			message = "Content-Type header is missing"

			return echo.NewHTTPError(http.StatusBadRequest, []string{message})
		}

		return next(context)
	}
}

func notAcceptableMiddleware(next echo.HandlerFunc) echo.HandlerFunc {
	return func(context echo.Context) error {
		var message string

		if !isValidAcceptHeader(context) {
			message = "JSON responses must be accepted"

			return echo.NewHTTPError(http.StatusNotAcceptable, []string{message})
		}

		return next(context)
	}
}

func unsupportedMediaTypeMiddleware(next echo.HandlerFunc) echo.HandlerFunc {
	return func(context echo.Context) error {
		var message string

		if context.Request().Method == echo.POST {
			if !isValidContentTypeHeader(context) || !isValidCharacterEncoding(context) {
				message = "Request body must be UTF-8 encoded JSON"

				return echo.NewHTTPError(http.StatusUnsupportedMediaType, []string{message})
			}
		}

		return next(context)
	}
}

func notImplementedMiddleware(next echo.HandlerFunc) echo.HandlerFunc {
	return func(context echo.Context) error {
		if context.Request().Method != echo.POST && context.Request().Method != echo.OPTIONS {
			return echo.NewHTTPError(http.StatusNotImplemented)
		}

		return next(context)
	}
}

func corsMiddleware(next echo.HandlerFunc) echo.HandlerFunc {
	return func(context echo.Context) error {
		context.Response().Header().Set("Access-Control-Allow-Origin", "*")
		context.Response().Header().Set("Access-Control-Allow-Methods", "OPTIONS, POST")
		context.Response().Header().Set("Access-Control-Allow-Headers", "Content-Type, Accept, Authorization")

		return next(context)
	}
}

func hasAcceptHeader(context echo.Context) bool {
	if header := context.Request().Header.Get("Accept"); strings.TrimSpace(header) != "" {
		return true
	}

	return false
}

func isValidAcceptHeader(context echo.Context) bool {
	accepted := "application/json"

	if header := context.Request().Header.Get("Accept"); strings.Contains(strings.ToLower(header), accepted) || header == "*/*" {
		return true
	}

	return false
}

func hasContentTypeHeader(context echo.Context) bool {
	if header := context.Request().Header.Get("Content-Type"); strings.TrimSpace(header) != "" {
		return true
	}

	return false
}

func isValidContentTypeHeader(context echo.Context) bool {
	text := "text/plain"
	json := "application/json"

	if header := context.Request().Header.Get("Content-Type"); strings.Contains(strings.ToLower(header), text) || strings.Contains(strings.ToLower(header), json) {
		return true
	}

	return false
}

func isValidCharacterEncoding(context echo.Context) bool {
	charset := "charset=utf-8"

	if header := context.Request().Header.Get("Content-Type"); strings.HasSuffix(strings.ToLower(header), charset) {
		return true
	}

	return false
}

func Handler(port int, handlers map[string]echo.HandlerFunc) http.Handler {
	e := echo.New()
	validate := validator.New()
	env := os.Getenv("HERMES_RATINGS_ENV")

	parser.RegisterCustomValidators(validate)

	if env == "DEV" {
		e.Logger.SetLevel(log.DEBUG)
		e.Debug = true
	} else {
		e.Pre(middleware.HTTPSRedirect())
		e.Logger.SetLevel(log.ERROR)
	}

	e.Use(middleware.Secure())
	e.Use(middleware.Logger())
	e.Use(middleware.Recover())
	e.Use(middleware.BodyLimit("20K"))
	e.Use(notImplementedMiddleware)
	e.Use(notAcceptableMiddleware)
	e.Use(badRequestMiddleware)
	e.Use(unsupportedMediaTypeMiddleware)
	e.Use(corsMiddleware)

	e.OPTIONS("/", handlers["OptionsRoot"])
	e.OPTIONS("/ratings", handlers["OptionsRatings"])
	e.POST("/ratings", handlers["PostRatings"])

	e.HTTPErrorHandler = responses.ErrorHandler
	e.Validator = &RequestValidator{validator: validate}
	e.Server.Addr = ":" + strconv.Itoa(port)

	return e
}
<commit_msg>Added stats endpoint<commit_after>package handler

import (
	"net/http"
	"os"
	"strconv"
	"strings"

	"hermes/stats/responses"

	"github.com/go-playground/validator"
	"github.com/labstack/echo"
	"github.com/labstack/echo/middleware"
	"github.com/labstack/gommon/log"
)

type RequestValidator struct {
	validator *validator.Validate
}

func (rv *RequestValidator) Validate(request interface{}) error {
	return rv.validator.Struct(request)
}

func badRequestMiddleware(next echo.HandlerFunc) echo.HandlerFunc {
	return func(context echo.Context) error {
		var message string

		if !hasAcceptHeader(context) {
			message = "Accept header is missing"

			return echo.NewHTTPError(http.StatusBadRequest, []string{message})
		}

		if context.Request().Method == echo.OPTIONS && hasContentTypeHeader(context) {
			message = "OPTIONS requests must have no body"

			return echo.NewHTTPError(http.StatusBadRequest, []string{message})
		}

		if context.Request().Method == echo.POST && !hasContentTypeHeader(context) {
			message = "Content-Type header is missing"

			return echo.NewHTTPError(http.StatusBadRequest, []string{message})
		}

		return next(context)
	}
}

func notAcceptableMiddleware(next echo.HandlerFunc) echo.HandlerFunc {
	return func(context echo.Context) error {
		var message string

		if !isValidAcceptHeader(context) {
			message = "JSON responses must be accepted"

			return echo.NewHTTPError(http.StatusNotAcceptable, []string{message})
		}

		return next(context)
	}
}

func unsupportedMediaTypeMiddleware(next echo.HandlerFunc) echo.HandlerFunc {
	return func(context echo.Context) error {
		var message string

		if context.Request().Method == echo.POST {
			if !isValidContentTypeHeader(context) || !isValidCharacterEncoding(context) {
				message = "Request body must be UTF-8 encoded JSON"

				return echo.NewHTTPError(http.StatusUnsupportedMediaType, []string{message})
			}
		}

		return next(context)
	}
}

func notImplementedMiddleware(next echo.HandlerFunc) echo.HandlerFunc {
	return func(context echo.Context) error {
		if context.Request().Method != echo.POST && context.Request().Method != echo.OPTIONS {
			return echo.NewHTTPError(http.StatusNotImplemented)
		}

		return next(context)
	}
}

func corsMiddleware(next echo.HandlerFunc) echo.HandlerFunc {
	return func(context echo.Context) error {
		context.Response().Header().Set("Access-Control-Allow-Origin", "*")
		context.Response().Header().Set("Access-Control-Allow-Methods", "OPTIONS, POST")
		context.Response().Header().Set("Access-Control-Allow-Headers", "Content-Type, Accept, Authorization")

		return next(context)
	}
}

func hasAcceptHeader(context echo.Context) bool {
	if header := context.Request().Header.Get("Accept"); strings.TrimSpace(header) != "" {
		return true
	}

	return false
}

func isValidAcceptHeader(context echo.Context) bool {
	accepted := "application/json"

	if header := context.Request().Header.Get("Accept"); strings.Contains(strings.ToLower(header), accepted) || header == "*/*" {
		return true
	}

	return false
}

func hasContentTypeHeader(context echo.Context) bool {
	if header := context.Request().Header.Get("Content-Type"); strings.TrimSpace(header) != "" {
		return true
	}

	return false
}

func isValidContentTypeHeader(context echo.Context) bool {
	text := "text/plain"
	json := "application/json"

	if header := context.Request().Header.Get("Content-Type"); strings.Contains(strings.ToLower(header), text) || strings.Contains(strings.ToLower(header), json) {
		return true
	}

	return false
}

func isValidCharacterEncoding(context echo.Context) bool {
	charset := "charset=utf-8"

	if header := context.Request().Header.Get("Content-Type"); strings.HasSuffix(strings.ToLower(header), charset) {
		return true
	}

	return false
}

func Handler(port int, handlers map[string]echo.HandlerFunc) http.Handler {
	e := echo.New()
	validate := validator.New()
	env := os.Getenv("HERMES_STATS_ENV")

	if env == "DEV" {
		e.Logger.SetLevel(log.DEBUG)
		e.Debug = true
	} else {
		e.Pre(middleware.HTTPSRedirect())
		e.Logger.SetLevel(log.ERROR)
	}

	e.Use(middleware.Secure())
	e.Use(middleware.Logger())
	e.Use(middleware.Recover())
	e.Use(middleware.BodyLimit("20K"))
	e.Use(notImplementedMiddleware)
	e.Use(notAcceptableMiddleware)
	e.Use(badRequestMiddleware)
	e.Use(unsupportedMediaTypeMiddleware)
	e.Use(corsMiddleware)

	e.POST("/stats", handlers["PostStats"])

	e.HTTPErrorHandler = responses.ErrorHandler
	e.Validator = &RequestValidator{validator: validate}
	e.Server.Addr = ":" + strconv.Itoa(port)

	return e
}
<|endoftext|>


<commit_before>package log

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"runtime"
	"strings"
	"sync"
	"time"
)

// Level identifies the sort of log: error, info, debug, etc.
// Level implements flag.Value interface and can be set via
// user defined flag.
type Level int32

const (
	FATAL Level = iota - 3
	ERROR
	WARNING
	INFO
	DEBUG
	VDEBUG
	VVDEBUG
	VVVDEBUG
)

func (l *Level) Set(v string) error {
	lvlName := strings.ToLower(v)
	for lvl, name := range levelNames {
		if name == lvlName {
			*l = lvl
			return nil
		}
	}

	return fmt.Errorf("log: unknown log level name: %q", v)
}

func (l Level) String() string {
	return levelNames[l]
}

var levelNames = map[Level]string{
	FATAL:    "fatal",
	ERROR:    "error",
	WARNING:  "warning",
	INFO:     "info",
	DEBUG:    "debug",
	VDEBUG:   "vdebug",
	VVDEBUG:  "vvdebug",
	VVVDEBUG: "vvvdebug",
}

type Logger struct {
	mu  sync.Mutex
	lvl Level
	w   io.Writer
}

func New(lvl Level) *Logger {
	return &Logger{lvl: lvl, w: os.Stderr}
}

func (l *Logger) print(lvl Level, args ...interface{}) {
	buf := &bytes.Buffer{}
	fmt.Fprintln(buf, args...)
	l.output(lvl, buf.Bytes())
}

func (l *Logger) printf(lvl Level, format string, args ...interface{}) {
	buf := &bytes.Buffer{}
	fmt.Fprintf(buf, format, args...)

	if buf.Len() == 0 {
		return
	}

	if buf.Bytes()[buf.Len()-1] != '\n' {
		buf.WriteByte('\n')
	}
	l.output(lvl, buf.Bytes())
}

func (l *Logger) output(lvl Level, buf []byte) {
	if lvl > l.lvl {
		return
	}

	h := l.header(lvl)

	l.mu.Lock()
	defer l.mu.Unlock()

	l.w.Write(h)
	l.w.Write(buf)
}

func (l *Logger) header(lvl Level) []byte {
	// header format:
	//  YY/MM/DD HH:MM:SS.UUUUUU file:line] LEVEL:

	t := time.Now()
	year, month, day := t.Date()
	year %= 1000
	hour, minute, second := t.Clock()
	usecond := t.Nanosecond() / 1e3

	_, file, line, ok := runtime.Caller(4)
	if !ok {
		file, line = "???", 0
	} else {
		index := strings.LastIndex(file, "/")
		if index != -1 && index != len(file) {
			file = file[index+1:]
		}
	}

	level := levelNames[lvl]

	// TODO: don't use Sprintf because it's slow.
	format := "%.2d/%.2d/%.2d %.2d:%.2d:%.2d.%.6d %s:%d] %s: "
	buf := &bytes.Buffer{}
	fmt.Fprintf(buf, format, day, month, year, hour, minute, second,
		usecond, file, line, level)

	return buf.Bytes()
}

// VVVDebug logs with VVVDEBUG level.
// Arguments are handled in the manner of fmt.Print.
func (l *Logger) VVVDebug(args ...interface{}) {
	l.print(VVVDEBUG, args...)
}

// VVVDebugf logs with VVVDEBUG level.
// Arguments are handled in the manner of fmt.Printf.
func (l *Logger) VVVDebugf(format string, args ...interface{}) {
	l.printf(VVVDEBUG, format, args...)
}

// VVDebug logs with VVDEBUG level.
// Arguments are handled in the manner of fmt.Print.
func (l *Logger) VVDebug(args ...interface{}) {
	l.print(VVDEBUG, args...)
}

// VVDebugf logs with VVDEBUG level.
// Arguments are handled in the manner of fmt.Printf.
func (l *Logger) VVDebugf(format string, args ...interface{}) {
	l.printf(VVDEBUG, format, args...)
}

// VDebug logs with VDEBUG level.
// Arguments are handled in the manner of fmt.Print.
func (l *Logger) VDebug(args ...interface{}) {
	l.print(VDEBUG, args...)
}

// VDebugf logs with VDEBUG level.
// Arguments are handled in the manner of fmt.Printf.
func (l *Logger) VDebugf(format string, args ...interface{}) {
	l.printf(VDEBUG, format, args...)
}

// Debug logs with DEBUG level.
// Arguments are handled in the manner of fmt.Print.
func (l *Logger) Debug(args ...interface{}) {
	l.print(DEBUG, args...)
}

// Debugf logs with DEBUG level.
// Arguments are handled in the manner of fmt.Printf.
func (l *Logger) Debugf(format string, args ...interface{}) {
	l.printf(DEBUG, format, args...)
}

// Info logs with INFO level.
// Arguments are handled in the manner of fmt.Print.
func (l *Logger) Info(args ...interface{}) {
	l.print(INFO, args...)
}

// Infof logs with INFO level.
// Arguments are handled in the manner of fmt.Printf.
func (l *Logger) Infof(format string, args ...interface{}) {
	l.printf(INFO, format, args...)
}

// Warning logs with WARNING level.
// Arguments are handled in the manner of fmt.Print.
func (l *Logger) Warning(args ...interface{}) {
	l.print(WARNING, args...)
}

// Warningf logs with WARNING level.
// Arguments are handled in the manner of fmt.Printf.
func (l *Logger) Warningf(format string, args ...interface{}) {
	l.printf(WARNING, format, args...)
}

// Error logs with ERROR level.
// Arguments are handled in the manner of fmt.Print.
func (l *Logger) Error(args ...interface{}) {
	l.print(ERROR, args...)
}

// Errorf logs with ERROR level.
// Arguments are handled in the manner of fmt.Printf.
func (l *Logger) Errorf(format string, args ...interface{}) {
	l.printf(ERROR, format, args...)
}

// Fatal logs with FATAL level, including a stack trace
// of all goroutins, than calls os.Exit(1).
// Arguments are handled in the manner of fmt.Print.
func (l *Logger) Fatal(args ...interface{}) {
	l.print(FATAL, args...)
	// TODO: print stack
	os.Exit(1)
}

// Fatalf logs with FATAL level, including a stack trace
// of all goroutins, than calls os.Exit(1).
// Arguments are handled in the manner of fmt.Printf.
func (l *Logger) Fatalf(format string, args ...interface{}) {
	l.printf(FATAL, format, args...)
	// TODO: print stack
	os.Exit(1)
}

// SetLevel sets log level.
// Logs at or above this level go to log writer.
func (l *Logger) SetLevel(lvl Level) {
	l.lvl = lvl
}

var l = New(INFO)

// VVVDebug logs with VVVDEBUG level.
// Arguments are handled in the manner of fmt.Print.
func VVVDebug(args ...interface{}) {
	l.print(VVVDEBUG, args...)
}

// VVVDebugf logs with VVVDEBUG level.
// Arguments are handled in the manner of fmt.Printf.
func VVVDebugf(format string, args ...interface{}) {
	l.printf(VVVDEBUG, format, args...)
}

// VVDebug logs with VVDEBUG level.
// Arguments are handled in the manner of fmt.Print.
func VVDebug(args ...interface{}) {
	l.print(VVDEBUG, args...)
}

// VVDebugf logs with VVDEBUG level.
// Arguments are handled in the manner of fmt.Printf.
func VVDebugf(format string, args ...interface{}) {
	l.printf(VVDEBUG, format, args...)
}

// VDebug logs with VDEBUG level.
// Arguments are handled in the manner of fmt.Print.
func VDebug(args ...interface{}) {
	l.print(VDEBUG, args...)
}

// VDebugf logs with VDEBUG level.
// Arguments are handled in the manner of fmt.Printf.
func VDebugf(format string, args ...interface{}) {
	l.printf(VDEBUG, format, args...)
}

// Debug logs with DEBUG level.
// Arguments are handled in the manner of fmt.Print.
func Debug(args ...interface{}) {
	l.print(DEBUG, args...)
}

// Debugf logs with DEBUG level.
// Arguments are handled in the manner of fmt.Printf.
func Debugf(format string, args ...interface{}) {
	l.printf(DEBUG, format, args...)
}

// Info logs with INFO level.
// Arguments are handled in the manner of fmt.Print.
func Info(args ...interface{}) {
	l.print(INFO, args...)
}

// Infof logs with INFO level.
// Arguments are handled in the manner of fmt.Printf.
func Infof(format string, args ...interface{}) {
	l.printf(INFO, format, args...)
}

// Warning logs with WARNING level.
// Arguments are handled in the manner of fmt.Print.
func Warning(args ...interface{}) {
	l.print(WARNING, args...)
}

// Warningf logs with WARNING level.
// Arguments are handled in the manner of fmt.Printf.
func Warningf(format string, args ...interface{}) {
	l.printf(WARNING, format, args...)
}

// Error logs with ERROR level.
// Arguments are handled in the manner of fmt.Print.
func Error(args ...interface{}) {
	l.print(ERROR, args...)
}

// Errorf logs with ERROR level.
// Arguments are handled in the manner of fmt.Printf.
func Errorf(format string, args ...interface{}) {
	l.printf(ERROR, format, args...)
}

// Fatal logs with FATAL level, including a stack trace
// of all goroutins, than calls os.Exit(1).
// Arguments are handled in the manner of fmt.Print.
func Fatal(args ...interface{}) {
	l.print(FATAL, args...)
	// TODO: print stack
	os.Exit(1)
}

// Fatalf logs with FATAL level, including a stack trace
// of all goroutins, than calls os.Exit(1).
// Arguments are handled in the manner of fmt.Printf.
func Fatalf(format string, args ...interface{}) {
	l.printf(FATAL, format, args...)
	// TODO: print stack
	os.Exit(1)
}

// SetLevel sets log level.
// Logs at or above this level go to log writer.
func SetLevel(lvl Level) {
	l.SetLevel(lvl)
}
<commit_msg>protected SetLevel()<commit_after>package log

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"runtime"
	"strings"
	"sync"
	"time"
)

// Level identifies the sort of log: error, info, debug, etc.
// Level implements flag.Value interface and can be set via
// user defined flag.
type Level int32

const (
	FATAL Level = iota - 3
	ERROR
	WARNING
	INFO
	DEBUG
	VDEBUG
	VVDEBUG
	VVVDEBUG
)

func (l *Level) Set(v string) error {
	lvlName := strings.ToLower(v)
	for lvl, name := range levelNames {
		if name == lvlName {
			*l = lvl
			return nil
		}
	}

	return fmt.Errorf("log: unknown log level name: %q", v)
}

func (l Level) String() string {
	return levelNames[l]
}

var levelNames = map[Level]string{
	FATAL:    "fatal",
	ERROR:    "error",
	WARNING:  "warning",
	INFO:     "info",
	DEBUG:    "debug",
	VDEBUG:   "vdebug",
	VVDEBUG:  "vvdebug",
	VVVDEBUG: "vvvdebug",
}

type Logger struct {
	mu  sync.Mutex
	lvl Level
	w   io.Writer
}

func New(lvl Level) *Logger {
	return &Logger{lvl: lvl, w: os.Stderr}
}

func (l *Logger) print(lvl Level, args ...interface{}) {
	buf := &bytes.Buffer{}
	fmt.Fprintln(buf, args...)
	l.output(lvl, buf.Bytes())
}

func (l *Logger) printf(lvl Level, format string, args ...interface{}) {
	buf := &bytes.Buffer{}
	fmt.Fprintf(buf, format, args...)

	if buf.Len() == 0 {
		return
	}

	if buf.Bytes()[buf.Len()-1] != '\n' {
		buf.WriteByte('\n')
	}
	l.output(lvl, buf.Bytes())
}

func (l *Logger) output(lvl Level, buf []byte) {
	if lvl > l.lvl {
		return
	}

	h := l.header(lvl)

	l.mu.Lock()
	defer l.mu.Unlock()

	l.w.Write(h)
	l.w.Write(buf)
}

func (l *Logger) header(lvl Level) []byte {
	// header format:
	//  YY/MM/DD HH:MM:SS.UUUUUU file:line] LEVEL:

	t := time.Now()
	year, month, day := t.Date()
	year %= 1000
	hour, minute, second := t.Clock()
	usecond := t.Nanosecond() / 1e3

	_, file, line, ok := runtime.Caller(4)
	if !ok {
		file, line = "???", 0
	} else {
		index := strings.LastIndex(file, "/")
		if index != -1 && index != len(file) {
			file = file[index+1:]
		}
	}

	level := levelNames[lvl]

	// TODO: don't use Sprintf because it's slow.
	format := "%.2d/%.2d/%.2d %.2d:%.2d:%.2d.%.6d %s:%d] %s: "
	buf := &bytes.Buffer{}
	fmt.Fprintf(buf, format, day, month, year, hour, minute, second,
		usecond, file, line, level)

	return buf.Bytes()
}

// VVVDebug logs with VVVDEBUG level.
// Arguments are handled in the manner of fmt.Print.
func (l *Logger) VVVDebug(args ...interface{}) {
	l.print(VVVDEBUG, args...)
}

// VVVDebugf logs with VVVDEBUG level.
// Arguments are handled in the manner of fmt.Printf.
func (l *Logger) VVVDebugf(format string, args ...interface{}) {
	l.printf(VVVDEBUG, format, args...)
}

// VVDebug logs with VVDEBUG level.
// Arguments are handled in the manner of fmt.Print.
func (l *Logger) VVDebug(args ...interface{}) {
	l.print(VVDEBUG, args...)
}

// VVDebugf logs with VVDEBUG level.
// Arguments are handled in the manner of fmt.Printf.
func (l *Logger) VVDebugf(format string, args ...interface{}) {
	l.printf(VVDEBUG, format, args...)
}

// VDebug logs with VDEBUG level.
// Arguments are handled in the manner of fmt.Print.
func (l *Logger) VDebug(args ...interface{}) {
	l.print(VDEBUG, args...)
}

// VDebugf logs with VDEBUG level.
// Arguments are handled in the manner of fmt.Printf.
func (l *Logger) VDebugf(format string, args ...interface{}) {
	l.printf(VDEBUG, format, args...)
}

// Debug logs with DEBUG level.
// Arguments are handled in the manner of fmt.Print.
func (l *Logger) Debug(args ...interface{}) {
	l.print(DEBUG, args...)
}

// Debugf logs with DEBUG level.
// Arguments are handled in the manner of fmt.Printf.
func (l *Logger) Debugf(format string, args ...interface{}) {
	l.printf(DEBUG, format, args...)
}

// Info logs with INFO level.
// Arguments are handled in the manner of fmt.Print.
func (l *Logger) Info(args ...interface{}) {
	l.print(INFO, args...)
}

// Infof logs with INFO level.
// Arguments are handled in the manner of fmt.Printf.
func (l *Logger) Infof(format string, args ...interface{}) {
	l.printf(INFO, format, args...)
}

// Warning logs with WARNING level.
// Arguments are handled in the manner of fmt.Print.
func (l *Logger) Warning(args ...interface{}) {
	l.print(WARNING, args...)
}

// Warningf logs with WARNING level.
// Arguments are handled in the manner of fmt.Printf.
func (l *Logger) Warningf(format string, args ...interface{}) {
	l.printf(WARNING, format, args...)
}

// Error logs with ERROR level.
// Arguments are handled in the manner of fmt.Print.
func (l *Logger) Error(args ...interface{}) {
	l.print(ERROR, args...)
}

// Errorf logs with ERROR level.
// Arguments are handled in the manner of fmt.Printf.
func (l *Logger) Errorf(format string, args ...interface{}) {
	l.printf(ERROR, format, args...)
}

// Fatal logs with FATAL level, including a stack trace
// of all goroutins, than calls os.Exit(1).
// Arguments are handled in the manner of fmt.Print.
func (l *Logger) Fatal(args ...interface{}) {
	l.print(FATAL, args...)
	// TODO: print stack
	os.Exit(1)
}

// Fatalf logs with FATAL level, including a stack trace
// of all goroutins, than calls os.Exit(1).
// Arguments are handled in the manner of fmt.Printf.
func (l *Logger) Fatalf(format string, args ...interface{}) {
	l.printf(FATAL, format, args...)
	// TODO: print stack
	os.Exit(1)
}

// SetLevel sets log level.
// Logs at or above this level go to log writer.
func (l *Logger) SetLevel(lvl Level) {
	// TODO: use atomic and change lvl inplace.
	l.mu.Lock()
	defer l.mu.Unlock()
	l.lvl = lvl
}

var l = New(INFO)

// VVVDebug logs with VVVDEBUG level.
// Arguments are handled in the manner of fmt.Print.
func VVVDebug(args ...interface{}) {
	l.print(VVVDEBUG, args...)
}

// VVVDebugf logs with VVVDEBUG level.
// Arguments are handled in the manner of fmt.Printf.
func VVVDebugf(format string, args ...interface{}) {
	l.printf(VVVDEBUG, format, args...)
}

// VVDebug logs with VVDEBUG level.
// Arguments are handled in the manner of fmt.Print.
func VVDebug(args ...interface{}) {
	l.print(VVDEBUG, args...)
}

// VVDebugf logs with VVDEBUG level.
// Arguments are handled in the manner of fmt.Printf.
func VVDebugf(format string, args ...interface{}) {
	l.printf(VVDEBUG, format, args...)
}

// VDebug logs with VDEBUG level.
// Arguments are handled in the manner of fmt.Print.
func VDebug(args ...interface{}) {
	l.print(VDEBUG, args...)
}

// VDebugf logs with VDEBUG level.
// Arguments are handled in the manner of fmt.Printf.
func VDebugf(format string, args ...interface{}) {
	l.printf(VDEBUG, format, args...)
}

// Debug logs with DEBUG level.
// Arguments are handled in the manner of fmt.Print.
func Debug(args ...interface{}) {
	l.print(DEBUG, args...)
}

// Debugf logs with DEBUG level.
// Arguments are handled in the manner of fmt.Printf.
func Debugf(format string, args ...interface{}) {
	l.printf(DEBUG, format, args...)
}

// Info logs with INFO level.
// Arguments are handled in the manner of fmt.Print.
func Info(args ...interface{}) {
	l.print(INFO, args...)
}

// Infof logs with INFO level.
// Arguments are handled in the manner of fmt.Printf.
func Infof(format string, args ...interface{}) {
	l.printf(INFO, format, args...)
}

// Warning logs with WARNING level.
// Arguments are handled in the manner of fmt.Print.
func Warning(args ...interface{}) {
	l.print(WARNING, args...)
}

// Warningf logs with WARNING level.
// Arguments are handled in the manner of fmt.Printf.
func Warningf(format string, args ...interface{}) {
	l.printf(WARNING, format, args...)
}

// Error logs with ERROR level.
// Arguments are handled in the manner of fmt.Print.
func Error(args ...interface{}) {
	l.print(ERROR, args...)
}

// Errorf logs with ERROR level.
// Arguments are handled in the manner of fmt.Printf.
func Errorf(format string, args ...interface{}) {
	l.printf(ERROR, format, args...)
}

// Fatal logs with FATAL level, including a stack trace
// of all goroutins, than calls os.Exit(1).
// Arguments are handled in the manner of fmt.Print.
func Fatal(args ...interface{}) {
	l.print(FATAL, args...)
	// TODO: print stack
	os.Exit(1)
}

// Fatalf logs with FATAL level, including a stack trace
// of all goroutins, than calls os.Exit(1).
// Arguments are handled in the manner of fmt.Printf.
func Fatalf(format string, args ...interface{}) {
	l.printf(FATAL, format, args...)
	// TODO: print stack
	os.Exit(1)
}

// SetLevel sets log level.
// Logs at or above this level go to log writer.
func SetLevel(lvl Level) {
	l.SetLevel(lvl)
}
<|endoftext|>


<commit_before>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package kv

import (
	tikvstore "github.com/pingcap/tidb/store/tikv/kv"
)

// Priority value for transaction priority.TODO:remove it when br is ready
const (
	PriorityNormal = tikvstore.PriorityNormal
	PriorityLow    = tikvstore.PriorityLow
	PriorityHigh   = tikvstore.PriorityHigh
)

// UnionStore is a store that wraps a snapshot for read and a MemBuffer for buffered write.
// Also, it provides some transaction related utilities.
type UnionStore interface {
	Retriever

	// HasPresumeKeyNotExists returns whether the key presumed key not exists error for the lazy check.
	HasPresumeKeyNotExists(k Key) bool
	// UnmarkPresumeKeyNotExists deletes the key presume key not exists error flag for the lazy check.
	UnmarkPresumeKeyNotExists(k Key)

	// SetOption sets an option with a value, when val is nil, uses the default
	// value of this option.
	SetOption(opt int, val interface{})
	// DelOption deletes an option.
	DelOption(opt int)
	// GetOption gets an option.
	GetOption(opt int) interface{}
	// GetMemBuffer return the MemBuffer binding to this unionStore.
	GetMemBuffer() MemBuffer
}

// AssertionType is the type of a assertion.
type AssertionType int

// The AssertionType constants.
const (
	None AssertionType = iota
	Exist
	NotExist
)

// Options is an interface of a set of options. Each option is associated with a value.
type Options interface {
	// Get gets an option value.
	Get(opt int) (v interface{}, ok bool)
}
<commit_msg>kv/union_store:remove some unused type (#23620)<commit_after>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package kv

import (
	tikvstore "github.com/pingcap/tidb/store/tikv/kv"
)

// Priority value for transaction priority.TODO:remove it when br is ready
const (
	PriorityNormal = tikvstore.PriorityNormal
	PriorityLow    = tikvstore.PriorityLow
	PriorityHigh   = tikvstore.PriorityHigh
)

// UnionStore is a store that wraps a snapshot for read and a MemBuffer for buffered write.
// Also, it provides some transaction related utilities.
type UnionStore interface {
	Retriever

	// HasPresumeKeyNotExists returns whether the key presumed key not exists error for the lazy check.
	HasPresumeKeyNotExists(k Key) bool
	// UnmarkPresumeKeyNotExists deletes the key presume key not exists error flag for the lazy check.
	UnmarkPresumeKeyNotExists(k Key)

	// SetOption sets an option with a value, when val is nil, uses the default
	// value of this option.
	SetOption(opt int, val interface{})
	// DelOption deletes an option.
	DelOption(opt int)
	// GetOption gets an option.
	GetOption(opt int) interface{}
	// GetMemBuffer return the MemBuffer binding to this unionStore.
	GetMemBuffer() MemBuffer
}

// AssertionType is the type of a assertion.TODO:remove it when br is ready
type AssertionType int
<|endoftext|>


<commit_before>// This package provides a simple LRU cache. It is based on the
// LRU implementation in groupcache:
// https://github.com/golang/groupcache/tree/master/lru
package lru

import (
	"container/list"
	"errors"
	"sync"
)

// Cache is a thread-safe fixed size LRU cache.
type Cache struct {
	size      int
	evictList *list.List
	items     map[interface{}]*list.Element
	lock      sync.Mutex
}

// entry is used to hold a value in the evictList
type entry struct {
	key   interface{}
	value interface{}
}

// New creates an LRU of the given size
func New(size int) (*Cache, error) {
	if size <= 0 {
		return nil, errors.New("Must provide a positive size")
	}
	c := &Cache{
		size:      size,
		evictList: list.New(),
		items:     make(map[interface{}]*list.Element, size),
	}
	return c, nil
}

// Purge is used to completely clear the cache
func (c *Cache) Purge() {
	c.lock.Lock()
	defer c.lock.Unlock()
	c.evictList = list.New()
	c.items = make(map[interface{}]*list.Element, c.size)
}

// Add adds a value to the cache.
func (c *Cache) Add(key, value interface{}) {
	c.lock.Lock()
	defer c.lock.Unlock()

	// Check for existing item
	if ent, ok := c.items[key]; ok {
		c.evictList.MoveToFront(ent)
		ent.Value.(*entry).value = value
		return
	}

	// Add new item
	ent := &entry{key, value}
	entry := c.evictList.PushFront(ent)
	c.items[key] = entry

	// Verify size not exceeded
	if c.evictList.Len() > c.size {
		c.removeOldest()
	}
}

// Get looks up a key's value from the cache.
func (c *Cache) Get(key interface{}) (value interface{}, ok bool) {
	c.lock.Lock()
	defer c.lock.Unlock()

	if ent, ok := c.items[key]; ok {
		c.evictList.MoveToFront(ent)
		return ent.Value.(*entry).value, true
	}
	return
}

// Remove removes the provided key from the cache.
func (c *Cache) Remove(key interface{}) {
	c.lock.Lock()
	defer c.lock.Unlock()

	if ent, ok := c.items[key]; ok {
		c.removeElement(ent)
	}
}

// RemoveOldest removes the oldest item from the cache.
func (c *Cache) RemoveOldest() {
	c.lock.Lock()
	defer c.lock.Unlock()
	c.removeOldest()
}

// Keys returns a slice of the keys in the cache.
func (c *Cache) Keys() []interface{} {
	c.lock.Lock()
	defer c.lock.Unlock()

	keys := make([]interface{}, len(c.items))
	i := 0
	for k := range c.items {
		keys[i] = k
		i++
	}

	return keys
}

// removeOldest removes the oldest item from the cache.
func (c *Cache) removeOldest() {
	ent := c.evictList.Back()
	if ent != nil {
		c.removeElement(ent)
	}
}

// removeElement is used to remove a given list element from the cache
func (c *Cache) removeElement(e *list.Element) {
	c.evictList.Remove(e)
	kv := e.Value.(*entry)
	delete(c.items, kv.key)
}

// Len returns the number of items in the cache.
func (c *Cache) Len() int {
	c.lock.Lock()
	defer c.lock.Unlock()
	return c.evictList.Len()
}
<commit_msg>Add RWMutex for read-only functions<commit_after>// This package provides a simple LRU cache. It is based on the
// LRU implementation in groupcache:
// https://github.com/golang/groupcache/tree/master/lru
package lru

import (
	"container/list"
	"errors"
	"sync"
)

// Cache is a thread-safe fixed size LRU cache.
type Cache struct {
	size      int
	evictList *list.List
	items     map[interface{}]*list.Element
	lock      sync.RWMutex
}

// entry is used to hold a value in the evictList
type entry struct {
	key   interface{}
	value interface{}
}

// New creates an LRU of the given size
func New(size int) (*Cache, error) {
	if size <= 0 {
		return nil, errors.New("Must provide a positive size")
	}
	c := &Cache{
		size:      size,
		evictList: list.New(),
		items:     make(map[interface{}]*list.Element, size),
	}
	return c, nil
}

// Purge is used to completely clear the cache
func (c *Cache) Purge() {
	c.lock.Lock()
	defer c.lock.Unlock()
	c.evictList = list.New()
	c.items = make(map[interface{}]*list.Element, c.size)
}

// Add adds a value to the cache.
func (c *Cache) Add(key, value interface{}) {
	c.lock.Lock()
	defer c.lock.Unlock()

	// Check for existing item
	if ent, ok := c.items[key]; ok {
		c.evictList.MoveToFront(ent)
		ent.Value.(*entry).value = value
		return
	}

	// Add new item
	ent := &entry{key, value}
	entry := c.evictList.PushFront(ent)
	c.items[key] = entry

	// Verify size not exceeded
	if c.evictList.Len() > c.size {
		c.removeOldest()
	}
}

// Get looks up a key's value from the cache.
func (c *Cache) Get(key interface{}) (value interface{}, ok bool) {
	c.lock.Lock()
	defer c.lock.Unlock()

	if ent, ok := c.items[key]; ok {
		c.evictList.MoveToFront(ent)
		return ent.Value.(*entry).value, true
	}
	return
}

// Remove removes the provided key from the cache.
func (c *Cache) Remove(key interface{}) {
	c.lock.Lock()
	defer c.lock.Unlock()

	if ent, ok := c.items[key]; ok {
		c.removeElement(ent)
	}
}

// RemoveOldest removes the oldest item from the cache.
func (c *Cache) RemoveOldest() {
	c.lock.Lock()
	defer c.lock.Unlock()
	c.removeOldest()
}

// Keys returns a slice of the keys in the cache.
func (c *Cache) Keys() []interface{} {
	c.lock.RLock()
	defer c.lock.RUnlock()

	keys := make([]interface{}, len(c.items))
	i := 0
	for k := range c.items {
		keys[i] = k
		i++
	}

	return keys
}

// Len returns the number of items in the cache.
func (c *Cache) Len() int {
	c.lock.RLock()
	defer c.lock.RUnlock()
	return c.evictList.Len()
}

// removeOldest removes the oldest item from the cache.
func (c *Cache) removeOldest() {
	ent := c.evictList.Back()
	if ent != nil {
		c.removeElement(ent)
	}
}

// removeElement is used to remove a given list element from the cache
func (c *Cache) removeElement(e *list.Element) {
	c.evictList.Remove(e)
	kv := e.Value.(*entry)
	delete(c.items, kv.key)
}
<|endoftext|>


<commit_before>package libp2p

import (
	"context"
	"fmt"
	"regexp"
	"strings"
	"testing"

	crypto "github.com/libp2p/go-libp2p-crypto"
	host "github.com/libp2p/go-libp2p-host"
	pstore "github.com/libp2p/go-libp2p-peerstore"
	"github.com/libp2p/go-tcp-transport"
)

func TestNewHost(t *testing.T) {
	h, err := makeRandomHost(t, 9000)
	if err != nil {
		t.Fatal(err)
	}
	h.Close()
}

func TestBadTransportConstructor(t *testing.T) {
	ctx := context.Background()
	h, err := New(ctx, Transport(func() {}))
	if err == nil {
		h.Close()
		t.Fatal("expected an error")
	}
	if !strings.Contains(err.Error(), "libp2p_test.go") {
		t.Error("expected error to contain debugging info")
	}
}

func TestNoListenAddrs(t *testing.T) {
	ctx := context.Background()
	h, err := New(ctx, NoListenAddrs)
	if err != nil {
		t.Fatal(err)
	}
	defer h.Close()
	if len(h.Addrs()) != 0 {
		t.Fatal("expected no addresses")
	}
}

func TestNoTransports(t *testing.T) {
	ctx := context.Background()
	a, err := New(ctx, NoTransports)
	if err != nil {
		t.Fatal(err)
	}
	defer a.Close()

	b, err := New(ctx, ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))
	if err != nil {
		t.Fatal(err)
	}
	defer b.Close()

	err = a.Connect(ctx, pstore.PeerInfo{
		ID:    b.ID(),
		Addrs: b.Addrs(),
	})
	if err == nil {
		t.Error("dial should have failed as no transports have been configured")
	}
}

func TestInsecure(t *testing.T) {
	ctx := context.Background()
	h, err := New(ctx, NoSecurity)
	if err != nil {
		t.Fatal(err)
	}
	h.Close()
}

func TestDefaultListenAddrs(t *testing.T) {
	ctx := context.Background()

	re := regexp.MustCompile("/(ip)[4|6]/((0.0.0.0)|(::))/tcp/")
	re2 := regexp.MustCompile("/p2p-circuit")

	// Test 1: Setting the correct listen addresses if userDefined.Transport == nil && userDefined.ListenAddrs == nil
	h, err := New(ctx)
	if err != nil {
		t.Fatal(err)
	}
	for _, addr := range h.Network().ListenAddresses() {
		if re.FindStringSubmatchIndex(addr.String()) == nil &&
			re2.FindStringSubmatchIndex(addr.String()) == nil {
			t.Error("expected ip4 or ip6 or relay interface")
		}
	}

	h.Close()

	// Test 2: Listen addr only include relay if user defined transport is passed.
	h, err = New(
		ctx,
		Transport(tcp.NewTCPTransport),
	)
	if err != nil {
		t.Fatal(err)
	}

	if len(h.Network().ListenAddresses()) != 1 {
		t.Error("expected one listen addr with user defined transport")
	}
	if re2.FindStringSubmatchIndex(h.Network().ListenAddresses()[0].String()) == nil {
		t.Error("expected relay address")
	}
	h.Close()
}

func makeRandomHost(t *testing.T, port int) (host.Host, error) {
	ctx := context.Background()
	priv, _, err := crypto.GenerateKeyPair(crypto.RSA, 2048)
	if err != nil {
		t.Fatal(err)
	}

	opts := []Option{
		ListenAddrStrings(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", port)),
		Identity(priv),
		DefaultTransports,
		DefaultMuxers,
		DefaultSecurity,
		NATPortMap(),
	}

	return New(ctx, opts...)
}

func TestChainOptions(t *testing.T) {
	var cfg Config
	var optsRun []int
	optcount := 0
	newOpt := func() Option {
		index := optcount
		optcount++
		return func(c *Config) error {
			optsRun = append(optsRun, index)
			return nil
		}
	}
	if err := cfg.Apply(newOpt(), nil, ChainOptions(newOpt(), newOpt(), ChainOptions(), ChainOptions(nil, newOpt()))); err != nil {
		t.Fatal(err)
	}
	if optcount != len(optsRun) {
		t.Errorf("expected to have handled %d options, handled %d", optcount, len(optsRun))
	}
	for i, x := range optsRun {
		if i != x {
			t.Errorf("expected opt %d, got opt %d", i, x)
		}
	}
}
<commit_msg>chore: improve nil option tests<commit_after>package libp2p

import (
	"context"
	"fmt"
	"regexp"
	"strings"
	"testing"

	crypto "github.com/libp2p/go-libp2p-crypto"
	host "github.com/libp2p/go-libp2p-host"
	pstore "github.com/libp2p/go-libp2p-peerstore"
	"github.com/libp2p/go-tcp-transport"
)

func TestNewHost(t *testing.T) {
	h, err := makeRandomHost(t, 9000)
	if err != nil {
		t.Fatal(err)
	}
	h.Close()
}

func TestBadTransportConstructor(t *testing.T) {
	ctx := context.Background()
	h, err := New(ctx, Transport(func() {}))
	if err == nil {
		h.Close()
		t.Fatal("expected an error")
	}
	if !strings.Contains(err.Error(), "libp2p_test.go") {
		t.Error("expected error to contain debugging info")
	}
}

func TestNoListenAddrs(t *testing.T) {
	ctx := context.Background()
	h, err := New(ctx, NoListenAddrs)
	if err != nil {
		t.Fatal(err)
	}
	defer h.Close()
	if len(h.Addrs()) != 0 {
		t.Fatal("expected no addresses")
	}
}

func TestNoTransports(t *testing.T) {
	ctx := context.Background()
	a, err := New(ctx, NoTransports)
	if err != nil {
		t.Fatal(err)
	}
	defer a.Close()

	b, err := New(ctx, ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))
	if err != nil {
		t.Fatal(err)
	}
	defer b.Close()

	err = a.Connect(ctx, pstore.PeerInfo{
		ID:    b.ID(),
		Addrs: b.Addrs(),
	})
	if err == nil {
		t.Error("dial should have failed as no transports have been configured")
	}
}

func TestInsecure(t *testing.T) {
	ctx := context.Background()
	h, err := New(ctx, NoSecurity)
	if err != nil {
		t.Fatal(err)
	}
	h.Close()
}

func TestDefaultListenAddrs(t *testing.T) {
	ctx := context.Background()

	re := regexp.MustCompile("/(ip)[4|6]/((0.0.0.0)|(::))/tcp/")
	re2 := regexp.MustCompile("/p2p-circuit")

	// Test 1: Setting the correct listen addresses if userDefined.Transport == nil && userDefined.ListenAddrs == nil
	h, err := New(ctx)
	if err != nil {
		t.Fatal(err)
	}
	for _, addr := range h.Network().ListenAddresses() {
		if re.FindStringSubmatchIndex(addr.String()) == nil &&
			re2.FindStringSubmatchIndex(addr.String()) == nil {
			t.Error("expected ip4 or ip6 or relay interface")
		}
	}

	h.Close()

	// Test 2: Listen addr only include relay if user defined transport is passed.
	h, err = New(
		ctx,
		Transport(tcp.NewTCPTransport),
	)
	if err != nil {
		t.Fatal(err)
	}

	if len(h.Network().ListenAddresses()) != 1 {
		t.Error("expected one listen addr with user defined transport")
	}
	if re2.FindStringSubmatchIndex(h.Network().ListenAddresses()[0].String()) == nil {
		t.Error("expected relay address")
	}
	h.Close()
}

func makeRandomHost(t *testing.T, port int) (host.Host, error) {
	ctx := context.Background()
	priv, _, err := crypto.GenerateKeyPair(crypto.RSA, 2048)
	if err != nil {
		t.Fatal(err)
	}

	opts := []Option{
		ListenAddrStrings(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", port)),
		Identity(priv),
		DefaultTransports,
		DefaultMuxers,
		DefaultSecurity,
		NATPortMap(),
	}

	return New(ctx, opts...)
}

func TestChainOptions(t *testing.T) {
	var cfg Config
	var optsRun []int
	optcount := 0
	newOpt := func() Option {
		index := optcount
		optcount++
		return func(c *Config) error {
			optsRun = append(optsRun, index)
			return nil
		}
	}

	if err := cfg.Apply(newOpt(), nil, ChainOptions(newOpt(), newOpt(), ChainOptions(), ChainOptions(nil, newOpt()))); err != nil {
		t.Fatal(err)
	}

	// Make sure we ran all options.
	if optcount != 4 {
		t.Errorf("expected to have handled %d options, handled %d", optcount, len(optsRun))
	}

	// Make sure we ran the options in-order.
	for i, x := range optsRun {
		if i != x {
			t.Errorf("expected opt %d, got opt %d", i, x)
		}
	}
}
<|endoftext|>