<commit_before>package buf

import (
	"io"
	"runtime"
	"syscall"

	"v2ray.com/core/common/platform"
)

type allocStrategy struct {
	current uint32
}

func (s *allocStrategy) Current() uint32 {
	return s.current
}

func (s *allocStrategy) Adjust(n uint32) {
	if n >= s.current {
		s.current *= 4
	} else {
		s.current = n
	}

	if s.current > 32 {
		s.current = 32
	}

	if s.current == 0 {
		s.current = 1
	}
}

func (s *allocStrategy) Alloc() []*Buffer {
	bs := make([]*Buffer, s.current)
	for i := range bs {
		bs[i] = New()
	}
	return bs
}

type multiReader interface {
	Init([]*Buffer)
	Read(fd uintptr) int32
	Clear()
}

type ReadVReader struct {
	io.Reader
	rawConn syscall.RawConn
	mr      multiReader
	alloc   allocStrategy
}

func NewReadVReader(reader io.Reader, rawConn syscall.RawConn) *ReadVReader {
	return &ReadVReader{
		Reader:  reader,
		rawConn: rawConn,
		alloc: allocStrategy{
			current: 1,
		},
		mr: newMultiReader(),
	}
}

func (r *ReadVReader) readMulti() (MultiBuffer, error) {
	bs := r.alloc.Alloc()

	r.mr.Init(bs)
	var nBytes int32
	err := r.rawConn.Read(func(fd uintptr) bool {
		n := r.mr.Read(fd)
		if n < 0 {
			return false
		}

		nBytes = n
		return true
	})
	r.mr.Clear()

	if err != nil {
		mb := MultiBuffer(bs)
		mb.Release()
		return nil, err
	}

	if nBytes == 0 {
		mb := MultiBuffer(bs)
		mb.Release()
		return nil, io.EOF
	}

	nBuf := 0
	for nBuf < len(bs) {
		if nBytes <= 0 {
			break
		}
		end := int32(nBytes)
		if end > Size {
			end = Size
		}
		bs[nBuf].end = end
		nBytes -= end
		nBuf++
	}

	for i := nBuf; i < len(bs); i++ {
		bs[i].Release()
		bs[i] = nil
	}

	return MultiBuffer(bs[:nBuf]), nil
}

// ReadMultiBuffer implements Reader.
func (r *ReadVReader) ReadMultiBuffer() (MultiBuffer, error) {
	if r.alloc.Current() == 1 {
		b, err := readOne(r.Reader)
		if err != nil {
			return nil, err
		}
		if b.IsFull() {
			r.alloc.Adjust(1)
		}
		return NewMultiBufferValue(b), nil
	}

	mb, err := r.readMulti()
	if err != nil {
		return nil, err
	}
	r.alloc.Adjust(uint32(len(mb)))
	return mb, nil
}

var useReadv = false

func init() {
	const defaultFlagValue = "NOT_DEFINED_AT_ALL"
	value := platform.NewEnvFlag("v2ray.buf.readv").GetValue(func() string { return defaultFlagValue })
	switch value {
	case defaultFlagValue, "auto":
		if (runtime.GOARCH == "386" || runtime.GOARCH == "amd64" || runtime.GOARCH == "s390x") && (runtime.GOOS == "linux" || runtime.GOOS == "darwin") {
			useReadv = true
		}
	case "enable":
		useReadv = true
	}
}
<commit_msg>enable readv reader on windows<commit_after>package buf

import (
	"io"
	"runtime"
	"syscall"

	"v2ray.com/core/common/platform"
)

type allocStrategy struct {
	current uint32
}

func (s *allocStrategy) Current() uint32 {
	return s.current
}

func (s *allocStrategy) Adjust(n uint32) {
	if n >= s.current {
		s.current *= 4
	} else {
		s.current = n
	}

	if s.current > 32 {
		s.current = 32
	}

	if s.current == 0 {
		s.current = 1
	}
}

func (s *allocStrategy) Alloc() []*Buffer {
	bs := make([]*Buffer, s.current)
	for i := range bs {
		bs[i] = New()
	}
	return bs
}

type multiReader interface {
	Init([]*Buffer)
	Read(fd uintptr) int32
	Clear()
}

type ReadVReader struct {
	io.Reader
	rawConn syscall.RawConn
	mr      multiReader
	alloc   allocStrategy
}

func NewReadVReader(reader io.Reader, rawConn syscall.RawConn) *ReadVReader {
	return &ReadVReader{
		Reader:  reader,
		rawConn: rawConn,
		alloc: allocStrategy{
			current: 1,
		},
		mr: newMultiReader(),
	}
}

func (r *ReadVReader) readMulti() (MultiBuffer, error) {
	bs := r.alloc.Alloc()

	r.mr.Init(bs)
	var nBytes int32
	err := r.rawConn.Read(func(fd uintptr) bool {
		n := r.mr.Read(fd)
		if n < 0 {
			return false
		}

		nBytes = n
		return true
	})
	r.mr.Clear()

	if err != nil {
		mb := MultiBuffer(bs)
		mb.Release()
		return nil, err
	}

	if nBytes == 0 {
		mb := MultiBuffer(bs)
		mb.Release()
		return nil, io.EOF
	}

	nBuf := 0
	for nBuf < len(bs) {
		if nBytes <= 0 {
			break
		}
		end := int32(nBytes)
		if end > Size {
			end = Size
		}
		bs[nBuf].end = end
		nBytes -= end
		nBuf++
	}

	for i := nBuf; i < len(bs); i++ {
		bs[i].Release()
		bs[i] = nil
	}

	return MultiBuffer(bs[:nBuf]), nil
}

// ReadMultiBuffer implements Reader.
func (r *ReadVReader) ReadMultiBuffer() (MultiBuffer, error) {
	if r.alloc.Current() == 1 {
		b, err := readOne(r.Reader)
		if err != nil {
			return nil, err
		}
		if b.IsFull() {
			r.alloc.Adjust(1)
		}
		return NewMultiBufferValue(b), nil
	}

	mb, err := r.readMulti()
	if err != nil {
		return nil, err
	}
	r.alloc.Adjust(uint32(len(mb)))
	return mb, nil
}

var useReadv = false

func init() {
	const defaultFlagValue = "NOT_DEFINED_AT_ALL"
	value := platform.NewEnvFlag("v2ray.buf.readv").GetValue(func() string { return defaultFlagValue })
	switch value {
	case defaultFlagValue, "auto":
		if (runtime.GOARCH == "386" || runtime.GOARCH == "amd64" || runtime.GOARCH == "s390x") && (runtime.GOOS == "linux" || runtime.GOOS == "darwin" || runtime.GOOS == "windows") {
			useReadv = true
		}
	case "enable":
		useReadv = true
	}
}
<|endoftext|>


<commit_before>//go:build cit
// +build cit

package imagevalidation

import (
	"fmt"
	"io/ioutil"
	"path/filepath"
	"regexp"
	"testing"
)

var licenseNames = []string{
	"Apache License",
	"Artistic/GPL",
	"Artistic",
	"Autoconf",
	"\"Bitstream Vera\"",
	"BSD",
	"BZIP",
	"COMMON PUBLIC LICENSE VERSION 1.0",
	"config-h",
	"curl",
	"Expat",
	"GAP",
	"GD",
	"GFDL-1.3+",
	"GNU General Public License",
	"GNU GPL",
	"GNU LGPL",
	"GNU Lesser Public License",
	"GPL",
	"HPND",
	"IBM PUBLIC LICENSE VERSION 1.0",
	"ISC",
	"JPEG",
	"LGPL",
	"MIT",
	"MIT license",
	"MIT/X11 (BSD like)",
	"MPL-2.0",
	"no notice",
	"noderivs",
	"none",
	"PD",
	"PD-debian",
	"PERLDOCS",
	"permissive-fsf",
	"permissive-nowarranty",
	"probably-PD",
	"Paul Vixie\"s license",
	"public-domain",
	"REGCOMP",
	"S2P",
	"SDBM-PUBLIC-DOMAIN",
	"SIL Open Font License",
	"TEXT-SOUNDEX",
	"TEXT-TABS",
	"The OpenLDAP Public License",
	"Unicode",
	"X11",
	"X11-2",
	"ZLIB",
}

var licenses = []string{
	`Permission to use, copy, modify, distribute, and sell this software and its documentation for any purpose is hereby granted without fee, provided that the above copyright notice appear in all copies and that both that copyright notice and this permission notice appear in supporting documentation, and that the name of the authors not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. The authors makes no representations about the suitability of this software for any purpose. It is provided "as is" without express or implied warranty.`,
	`free software; you can redistribute it and/or modify it under the terms of the GNU.*General Public License.*as published by the Free Software Foundation`,
	`The main library is licensed under GNU Lesser General Public License (LGPL) version 2.1+, Gnutls Extra (i.e. openssl wrapper library, and library for code for "GnuTLS Inner Application" support) build system, testsuite and commandline utilities are licenced under the GNU General Public License version 3+. The Guile bindings use the same license as the respective underlying library, i.e. LGPLv2.1+ for the main library and GPLv3+ for Gnutls extra.`,
	`Permission is granted to anyone to use this.*for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions`,
	`This software is released under the terms of the GNU.*General Public License.*`,
	`All files in this package can be freely distributed and used according to the terms of the GNU.*General Public License, either version 2 or (at your opinion) any newer version. This is the same distribution policy as for the Linux kernel itself -- see /usr/src/linux/COPYING for details.`,
	`You are free to distribute this.*under the terms of the BSD License`,
	`All files in this.*can be freely distributed and used according to the terms of the GNU General Public License`,
	`all of the code is covered under the terms of the GPL.`,
	`is free software`,
	`You are free to distribute this software under the terms of the BSD License.`,
	`is licensed under the BSD license`,
	`(is|are|be) free to distribute`,
	`may freely distribute`,
	`(is|are|be) freely distributed`,
	`.*is available under the terms of the GNU.*Public License`,
	`This data is licenced under 2 different licenses 1\) GNU General Public License, version 2 or later 2\) XFree86 1.0 license This data can be used freely under either license.`,
	`.*is in the public domain.`,
	`is covered under the terms of the GNU Public License.`,
	`redistribute it freely`,
	`the complete text of the GNU General Public License can be found in`,
	`free for commercial and non-commercial use as long as the following conditions are aheared to`,
	`Permission to.*use.*distribute.*this.*for any purpose.*is.*granted`,
	`There are no restrictions on distributing unmodified copies of Vim except that they must include this license text.`,
	`Redistribution and use.*(is|are) permitted`,
	`Permission is.*granted.*deal.*without restriction, including without limitation the rights to use`,
	`All its programs.*may be redistributed under the terms of the GNU GPL, Version 2 or later`,
	`is distributed under the GNU.*General Public License`,
	`This software is distributed under the GNU General Public License`,
	`This package is dual-licensed under the Academic Free License version 2.1, and the GPL version 2.`,
	`may be used, modified and redistributed only under the terms of the GNU General Public License`,
	`has been placed in the public domain`,
	`And licensed under the terms of the GPL license`,
	`are distributed under the terms of the GNU.*General Public License`,
	`The keys in the keyrings don\'t fall under any copyright. Everything else in the package is covered by the GNU GPL.`,
	`the complete text of the GNU General Public License and of the GNU Lesser Public License can be found in`,
	`THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS IBM PUBLIC LICENSE`,
	`THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS COMMON PUBLIC LICENSE`,
	`GNU LESSER GENERAL PUBLIC LICENSE`,
	`Permission is hereby granted.*to any person obtaining a copy of.*and associated documentation files.*to deal in.*without restriction`,
	`Redistribution and use of this software and associated documentation ("Software"), with or without modification, are permitted`,
	`This code is multi Licensed under all/any one of.*LGPLv2.*New Style BSD.*MIT`,
	`LICENSE. You may copy and use the Software, subject to these conditions: 1. This Software is licensed for use only in conjunction with Intel component products. Use of the Software in conjunction with non-Intel component products is not licensed hereunder.`,
	`Brocade Linux Fibre Channel HBA Firmware`,
	`QLogic Linux Fibre Channel HBA Firmware`,
	`Unlimited distribution and/or modification is allowed as long as this copyright notice remains intact.`,
	`Permission is hereby granted to use.*this.*for any purpose`,
	`are in the public domain`,
	`is (available|distributed) under the terms of the GNU.*Public License`,
	`(libudev|libgudev|udev) is licensed under the GNU (L|)GPL`,
	`The Linux Console Tools are covered by the GPL`,
	`Some portions of os-prober`,
	`Netcat and the associated package is a product of Avian Research, and is freely available in full source form with no restrictions save an obligation to give credit where due.`,
	`Permission is hereby granted, without written agreement and without licence or royalty fees, to use, copy, modify, and distribute this software`,
	`Open Market permits you to use, copy, modify, distribute, and license this Software and the Documentation for any purpose, provided that existing copyright notices are retained in all copies and that this notice is included verbatim in any distributions. No written agreement, license, or royalty fee is required for any of the authorized uses.`,
	`This software is made available under the terms of *either* of the licenses found in LICENSE.APACHE or LICENSE.BSD.`,
	`This library.*is public domain software`,
	`Apache License`,
	`The Artistic License 2.0`,
	`FULLTEXT`,
	`DO WHAT THE HELL YOU WANT TO`, // Yes, this is a real license.
	`arping: GPL v2 or later`,      // iputils has a license summary file
	`PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2`,
	`This is the Python license. In short, you can use this product in commercial and non-commercial applications`,
}

const (
	licenseNameRegex = `(?i)(((Licen[sc]e|Copyright)\s*:\s*%[1]s)|((covered )?under (the )?%[1]s)|(under (the terms of )?the %[1]s))`
)

var licenseGlobs = []string{
	"/usr/share/doc/*/copyright",
	"/usr/share/doc/*/LICENSE",
	"/usr/share/licenses/*/LICENSE",
}

func isValidLicenseName(licenseCheck string) bool {
	for _, name := range licenseNames {
		var regexString = fmt.Sprintf(licenseNameRegex, name)
		re, err := regexp.Compile(regexString)
		if err != nil {
			return false
		}
		if re.MatchString(licenseCheck) {
			return true
		}
	}
	return false
}

func isValidLicenseText(licenseCheck string) bool {
	for _, licenseText := range licenses {
		// (?i) case insensitive
		re, err := regexp.Compile(`(?i)` + licenseText)
		if err != nil {
			return false
		}

		if re.MatchString(licenseCheck) {
			return true
		}
	}
	return false
}

func TestArePackagesLegal(t *testing.T) {
	for _, pathGlob := range licenseGlobs {
		filenames, err := filepath.Glob(pathGlob)
		if err != nil || len(filenames) == 0 {
			t.Logf("couldnt resolve glob %s", pathGlob)
			continue
		}

		for _, filename := range filenames {
			isLegal, err := isPackageLegal(filename)
			if err != nil {
				t.Fatalf(err.Error())
			}
			if !isLegal {
				t.Errorf("Found illegal package: %v", filename)
			}
		}
	}
}

func isPackageLegal(filepath string) (bool, error) {
	bytes, err := ioutil.ReadFile(filepath)
	if err != nil {
		return false, fmt.Errorf("error reading file: %v", err)
	}

	var licenseCheck string = string(bytes)

	// Remove comments.
	commentRegex, err := regexp.Compile(`\*|#`)
	if err != nil {
		return false, fmt.Errorf("invalid regular expression: %v", err)
	}
	licenseCheck = commentRegex.ReplaceAllString(licenseCheck, "")

	// Replace repeated whitespace and newlines with one space.
	whitespaceRegex, err := regexp.Compile(`\s+`)
	if err != nil {
		return false, fmt.Errorf("invalid regular expression: %v", err)
	}
	licenseCheck = whitespaceRegex.ReplaceAllString(licenseCheck, " ")
	if isValidLicenseName(licenseCheck) || isValidLicenseText(licenseCheck) {
		return true, nil
	}

	return false, nil
}
<commit_msg>Skip license test for Ubuntu Pro Fips. (#546)<commit_after>//go:build cit
// +build cit

package imagevalidation

import (
	"fmt"
	"io/ioutil"
	"path/filepath"
	"regexp"
	"strings"
	"testing"

	"github.com/GoogleCloudPlatform/guest-test-infra/imagetest/utils"
)

var licenseNames = []string{
	"Apache License",
	"Artistic/GPL",
	"Artistic",
	"Autoconf",
	"\"Bitstream Vera\"",
	"BSD",
	"BZIP",
	"COMMON PUBLIC LICENSE VERSION 1.0",
	"config-h",
	"curl",
	"Expat",
	"GAP",
	"GD",
	"GFDL-1.3+",
	"GNU General Public License",
	"GNU GPL",
	"GNU LGPL",
	"GNU Lesser Public License",
	"GPL",
	"HPND",
	"IBM PUBLIC LICENSE VERSION 1.0",
	"ISC",
	"JPEG",
	"LGPL",
	"MIT",
	"MIT license",
	"MIT/X11 (BSD like)",
	"MPL-2.0",
	"no notice",
	"noderivs",
	"none",
	"PD",
	"PD-debian",
	"PERLDOCS",
	"permissive-fsf",
	"permissive-nowarranty",
	"probably-PD",
	"Paul Vixie\"s license",
	"public-domain",
	"REGCOMP",
	"S2P",
	"SDBM-PUBLIC-DOMAIN",
	"SIL Open Font License",
	"TEXT-SOUNDEX",
	"TEXT-TABS",
	"The OpenLDAP Public License",
	"Unicode",
	"X11",
	"X11-2",
	"ZLIB",
}

var licenses = []string{
	`Permission to use, copy, modify, distribute, and sell this software and its documentation for any purpose is hereby granted without fee, provided that the above copyright notice appear in all copies and that both that copyright notice and this permission notice appear in supporting documentation, and that the name of the authors not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. The authors makes no representations about the suitability of this software for any purpose. It is provided "as is" without express or implied warranty.`,
	`free software; you can redistribute it and/or modify it under the terms of the GNU.*General Public License.*as published by the Free Software Foundation`,
	`The main library is licensed under GNU Lesser General Public License (LGPL) version 2.1+, Gnutls Extra (i.e. openssl wrapper library, and library for code for "GnuTLS Inner Application" support) build system, testsuite and commandline utilities are licenced under the GNU General Public License version 3+. The Guile bindings use the same license as the respective underlying library, i.e. LGPLv2.1+ for the main library and GPLv3+ for Gnutls extra.`,
	`Permission is granted to anyone to use this.*for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions`,
	`This software is released under the terms of the GNU.*General Public License.*`,
	`All files in this package can be freely distributed and used according to the terms of the GNU.*General Public License, either version 2 or (at your opinion) any newer version. This is the same distribution policy as for the Linux kernel itself -- see /usr/src/linux/COPYING for details.`,
	`You are free to distribute this.*under the terms of the BSD License`,
	`All files in this.*can be freely distributed and used according to the terms of the GNU General Public License`,
	`all of the code is covered under the terms of the GPL.`,
	`is free software`,
	`You are free to distribute this software under the terms of the BSD License.`,
	`is licensed under the BSD license`,
	`(is|are|be) free to distribute`,
	`may freely distribute`,
	`(is|are|be) freely distributed`,
	`.*is available under the terms of the GNU.*Public License`,
	`This data is licenced under 2 different licenses 1\) GNU General Public License, version 2 or later 2\) XFree86 1.0 license This data can be used freely under either license.`,
	`.*is in the public domain.`,
	`is covered under the terms of the GNU Public License.`,
	`redistribute it freely`,
	`the complete text of the GNU General Public License can be found in`,
	`free for commercial and non-commercial use as long as the following conditions are aheared to`,
	`Permission to.*use.*distribute.*this.*for any purpose.*is.*granted`,
	`There are no restrictions on distributing unmodified copies of Vim except that they must include this license text.`,
	`Redistribution and use.*(is|are) permitted`,
	`Permission is.*granted.*deal.*without restriction, including without limitation the rights to use`,
	`All its programs.*may be redistributed under the terms of the GNU GPL, Version 2 or later`,
	`is distributed under the GNU.*General Public License`,
	`This software is distributed under the GNU General Public License`,
	`This package is dual-licensed under the Academic Free License version 2.1, and the GPL version 2.`,
	`may be used, modified and redistributed only under the terms of the GNU General Public License`,
	`has been placed in the public domain`,
	`And licensed under the terms of the GPL license`,
	`are distributed under the terms of the GNU.*General Public License`,
	`The keys in the keyrings don\'t fall under any copyright. Everything else in the package is covered by the GNU GPL.`,
	`the complete text of the GNU General Public License and of the GNU Lesser Public License can be found in`,
	`THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS IBM PUBLIC LICENSE`,
	`THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS COMMON PUBLIC LICENSE`,
	`GNU LESSER GENERAL PUBLIC LICENSE`,
	`Permission is hereby granted.*to any person obtaining a copy of.*and associated documentation files.*to deal in.*without restriction`,
	`Redistribution and use of this software and associated documentation ("Software"), with or without modification, are permitted`,
	`This code is multi Licensed under all/any one of.*LGPLv2.*New Style BSD.*MIT`,
	`LICENSE. You may copy and use the Software, subject to these conditions: 1. This Software is licensed for use only in conjunction with Intel component products. Use of the Software in conjunction with non-Intel component products is not licensed hereunder.`,
	`Brocade Linux Fibre Channel HBA Firmware`,
	`QLogic Linux Fibre Channel HBA Firmware`,
	`Unlimited distribution and/or modification is allowed as long as this copyright notice remains intact.`,
	`Permission is hereby granted to use.*this.*for any purpose`,
	`are in the public domain`,
	`is (available|distributed) under the terms of the GNU.*Public License`,
	`(libudev|libgudev|udev) is licensed under the GNU (L|)GPL`,
	`The Linux Console Tools are covered by the GPL`,
	`Some portions of os-prober`,
	`Netcat and the associated package is a product of Avian Research, and is freely available in full source form with no restrictions save an obligation to give credit where due.`,
	`Permission is hereby granted, without written agreement and without licence or royalty fees, to use, copy, modify, and distribute this software`,
	`Open Market permits you to use, copy, modify, distribute, and license this Software and the Documentation for any purpose, provided that existing copyright notices are retained in all copies and that this notice is included verbatim in any distributions. No written agreement, license, or royalty fee is required for any of the authorized uses.`,
	`This software is made available under the terms of *either* of the licenses found in LICENSE.APACHE or LICENSE.BSD.`,
	`This library.*is public domain software`,
	`Apache License`,
	`The Artistic License 2.0`,
	`FULLTEXT`,
	`DO WHAT THE HELL YOU WANT TO`, // Yes, this is a real license.
	`arping: GPL v2 or later`,      // iputils has a license summary file
	`PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2`,
	`This is the Python license. In short, you can use this product in commercial and non-commercial applications`,
}

const (
	licenseNameRegex = `(?i)(((Licen[sc]e|Copyright)\s*:\s*%[1]s)|((covered )?under (the )?%[1]s)|(under (the terms of )?the %[1]s))`
)

var licenseGlobs = []string{
	"/usr/share/doc/*/copyright",
	"/usr/share/doc/*/LICENSE",
	"/usr/share/licenses/*/LICENSE",
}

func isValidLicenseName(licenseCheck string) bool {
	for _, name := range licenseNames {
		var regexString = fmt.Sprintf(licenseNameRegex, name)
		re, err := regexp.Compile(regexString)
		if err != nil {
			return false
		}
		if re.MatchString(licenseCheck) {
			return true
		}
	}
	return false
}

func isValidLicenseText(licenseCheck string) bool {
	for _, licenseText := range licenses {
		// (?i) case insensitive
		re, err := regexp.Compile(`(?i)` + licenseText)
		if err != nil {
			return false
		}

		if re.MatchString(licenseCheck) {
			return true
		}
	}
	return false
}

func TestArePackagesLegal(t *testing.T) {
	image, err := utils.GetMetadata("image")
	if err != nil {
		t.Fatalf("couldn't get image from metadata")
	}
	if strings.Contains(image, "ubuntu-pro-fips") {
		// Ubuntu Pro FIPS images have some non-standard packages.
		t.Skip("Not supported on Ubuntu Pro Fips")
	}
	for _, pathGlob := range licenseGlobs {
		filenames, err := filepath.Glob(pathGlob)
		if err != nil || len(filenames) == 0 {
			t.Logf("couldnt resolve glob %s", pathGlob)
			continue
		}

		for _, filename := range filenames {
			isLegal, err := isPackageLegal(filename)
			if err != nil {
				t.Fatalf(err.Error())
			}
			if !isLegal {
				t.Errorf("Found illegal package: %v", filename)
			}
		}
	}
}

func isPackageLegal(filepath string) (bool, error) {
	bytes, err := ioutil.ReadFile(filepath)
	if err != nil {
		return false, fmt.Errorf("error reading file: %v", err)
	}

	var licenseCheck string = string(bytes)

	// Remove comments.
	commentRegex, err := regexp.Compile(`\*|#`)
	if err != nil {
		return false, fmt.Errorf("invalid regular expression: %v", err)
	}
	licenseCheck = commentRegex.ReplaceAllString(licenseCheck, "")

	// Replace repeated whitespace and newlines with one space.
	whitespaceRegex, err := regexp.Compile(`\s+`)
	if err != nil {
		return false, fmt.Errorf("invalid regular expression: %v", err)
	}
	licenseCheck = whitespaceRegex.ReplaceAllString(licenseCheck, " ")
	if isValidLicenseName(licenseCheck) || isValidLicenseText(licenseCheck) {
		return true, nil
	}

	return false, nil
}
<|endoftext|>


<commit_before>package protocol

import (
	"runtime"

	v2net "v2ray.com/core/common/net"
	"v2ray.com/core/common/uuid"
)

type RequestCommand byte

const (
	RequestCommandTCP = RequestCommand(0x01)
	RequestCommandUDP = RequestCommand(0x02)
)

type RequestOption byte

const (
	RequestOptionChunkStream     = RequestOption(0x01)
	RequestOptionConnectionReuse = RequestOption(0x02)
)

func (v RequestOption) Has(option RequestOption) bool {
	return (v & option) == option
}

func (v *RequestOption) Set(option RequestOption) {
	*v = (*v | option)
}

func (v *RequestOption) Clear(option RequestOption) {
	*v = (*v & (^option))
}

type Security byte

func (v Security) Is(t SecurityType) bool {
	return v == Security(t)
}

type RequestHeader struct {
	Version  byte
	User     *User
	Command  RequestCommand
	Option   RequestOption
	Security Security
	Address  v2net.Address
	Port     v2net.Port
}

func (v *RequestHeader) Destination() v2net.Destination {
	if v.Command == RequestCommandUDP {
		return v2net.UDPDestination(v.Address, v.Port)
	}
	return v2net.TCPDestination(v.Address, v.Port)
}

type ResponseOption byte

const (
	ResponseOptionConnectionReuse = ResponseOption(1)
)

func (v *ResponseOption) Set(option ResponseOption) {
	*v = (*v | option)
}

func (v ResponseOption) Has(option ResponseOption) bool {
	return (v | option) == option
}

func (v *ResponseOption) Clear(option ResponseOption) {
	*v = (*v & (^option))
}

type ResponseCommand interface{}

type ResponseHeader struct {
	Option  ResponseOption
	Command ResponseCommand
}

type CommandSwitchAccount struct {
	Host     v2net.Address
	Port     v2net.Port
	ID       *uuid.UUID
	AlterIds uint16
	Level    uint32
	ValidMin byte
}

func (v *SecurityConfig) AsSecurity() Security {
	if v == nil {
		return Security(SecurityType_LEGACY)
	}
	if v.Type == SecurityType_AUTO {
		if runtime.GOARCH == "amd64" || runtime.GOARCH == "s390x" {
			return Security(SecurityType_AES128_GCM)
		}
		return Security(SecurityType_CHACHA20_POLY1305)
	}
	return Security(v.Type)
}
<commit_msg>Fix response option<commit_after>package protocol

import (
	"runtime"

	v2net "v2ray.com/core/common/net"
	"v2ray.com/core/common/uuid"
)

type RequestCommand byte

const (
	RequestCommandTCP = RequestCommand(0x01)
	RequestCommandUDP = RequestCommand(0x02)
)

type RequestOption byte

const (
	RequestOptionChunkStream     = RequestOption(0x01)
	RequestOptionConnectionReuse = RequestOption(0x02)
)

func (v RequestOption) Has(option RequestOption) bool {
	return (v & option) == option
}

func (v *RequestOption) Set(option RequestOption) {
	*v = (*v | option)
}

func (v *RequestOption) Clear(option RequestOption) {
	*v = (*v & (^option))
}

type Security byte

func (v Security) Is(t SecurityType) bool {
	return v == Security(t)
}

type RequestHeader struct {
	Version  byte
	User     *User
	Command  RequestCommand
	Option   RequestOption
	Security Security
	Address  v2net.Address
	Port     v2net.Port
}

func (v *RequestHeader) Destination() v2net.Destination {
	if v.Command == RequestCommandUDP {
		return v2net.UDPDestination(v.Address, v.Port)
	}
	return v2net.TCPDestination(v.Address, v.Port)
}

type ResponseOption byte

const (
	ResponseOptionConnectionReuse = ResponseOption(1)
)

func (v *ResponseOption) Set(option ResponseOption) {
	*v = (*v | option)
}

func (v ResponseOption) Has(option ResponseOption) bool {
	return (v & option) == option
}

func (v *ResponseOption) Clear(option ResponseOption) {
	*v = (*v & (^option))
}

type ResponseCommand interface{}

type ResponseHeader struct {
	Option  ResponseOption
	Command ResponseCommand
}

type CommandSwitchAccount struct {
	Host     v2net.Address
	Port     v2net.Port
	ID       *uuid.UUID
	AlterIds uint16
	Level    uint32
	ValidMin byte
}

func (v *SecurityConfig) AsSecurity() Security {
	if v == nil {
		return Security(SecurityType_LEGACY)
	}
	if v.Type == SecurityType_AUTO {
		if runtime.GOARCH == "amd64" || runtime.GOARCH == "s390x" {
			return Security(SecurityType_AES128_GCM)
		}
		return Security(SecurityType_CHACHA20_POLY1305)
	}
	return Security(v.Type)
}
<|endoftext|>


<commit_before>package common

import (
	"encoding/hex"
)

const VideoProfileIDSize = 8
const VideoProfileIDBytes = 4

type VideoProfileByteMap map[[VideoProfileIDBytes]byte]string

var VideoProfileNameLookup = map[string]string{
	"a7ac137a": "P720p60fps16x9",
	"49d54ea9": "P720p30fps16x9",
	"79332fe7": "P720p30fps4x3",
	"5ecf4b52": "P576p30fps16x9",
	"93c717e7": "P360p30fps16x9",
	"b60382a0": "P360p30fps4x3",
	"c0a6517a": "P240p30fps16x9",
	"d435c53a": "P240p30fps4x3",
	"fca40bf9": "P144p30fps16x9",
}

func makeVideoProfileByteMap() VideoProfileByteMap {
	var ret = make(VideoProfileByteMap, 0)
	for k, v := range VideoProfileNameLookup {
		b, err := hex.DecodeString(k)
		if err != nil || len(b) != VideoProfileIDBytes {
			continue
		}
		var key [VideoProfileIDBytes]byte
		copy(key[:], b)
		ret[key] = v
	}
	return ret
}

var VideoProfileByteLookup = makeVideoProfileByteMap()
<commit_msg>adding 25fps<commit_after>package common

import (
	"encoding/hex"
)

const VideoProfileIDSize = 8
const VideoProfileIDBytes = 4

type VideoProfileByteMap map[[VideoProfileIDBytes]byte]string

var VideoProfileNameLookup = map[string]string{
	"a7ac137a": "P720p60fps16x9",
	"49d54ea9": "P720p30fps16x9",
	"e4a64019": "P720p25fps16x9",
	"79332fe7": "P720p30fps4x3",
	"5ecf4b52": "P576p30fps16x9",
	"8b1843d6": "P576p25fps16x9",
	"93c717e7": "P360p30fps16x9",
	"7cd40fc7": "P360p25fps16x9",
	"b60382a0": "P360p30fps4x3",
	"c0a6517a": "P240p30fps16x9",
	"1301a7d0": "P240p25fps16x9",
	"d435c53a": "P240p30fps4x3",
	"fca40bf9": "P144p30fps16x9",
	"03f01d1f": "P144p25fps16x9",
}

func makeVideoProfileByteMap() VideoProfileByteMap {
	var ret = make(VideoProfileByteMap, 0)
	for k, v := range VideoProfileNameLookup {
		b, err := hex.DecodeString(k)
		if err != nil || len(b) != VideoProfileIDBytes {
			continue
		}
		var key [VideoProfileIDBytes]byte
		copy(key[:], b)
		ret[key] = v
	}
	return ret
}

var VideoProfileByteLookup = makeVideoProfileByteMap()
<|endoftext|>


<commit_before>package hoverfly
<commit_msg>test start hoverfly listener<commit_after>package hoverfly

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"testing"
)

func TestHoverflyListener(t *testing.T) {
	server, dbClient := testTools(201, `{'message': 'here'}`)
	defer server.Close()

	proxyPort := "9777"

	dbClient.Cfg.ProxyPort = proxyPort
	// starting hoverfly
	proxy, _ := GetNewHoverfly(dbClient.Cfg, dbClient.Cache)
	StartHoverflyProxy(dbClient.Cfg, proxy)

	// checking whether it's running
	response, err := http.Get(fmt.Sprintf("http://localhost:%s/", proxyPort))
	expect(t, err, nil)

	expect(t, response.StatusCode, 500)

	body, err := ioutil.ReadAll(response.Body)
	expect(t, err, nil)
	expect(t, strings.Contains(string(body), "is a proxy server"), true)
}
<|endoftext|>


<commit_before>package lncfg

import "fmt"

const (
	// DefaultReadWorkers is the default maximum number of concurrent
	// workers used by the daemon's read pool.
	DefaultReadWorkers = 100

	// DefaultWriteWorkers is the default maximum number of concurrent
	// workers used by the daemon's write pool.
	DefaultWriteWorkers = 100

	// DefaultSigWorkers is the default maximum number of concurrent workers
	// used by the daemon's sig pool.
	DefaultSigWorkers = 8
)

// Workers exposes CLI configuration for turning resources consumed by worker
// pools.
type Workers struct {
	// Read is the maximum number of concurrent read pool workers.
	Read int `long:"read" description:"Maximum number of concurrent read pool workers."`

	// Write is the maximum number of concurrent write pool workers.
	Write int `long:"write" description:"Maximum number of concurrent write pool workers."`

	// Sig is the maximum number of concurrent sig pool workers.
	Sig int `long:"sig" description:"Maximum number of concurrent sig pool workers."`
}

// Validate checks the Workers configuration to ensure that the input values are
// sane.
func (w *Workers) Validate() error {
	if w.Read <= 0 {
		return fmt.Errorf("number of read workers (%d) must be "+
			"positive", w.Read)
	}
	if w.Write <= 0 {
		return fmt.Errorf("number of write workers (%d) must be "+
			"positive", w.Write)
	}
	if w.Sig <= 0 {
		return fmt.Errorf("number of sig workers (%d) must be "+
			"positive", w.Sig)
	}

	return nil
}

// Compile-time constraint to ensure Workers implements the Validator interface.
var _ Validator = (*Workers)(nil)
<commit_msg>lncfg/workers: reduce default write pool workers to 8<commit_after>package lncfg

import "fmt"

const (
	// DefaultReadWorkers is the default maximum number of concurrent
	// workers used by the daemon's read pool.
	DefaultReadWorkers = 100

	// DefaultWriteWorkers is the default maximum number of concurrent
	// workers used by the daemon's write pool.
	DefaultWriteWorkers = 8

	// DefaultSigWorkers is the default maximum number of concurrent workers
	// used by the daemon's sig pool.
	DefaultSigWorkers = 8
)

// Workers exposes CLI configuration for turning resources consumed by worker
// pools.
type Workers struct {
	// Read is the maximum number of concurrent read pool workers.
	Read int `long:"read" description:"Maximum number of concurrent read pool workers. This number should be proportional to the number of peers."`

	// Write is the maximum number of concurrent write pool workers.
	Write int `long:"write" description:"Maximum number of concurrent write pool workers. This number should be proportional to the number of CPUs on the host. "`

	// Sig is the maximum number of concurrent sig pool workers.
	Sig int `long:"sig" description:"Maximum number of concurrent sig pool workers. This number should be proportional to the number of CPUs on the host."`
}

// Validate checks the Workers configuration to ensure that the input values are
// sane.
func (w *Workers) Validate() error {
	if w.Read <= 0 {
		return fmt.Errorf("number of read workers (%d) must be "+
			"positive", w.Read)
	}
	if w.Write <= 0 {
		return fmt.Errorf("number of write workers (%d) must be "+
			"positive", w.Write)
	}
	if w.Sig <= 0 {
		return fmt.Errorf("number of sig workers (%d) must be "+
			"positive", w.Sig)
	}

	return nil
}

// Compile-time constraint to ensure Workers implements the Validator interface.
var _ Validator = (*Workers)(nil)
<|endoftext|>


<commit_before>package main

import (
	info "github.com/google/cadvisor/info/v1"

	"errors"
	"fmt"
	"log"
	"net/http"
	"strings"

	"github.com/google/cadvisor/client"
	"sync"
	"time"
)

var filter = "dummy"                 //the name of the servers the clients want to contact
var monitor = "http://cadvisor:8080" // URL where the cAdvisor container is
var port = ":6666"                   // port we will listen on
var mapContainers = make(map[string]uint64)
var mutex sync.Mutex

func updateContainers() {

	listContainers()

	for _ = range time.Tick(3 * time.Second) {
		log.Println("Starting list of containers update...")
		listContainers()
		log.Print("Done.\n\n")
	}
}

func listContainers() {

	allContainers, _ := getAllContainerInfo(monitor)

	// resetting mapContainers
	mutex.Lock()
	for key := range mapContainers {
		delete(mapContainers, key)
	}

	// filtering data and filling mapContainers
	for _, item := range allContainers {
		alias := item.Aliases[0]
		kbFree := item.Stats[0].Memory.Usage

		if strings.Contains(alias, filter) {
			mapContainers[alias] = kbFree
		}
	}

	log.Println("\tFound", len(mapContainers), "containers")
	for key, value := range mapContainers {
		log.Printf("\t\t%s %d", key, value)
	}
	mutex.Unlock()
}

// determine the container with the more available RAM
func getLessLoaded() (string, error) {

	var lessLoaded string

	mutex.Lock()
	for key := range mapContainers {

		if lessLoaded == "" {
			lessLoaded = key
		} else if mapContainers[key] < mapContainers[lessLoaded] {
			lessLoaded = key
		}
	}
	mutex.Unlock()

	if lessLoaded == "" {
		return "", errors.New("No server found...")
	}
	return lessLoaded, nil
}

func handleRoot(w http.ResponseWriter, r *http.Request) {
	server, err := getLessLoaded()

	if detectError(err, false) {
		w.WriteHeader(500) //warn the user that the server encountered a problem
	} else {
		w.WriteHeader(200)
	}

	w.Write([]byte(server))
}

func getAllContainerInfo(cadvisor string) ([]info.ContainerInfo, error) {

	client, err := client.NewClient(cadvisor)
	if detectError(err, true) {
		return nil, err
	}

	request := info.DefaultContainerInfoRequest()
	allContainers, err := client.AllDockerContainers(&request)
	if detectError(err, true) {
		return nil, err
	}

	return allContainers, nil
}

func detectError(err error, doLog bool) bool {

	if err != nil {
		if doLog {
			log.Println(err)
		}
		return true
	}
	return false
}

func main() {

	go updateContainers()

	http.HandleFunc("/", handleRoot)

	fmt.Println("Listening on http://127.0.0.1" + port)
	log.Fatal(http.ListenAndServe(port, nil))
}
<commit_msg>Utilisation des variables d'environnement<commit_after>package main

import (
	info "github.com/google/cadvisor/info/v1"

	"errors"
	"fmt"
	"log"
	"net/http"
	"strings"

	"github.com/google/cadvisor/client"
	"os"
	"sync"
	"time"
)

// monitor : URL where the cAdvisor container is
// filter  : the name of the servers the clients want to contact
// port    : port we will listen on
var filter, monitor, port string
var mapContainers = make(map[string]uint64)
var mutex sync.Mutex

func updateContainers() {

	listContainers()

	for _ = range time.Tick(3 * time.Second) {
		log.Println("Starting list of containers update...")
		listContainers()
		log.Print("Done.\n\n")
	}
}

func listContainers() {

	allContainers, _ := getAllContainerInfo(monitor)

	// resetting mapContainers
	mutex.Lock()
	for key := range mapContainers {
		delete(mapContainers, key)
	}

	// filtering data and filling mapContainers
	for _, item := range allContainers {
		alias := item.Aliases[0]
		kbFree := item.Stats[0].Memory.Usage

		if strings.Contains(alias, filter) {
			mapContainers[alias] = kbFree
		}
	}

	log.Println("\tFound", len(mapContainers), "containers")
	for key, value := range mapContainers {
		log.Printf("\t\t%s %d", key, value)
	}
	mutex.Unlock()
}

// determine the container with the more available RAM
func getLessLoaded() (string, error) {

	var lessLoaded string

	mutex.Lock()
	for key := range mapContainers {

		if lessLoaded == "" {
			lessLoaded = key
		} else if mapContainers[key] < mapContainers[lessLoaded] {
			lessLoaded = key
		}
	}
	mutex.Unlock()

	if lessLoaded == "" {
		return "", errors.New("No server found...")
	}
	return lessLoaded, nil
}

func handleRoot(w http.ResponseWriter, r *http.Request) {
	server, err := getLessLoaded()

	if detectError(err, false) {
		w.WriteHeader(500) //warn the user that the server encountered a problem
	} else {
		w.WriteHeader(200)
	}

	w.Write([]byte(server))
}

func getAllContainerInfo(cadvisor string) ([]info.ContainerInfo, error) {

	client, err := client.NewClient(cadvisor)
	if detectError(err, true) {
		return nil, err
	}

	request := info.DefaultContainerInfoRequest()
	allContainers, err := client.AllDockerContainers(&request)
	if detectError(err, true) {
		return nil, err
	}

	return allContainers, nil
}

func detectError(err error, doLog bool) bool {

	if err != nil {
		if doLog {
			log.Println(err)
		}
		return true
	}
	return false
}

func main() {

	filter = os.Getenv("FILTER")
	monitor = os.Getenv("MONITOR")
	port = os.Getenv("HTTP_PORT")

	if filter == "" {
		log.Fatalln("FILTER environment variable is missing")
	}
	if monitor == "" {
		log.Fatalln("MONITOR environment variable is missing")
	}

	if port == "" {
		log.Fatalln("HTTP_PORT environment variable is missing")
	}

	go updateContainers()

	http.HandleFunc("/", handleRoot)

	fmt.Println("Listening on http://127.0.0.1" + port)
	log.Fatal(http.ListenAndServe(port, nil))
}
<|endoftext|>


<commit_before>package connectfour

import (
	"fmt"

	"github.com/Sirupsen/logrus"
	"github.com/moul/bolosseum/bots"
	"github.com/moul/bolosseum/games"
)

var pieces = []string{"X", "O"}
var Rows = 6
var Cols = 7

type ConnectfourGame struct {
	games.BotsBasedGame

	board [][]string `json:"board",omitempty`
}

func NewGame() (*ConnectfourGame, error) {
	game := ConnectfourGame{}
	game.Bots = make([]bots.Bot, 0)
	game.board = make([][]string, Rows)
	for i := 0; i < Rows; i++ {
		game.board[i] = make([]string, Cols)
	}
	return &game, nil
}

func (g *ConnectfourGame) CheckArgs(args []string) error {
	if len(args) != 2 {
		return fmt.Errorf("You need to specify 2 bots")
	}
	return nil
}

func (g *ConnectfourGame) checkBoard() (bots.Bot, error) {
	for idx, piece := range pieces {
		// horizontal
		for y := 0; y < Rows; y++ {
			continuous := 0
			for x := 0; x < Cols; x++ {
				if g.board[y][x] == piece {
					continuous++
					if continuous == 4 {
						return g.Bots[idx], nil
					}
				} else {
					continuous = 0
				}
			}
		}

		// vertical
		// FIXME

		// diagonals
		// FIXME
	}
	return nil, nil
}

func (g *ConnectfourGame) Run(gameID string) error {
	if err := bots.InitTurnBasedBots(g.Bots, g.Name(), gameID); err != nil {
		return err
	}

	// play
	for turn := 0; ; turn++ {
		idx := turn % 2
		bot := g.Bots[idx]
		piece := pieces[idx]

		reply, err := bot.SendMessage(bots.QuestionMessage{
			GameID:      gameID,
			Game:        g.Name(),
			Action:      "play-turn",
			Board:       g.board,
			You:         piece,
			PlayerIndex: idx,
		})
		if err != nil {
			return err
		}

		x := int(reply.Play.(float64))
		placed := false
		for y := Rows - 1; y >= 0; y-- {
			if g.board[y][x] == "" {
				g.board[y][x] = piece
				placed = true
				break
			}
			if placed {
				break
			}
		}
		if !placed {
			return fmt.Errorf("Invalid location")
		}

		// check board
		winner, err := g.checkBoard()
		if err != nil {
			return err
		}
		if winner != nil {
			logrus.Warnf("Player %d (%s) won", idx, winner.Name())
			return nil
		}
	}

	logrus.Warnf("Draw")
	return nil
}

func (g *ConnectfourGame) Name() string {
	return "connectfour"
}
<commit_msg>Reverse Y-axis order<commit_after>package connectfour

import (
	"fmt"

	"github.com/Sirupsen/logrus"
	"github.com/moul/bolosseum/bots"
	"github.com/moul/bolosseum/games"
)

var pieces = []string{"X", "O"}
var Rows = 6
var Cols = 7

type ConnectfourGame struct {
	games.BotsBasedGame

	board [][]string `json:"board",omitempty`
}

func NewGame() (*ConnectfourGame, error) {
	game := ConnectfourGame{}
	game.Bots = make([]bots.Bot, 0)
	game.board = make([][]string, Rows)
	for i := 0; i < Rows; i++ {
		game.board[i] = make([]string, Cols)
	}
	return &game, nil
}

func (g *ConnectfourGame) CheckArgs(args []string) error {
	if len(args) != 2 {
		return fmt.Errorf("You need to specify 2 bots")
	}
	return nil
}

func (g *ConnectfourGame) checkBoard() (bots.Bot, error) {
	for idx, piece := range pieces {
		// horizontal
		for y := 0; y < Rows; y++ {
			continuous := 0
			for x := 0; x < Cols; x++ {
				if g.board[y][x] == piece {
					continuous++
					if continuous == 4 {
						return g.Bots[idx], nil
					}
				} else {
					continuous = 0
				}
			}
		}

		// vertical
		// FIXME

		// diagonals
		// FIXME
	}
	return nil, nil
}

func (g *ConnectfourGame) Run(gameID string) error {
	if err := bots.InitTurnBasedBots(g.Bots, g.Name(), gameID); err != nil {
		return err
	}

	// play
	for turn := 0; ; turn++ {
		idx := turn % 2
		bot := g.Bots[idx]
		piece := pieces[idx]

		reply, err := bot.SendMessage(bots.QuestionMessage{
			GameID:      gameID,
			Game:        g.Name(),
			Action:      "play-turn",
			Board:       g.board,
			You:         piece,
			PlayerIndex: idx,
		})
		if err != nil {
			return err
		}

		x := int(reply.Play.(float64))
		placed := false
		for y := 0; y < Rows; y++ {
			if g.board[y][x] == "" {
				g.board[y][x] = piece
				placed = true
				break
			}
			if placed {
				break
			}
		}
		if !placed {
			return fmt.Errorf("Invalid location")
		}

		// check board
		winner, err := g.checkBoard()
		if err != nil {
			return err
		}
		if winner != nil {
			logrus.Warnf("Player %d (%s) won", idx, winner.Name())
			return nil
		}
	}

	logrus.Warnf("Draw")
	return nil
}

func (g *ConnectfourGame) Name() string {
	return "connectfour"
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"os"
	"text/template"

	bloomsky "github.com/patrickalin/bloomsky-api-go"
	"github.com/sirupsen/logrus"
)

type console struct {
	in           chan bloomsky.Bloomsky
	textTemplate *template.Template
}

//InitConsole listen on the chanel
func createConsole(messages chan bloomsky.Bloomsky) (console, error) {
	f := map[string]interface{}{"T": config.translateFunc}
	//Get template
	c := console{in: messages, textTemplate: GetTemplate("bloomsky.txt", "tmpl/bloomsky.txt", f, config.dev)}
	logrus.WithFields(logrus.Fields{
		"fct": "exportConsole.initConsole",
	}).Info("Init console")
	return c, nil
}

func (c *console) listen(context context.Context) {
	go func() {

		/*
			TODO

			g, err := gocui.NewGui(gocui.OutputNormal)
			if err != nil {
				log.Panicln(err)
			}
			defer g.Close()

			g.SetManagerFunc(layout)

			if err := g.SetKeybinding("", gocui.KeyCtrlC, gocui.ModNone, quit); err != nil {
				log.Panicln(err)
			}

			if err := g.MainLoop(); err != nil && err != gocui.ErrQuit {
				log.Panicln(err)
			}*/

		log.WithFields(logrus.Fields{
			"fct": "exportConsole.listen",
		}).Info("Init the queue console to display message")

		for {
			msg := <-c.in

			if err := c.textTemplate.Execute(os.Stdout, msg); err != nil {
				fmt.Printf("%v", err)
			}
		}

	}()

}

/*
func layout(g *gocui.Gui) error {
	maxX, maxY := g.Size()
	if v, err := g.SetView("hello", maxX/2-7, maxY/2, maxX/2+7, maxY/2+2); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		fmt.Fprintln(v, "Hello world!")
	}
	return nil
}

func quit(g *gocui.Gui, v *gocui.View) error {
	return gocui.ErrQuit
}*/
<commit_msg>using context  in coonsole<commit_after>package main

import (
	"context"
	"fmt"
	"os"
	"text/template"

	bloomsky "github.com/patrickalin/bloomsky-api-go"
	"github.com/sirupsen/logrus"
)

type console struct {
	in           chan bloomsky.Bloomsky
	textTemplate *template.Template
}

//InitConsole listen on the chanel
func createConsole(messages chan bloomsky.Bloomsky) (console, error) {
	f := map[string]interface{}{"T": config.translateFunc}
	//Get template
	c := console{in: messages, textTemplate: GetTemplate("bloomsky.txt", "tmpl/bloomsky.txt", f, config.dev)}
	logrus.WithFields(logrus.Fields{
		"fct": "exportConsole.initConsole",
	}).Info("Init console")
	return c, nil
}

func (c *console) listen(context context.Context) {
	go func() {

		/*
			TODO

			g, err := gocui.NewGui(gocui.OutputNormal)
			if err != nil {
				log.Panicln(err)
			}
			defer g.Close()

			g.SetManagerFunc(layout)

			if err := g.SetKeybinding("", gocui.KeyCtrlC, gocui.ModNone, quit); err != nil {
				log.Panicln(err)
			}

			if err := g.MainLoop(); err != nil && err != gocui.ErrQuit {
				log.Panicln(err)
			}*/

		log.WithFields(logrus.Fields{
			"fct": "exportConsole.listen",
		}).Info("Init the queue console to display message")

		for {
			select {
			case msg := <-c.in:
				if err := c.textTemplate.Execute(os.Stdout, msg); err != nil {
					fmt.Printf("%v", err)
				}

			case <-context.Done():
				fmt.Println("console done")
			}
		}
	}()

}

/*
func layout(g *gocui.Gui) error {
	maxX, maxY := g.Size()
	if v, err := g.SetView("hello", maxX/2-7, maxY/2, maxX/2+7, maxY/2+2); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		fmt.Fprintln(v, "Hello world!")
	}
	return nil
}

func quit(g *gocui.Gui, v *gocui.View) error {
	return gocui.ErrQuit
}*/
<|endoftext|>


<commit_before>// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.

// Package bind generates Ethereum contract Go bindings.
//
// Detailed usage document and tutorial available on the go-ethereum Wiki page:
// https://github.com/ethereum/go-ethereum/wiki/Native-DApps:-Go-bindings-to-Ethereum-contracts
package bind

import (
	"bytes"
	"fmt"
	"strings"
	"text/template"
	"unicode"

	"github.com/ethereum/go-ethereum/accounts/abi"
	"golang.org/x/tools/imports"
)

// Bind generates a Go wrapper around a contract ABI. This wrapper isn't meant
// to be used as is in client code, but rather as an intermediate struct which
// enforces compile time type safety and naming convention opposed to having to
// manually maintain hard coded strings that break on runtime.
func Bind(types []string, abis []string, bytecodes []string, pkg string) (string, error) {
	// Process each individual contract requested binding
	contracts := make(map[string]*tmplContract)

	for i := 0; i < len(types); i++ {
		// Parse the actual ABI to generate the binding for
		evmABI, err := abi.JSON(strings.NewReader(abis[i]))
		if err != nil {
			return "", err
		}
		// Strip any whitespace from the JSON ABI
		strippedABI := strings.Map(func(r rune) rune {
			if unicode.IsSpace(r) {
				return -1
			}
			return r
		}, abis[i])

		// Extract the call and transact methods, and sort them alphabetically
		var (
			calls     = make(map[string]*tmplMethod)
			transacts = make(map[string]*tmplMethod)
		)
		for _, original := range evmABI.Methods {
			// Normalize the method for capital cases and non-anonymous inputs/outputs
			normalized := original
			normalized.Name = capitalise(original.Name)

			normalized.Inputs = make([]abi.Argument, len(original.Inputs))
			copy(normalized.Inputs, original.Inputs)
			for j, input := range normalized.Inputs {
				if input.Name == "" {
					normalized.Inputs[j].Name = fmt.Sprintf("arg%d", j)
				}
			}
			normalized.Outputs = make([]abi.Argument, len(original.Outputs))
			copy(normalized.Outputs, original.Outputs)
			for j, output := range normalized.Outputs {
				if output.Name != "" {
					normalized.Outputs[j].Name = capitalise(output.Name)
				}
			}
			// Append the methos to the call or transact lists
			if original.Const {
				calls[original.Name] = &tmplMethod{Original: original, Normalized: normalized, Structured: structured(original)}
			} else {
				transacts[original.Name] = &tmplMethod{Original: original, Normalized: normalized, Structured: structured(original)}
			}
		}
		contracts[types[i]] = &tmplContract{
			Type:        capitalise(types[i]),
			InputABI:    strippedABI,
			InputBin:    strings.TrimSpace(bytecodes[i]),
			Constructor: evmABI.Constructor,
			Calls:       calls,
			Transacts:   transacts,
		}
	}
	// Generate the contract template data content and render it
	data := &tmplData{
		Package:   pkg,
		Contracts: contracts,
	}
	buffer := new(bytes.Buffer)

	funcs := map[string]interface{}{
		"bindtype": bindType,
	}
	tmpl := template.Must(template.New("").Funcs(funcs).Parse(tmplSource))
	if err := tmpl.Execute(buffer, data); err != nil {
		return "", err
	}
	// Pass the code through goimports to clean it up and double check
	code, err := imports.Process("", buffer.Bytes(), nil)
	if err != nil {
		return "", fmt.Errorf("%v\n%s", err, buffer)
	}
	return string(code), nil
}

// bindType converts a Solidity type to a Go one. Since there is no clear mapping
// from all Solidity types to Go ones (e.g. uint17), those that cannot be exactly
// mapped will use an upscaled type (e.g. *big.Int).
func bindType(kind abi.Type) string {
	stringKind := kind.String()

	switch {
	case stringKind == "address":
		return "common.Address"

	case stringKind == "hash":
		return "common.Hash"

	case strings.HasPrefix(stringKind, "bytes"):
		if stringKind == "bytes" {
			return "[]byte"
		}
		return fmt.Sprintf("[%s]byte", stringKind[5:])

	case strings.HasPrefix(stringKind, "int"):
		switch stringKind[:3] {
		case "8", "16", "32", "64":
			return stringKind
		}
		return "*big.Int"

	case strings.HasPrefix(stringKind, "uint"):
		switch stringKind[:4] {
		case "8", "16", "32", "64":
			return stringKind
		}
		return "*big.Int"

	default:
		return stringKind
	}
}

// capitalise makes the first character of a string upper case.
func capitalise(input string) string {
	return strings.ToUpper(input[:1]) + input[1:]
}

// structured checks whether a method has enough information to return a proper
// Go struct ot if flat returns are needed.
func structured(method abi.Method) bool {
	if len(method.Outputs) < 2 {
		return false
	}
	for _, out := range method.Outputs {
		if out.Name == "" {
			return false
		}
	}
	return true
}
<commit_msg>accounts/abi/bind: support account slices, drop hash type<commit_after>// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.

// Package bind generates Ethereum contract Go bindings.
//
// Detailed usage document and tutorial available on the go-ethereum Wiki page:
// https://github.com/ethereum/go-ethereum/wiki/Native-DApps:-Go-bindings-to-Ethereum-contracts
package bind

import (
	"bytes"
	"fmt"
	"strings"
	"text/template"
	"unicode"

	"github.com/ethereum/go-ethereum/accounts/abi"
	"golang.org/x/tools/imports"
)

// Bind generates a Go wrapper around a contract ABI. This wrapper isn't meant
// to be used as is in client code, but rather as an intermediate struct which
// enforces compile time type safety and naming convention opposed to having to
// manually maintain hard coded strings that break on runtime.
func Bind(types []string, abis []string, bytecodes []string, pkg string) (string, error) {
	// Process each individual contract requested binding
	contracts := make(map[string]*tmplContract)

	for i := 0; i < len(types); i++ {
		// Parse the actual ABI to generate the binding for
		evmABI, err := abi.JSON(strings.NewReader(abis[i]))
		if err != nil {
			return "", err
		}
		// Strip any whitespace from the JSON ABI
		strippedABI := strings.Map(func(r rune) rune {
			if unicode.IsSpace(r) {
				return -1
			}
			return r
		}, abis[i])

		// Extract the call and transact methods, and sort them alphabetically
		var (
			calls     = make(map[string]*tmplMethod)
			transacts = make(map[string]*tmplMethod)
		)
		for _, original := range evmABI.Methods {
			// Normalize the method for capital cases and non-anonymous inputs/outputs
			normalized := original
			normalized.Name = capitalise(original.Name)

			normalized.Inputs = make([]abi.Argument, len(original.Inputs))
			copy(normalized.Inputs, original.Inputs)
			for j, input := range normalized.Inputs {
				if input.Name == "" {
					normalized.Inputs[j].Name = fmt.Sprintf("arg%d", j)
				}
			}
			normalized.Outputs = make([]abi.Argument, len(original.Outputs))
			copy(normalized.Outputs, original.Outputs)
			for j, output := range normalized.Outputs {
				if output.Name != "" {
					normalized.Outputs[j].Name = capitalise(output.Name)
				}
			}
			// Append the methos to the call or transact lists
			if original.Const {
				calls[original.Name] = &tmplMethod{Original: original, Normalized: normalized, Structured: structured(original)}
			} else {
				transacts[original.Name] = &tmplMethod{Original: original, Normalized: normalized, Structured: structured(original)}
			}
		}
		contracts[types[i]] = &tmplContract{
			Type:        capitalise(types[i]),
			InputABI:    strippedABI,
			InputBin:    strings.TrimSpace(bytecodes[i]),
			Constructor: evmABI.Constructor,
			Calls:       calls,
			Transacts:   transacts,
		}
	}
	// Generate the contract template data content and render it
	data := &tmplData{
		Package:   pkg,
		Contracts: contracts,
	}
	buffer := new(bytes.Buffer)

	funcs := map[string]interface{}{
		"bindtype": bindType,
	}
	tmpl := template.Must(template.New("").Funcs(funcs).Parse(tmplSource))
	if err := tmpl.Execute(buffer, data); err != nil {
		return "", err
	}
	// Pass the code through goimports to clean it up and double check
	code, err := imports.Process("", buffer.Bytes(), nil)
	if err != nil {
		return "", fmt.Errorf("%v\n%s", err, buffer)
	}
	return string(code), nil
}

// bindType converts a Solidity type to a Go one. Since there is no clear mapping
// from all Solidity types to Go ones (e.g. uint17), those that cannot be exactly
// mapped will use an upscaled type (e.g. *big.Int).
func bindType(kind abi.Type) string {
	stringKind := kind.String()

	switch {
	case stringKind == "address":
		return "common.Address"

	case stringKind == "address[]":
		return "[]common.Address"

	case strings.HasPrefix(stringKind, "bytes"):
		if stringKind == "bytes" {
			return "[]byte"
		}
		return fmt.Sprintf("[%s]byte", stringKind[5:])

	case strings.HasPrefix(stringKind, "int"):
		switch stringKind[:3] {
		case "8", "16", "32", "64":
			return stringKind
		}
		return "*big.Int"

	case strings.HasPrefix(stringKind, "uint"):
		switch stringKind[:4] {
		case "8", "16", "32", "64":
			return stringKind
		}
		return "*big.Int"

	default:
		return stringKind
	}
}

// capitalise makes the first character of a string upper case.
func capitalise(input string) string {
	return strings.ToUpper(input[:1]) + input[1:]
}

// structured checks whether a method has enough information to return a proper
// Go struct ot if flat returns are needed.
func structured(method abi.Method) bool {
	if len(method.Outputs) < 2 {
		return false
	}
	for _, out := range method.Outputs {
		if out.Name == "" {
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"code.google.com/p/go-uuid/uuid"
	"encoding/json"
	"fmt"
	"github.com/oschwald/geoip2-golang"
	"net"
	"net/url"
	"os"
	"path"
	"regexp"
	"strconv"
	"strings"
	"time"
)

type HostLogFile struct {
	Host    string
	Index   string
	Path    string
	Lines   int
	Created time.Time
	Output  chan *HostLogFile
	Buffer  []string
}

type LogFileParser struct {
	Id           string
	tmpDir       string
	tmpHostFiles map[string]*HostLogFile
	Output       chan *HostLogFile
	GeoipReader  *geoip2.Reader
	config       map[string]interface{}
}

func NewLogFileParser(output chan *HostLogFile, geoipreader *geoip2.Reader, config map[string]interface{}) *LogFileParser {
	tmpdir := config["parser"].(map[string]interface{})["tmpdir"].(string)
	id := uuid.New()
	a := &LogFileParser{tmpDir: tmpdir, tmpHostFiles: map[string]*HostLogFile{}, Output: output, Id: id, GeoipReader: geoipreader, config: config}
	os.MkdirAll(a.tmpDir, 0700)
	return a
}

type ISP struct {
	Name         string
	Organization string
}

type Location struct {
	IsoCode string
	Name    string
}

type IndexableLogFile struct {
	Id            string            `json:"_id,omitempty"`
	Timestamp     string            `json:"@timestamp"`
	Host          string            `json:"host,omitempty"`
	IP            string            `json:"ip,omitempty"`
	Path          string            `json:"path,omitempty"`
	Query         map[string]string `json:"query,omitempty"`
	Verb          string            `json:"verb,omitempty"`
	Status        int               `json:"status"`
	RequestBytes  int               `json:"request_bytes"`
	ResponseBytes int               `json:"response_bytes"`
	ResponseTime  int               `json:"response_time"`
	UserAgent     string            `json:"user_agent,omitempty"`
	City          string            `json:"city, omitempty"`
	Country       Location          `json:"country, omitempty"`
	Continent     Location          `json:"continent, omitempty"`
	Location      interface{}       `json:"location, omitempty"`
	ISP           ISP               `json:"isp, omitempty"`
	Coordinates   string            `json:"coordinates,omitempty"`
}

type RawAccessLogLine struct {
	Host           string `json:"host,omitempty"`
	ForwardedFor   string `json:"http_x_forwarded_for"`
	LocalTime      string `json:"time_local"`
	Request        string `json:"request"`
	StatusCode     string `json:"status"`
	RequestLength  string `json:"request_length"`
	ResponseLength string `json:"bytes_sent"`
	UserAgent      string `json:"user_agent"`
	ReponseTime    string `json:"request_time"`
}

func (line *RawAccessLogLine) ToIndexable(id string, georeader *geoip2.Reader) (*IndexableLogFile, error) {

	timestamp, err := time.Parse("02/Jan/2006:15:04:05 -0700", line.LocalTime)
	if err != nil {
		errLogger.Printf("%v\n", err)
		panic(err)
	}

	s := strings.Split(line.Request, " ")
	verb := ""
	path := ""
	query := ""
	if len(s) > 2 {
		verb = s[0]
		path = s[1]
	} else {
		path = s[0]
	}

	if strings.Index(path, "?") >= 0 {
		query = path[strings.Index(path, "?")+1:]
		path = path[0:strings.Index(path, "?")]
	}

	status, err := strconv.Atoi(line.StatusCode)
	if err != nil {
		return nil, err
	}

	reqBytes, err := strconv.Atoi(line.RequestLength)
	if err != nil {
		return nil, err
	}

	respBytes, err := strconv.Atoi(line.ResponseLength)
	if err != nil {
		return nil, err
	}

	respTimef, err := strconv.ParseFloat(line.ReponseTime, 64)
	if err != nil {
		return nil, err
	}
	respTime := int(respTimef * 1000)

	ips := strings.Split(line.ForwardedFor, ",")

	if len(ips) <= 0 {
		return nil, fmt.Errorf("no ip found: %v", line)
	}

	ip := strings.Trim(ips[len(ips)-1], " ")

	//log.Printf("IP: %s", ip)

	ispinfo, _ := georeader.ISP(net.ParseIP(ip))
	location, _ := georeader.City(net.ParseIP(ip))

	queryMap := parseQueryString(query)

	getOrDefault := func(m map[string]string, key string, def string) string {
		if val, exists := m["en"]; exists {
			return val
		} else {
			return def
		}
	}

	return &IndexableLogFile{
		Id:            id,
		Coordinates:   fmt.Sprintf("%v,%v", location.Location.Latitude, location.Location.Longitude),
		ISP:           ISP{ispinfo.ISP, ispinfo.Organization},
		Continent:     Location{location.Continent.Code, getOrDefault(location.Continent.Names, "en", "unknown")},
		Country:       Location{location.Country.IsoCode, getOrDefault(location.Country.Names, "en", "unknown")},
		City:          getOrDefault(location.City.Names, "en", "unknown"),
		Location:      location.Location,
		Host:          strings.ToLower(line.Host),
		IP:            strings.ToLower(line.ForwardedFor),
		Timestamp:     timestamp.UTC().Format(time.RFC3339),
		Path:          strings.ToLower(path),
		Query:         queryMap,
		Verb:          strings.ToUpper(verb),
		Status:        status,
		RequestBytes:  reqBytes,
		ResponseBytes: respBytes,
		ResponseTime:  respTime,
		UserAgent:     strings.ToLower(line.UserAgent),
	}, nil

}

func (parser *LogFileParser) Watch(fileChannel chan string) {

	threads := make(chan int, 8)
	run := func(file string, done chan int, parser *LogFileParser) {
		time.Sleep(2 * time.Second)
		p := NewLogFileParser(parser.Output, parser.GeoipReader, parser.config)
		defer func() {
			p.Flush()
			if err := os.Remove(file); err != nil {
				errLogger.Printf("unable to delete file: %v, error: %v", err)
			}
			<-done
		}()
		if err := p.ParseFile(file); err != nil {
			errLogger.Printf("unbale to parse file: %v, error: %v", file, err)
		}
	}

	timeout := make(chan int, 1)

	for {
		go func() {
			time.Sleep(1 * time.Second)
			timeout <- 1
		}()
		select {
		case file := <-fileChannel:
			threads <- 1
			infoLogger.Printf("got file %s", file)
			go run(file, threads, parser)
			break
		case <-timeout:
			break
		}
	}
}

func (parser *LogFileParser) ParseFile(filePath string) error {

	infoLogger.Printf("parsing file %s", filePath)

	file, err := os.Open(filePath)
	if err != nil {
		errLogger.Printf("opening file: %s, error: %v", filePath, err)
		return err
	}
	defer file.Close()

	linenumber := 0
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {

		linenumber++

		if err := scanner.Err(); err != nil {
			errLogger.Printf("reading file: %s, error: %v", filePath, err)
			return err
		}

		line := scanner.Text()

		data, err := parser.ParseLine(line, filePath, linenumber)
		if err != nil {
			errLogger.Printf("parsing line: %s, error: %v", line, err)
			continue
		}

		data.Host = strings.ToLower(strings.TrimSpace(data.Host))

		parser.Store(data)

		//fmt.Println(scanner.Text()) // Println will add back the final '\n'
	}

	return nil
}

func (parser *LogFileParser) Flush() {
	infoLogger.Printf("flushing: %v", parser.Id)
	for _, value := range parser.tmpHostFiles {
		infoLogger.Printf("flushing file '%s'", value.Path)
		value.Flush()
		parser.Output <- value
	}
}

func (parser *LogFileParser) Store(logfile *IndexableLogFile) error {
	jsonBytes, err := json.Marshal(logfile)
	if err != nil {
		return nil
	}

	if v, exists := parser.tmpHostFiles[logfile.Index()]; !exists {
		parser.tmpHostFiles[logfile.Index()] = parser.NewTmpFile(logfile)
	} else if v.Lines > 20000 {
		infoLogger.Printf("purging file '%s'", v.Path)
		v.Flush()
		parser.Output <- v
		parser.tmpHostFiles[logfile.Index()] = parser.NewTmpFile(logfile)
	}

	line1map := map[string]interface{}{
		"index": map[string]interface{}{
			"_id": logfile.Id,
		},
	}
	line1mapbytes, _ := json.Marshal(line1map)
	if err := parser.tmpHostFiles[logfile.Index()].Append(string(line1mapbytes)); err != nil {
		errLogger.Println(err.Error())
		return err
	}
	if err := parser.tmpHostFiles[logfile.Index()].Append(string(jsonBytes)); err != nil {
		errLogger.Println(err.Error())
		return err
	}
	return nil
}

func (appender *HostLogFile) Append(line string) error {
	appender.Buffer = append(appender.Buffer, line)
	appender.Lines++
	if len(appender.Buffer) > 20000 {
		appender.Flush()
	}
	return nil
}

func (appender *HostLogFile) Flush() error {

	infoLogger.Printf("flushing %v lines -> %s", len(appender.Buffer), appender.Path)

	f, err := os.OpenFile(appender.Path, os.O_APPEND|os.O_CREATE|os.O_RDWR, 0600)
	if err != nil {
		return err
	}
	defer f.Close()

	defer func() {
		appender.Buffer = make([]string, 0)
	}()

	for _, line := range appender.Buffer {
		if _, err = f.WriteString(line + "\n"); err != nil {
			return err
		}
	}

	return nil
}

func (logfile *IndexableLogFile) Index() string {
	logtime, _ := time.Parse(time.RFC3339, logfile.Timestamp)
	index := fmt.Sprintf("accesslogs.%s", logtime.Format("2006.01.02"))
	return index
}

func (parser *LogFileParser) NewTmpFile(logfile *IndexableLogFile) *HostLogFile {
	file := path.Join(parser.tmpDir, fmt.Sprintf("%s_%s_%s_%v.log", strings.ToLower(strings.TrimSpace(logfile.Host)), logfile.Index(), parser.Id, time.Now().Unix()))
	return &HostLogFile{Path: file, Lines: 0, Created: time.Now(), Host: logfile.Host, Index: logfile.Index(), Buffer: []string{}}
}

func (parser *LogFileParser) ParseLine(line string, filename string, linenumber int) (*IndexableLogFile, error) {

	if strings.LastIndex(line, ",") == len(line)-1 {
		line = line[0 : len(line)-1]
	}

	rawLogEntry := &RawAccessLogLine{}
	err := json.Unmarshal([]byte(line), rawLogEntry)

	if err != nil {
		errLogger.Printf("unable to parse json '%s', err: %v", line, err)
		return nil, err
	}

	idRegexp := regexp.MustCompile("[^0-9a-zA-Z]+")
	id := idRegexp.ReplaceAllString(fmt.Sprintf("%v:%v", filename, linenumber), "_")
	converted, err := rawLogEntry.ToIndexable(id, parser.GeoipReader)

	if err != nil {
		errLogger.Printf("%s => %v", line, err)
		return nil, err
	}

	//index(converted)

	return converted, nil
}

func parseQueryString(str string) map[string]string {
	str = strings.Replace(str, "\u0026", "&", -1)
	list := strings.Split(str, "&")
	m := map[string]string{}
	for _, pair := range list {
		if strings.Index(pair, "=") < 0 {
			m[pair] = ""
			continue
		}
		key := pair[0:strings.Index(pair, "=")]
		value, _ := url.QueryUnescape(pair[strings.Index(pair, "=")+1:])
		m[key] = value
	}
	return m
}
<commit_msg>hffgh<commit_after>package main

import (
	"bufio"
	"code.google.com/p/go-uuid/uuid"
	"encoding/json"
	"fmt"
	"github.com/oschwald/geoip2-golang"
	"net"
	"net/url"
	"os"
	"path"
	"regexp"
	"strconv"
	"strings"
	"time"
)

type HostLogFile struct {
	Host    string
	Index   string
	Path    string
	Lines   int
	Created time.Time
	Output  chan *HostLogFile
	Buffer  []string
}

type LogFileParser struct {
	Id           string
	tmpDir       string
	tmpHostFiles map[string]*HostLogFile
	Output       chan *HostLogFile
	GeoipReader  *geoip2.Reader
	config       map[string]interface{}
}

func NewLogFileParser(output chan *HostLogFile, geoipreader *geoip2.Reader, config map[string]interface{}) *LogFileParser {
	tmpdir := config["parser"].(map[string]interface{})["tmpdir"].(string)
	id := uuid.New()
	a := &LogFileParser{tmpDir: tmpdir, tmpHostFiles: map[string]*HostLogFile{}, Output: output, Id: id, GeoipReader: geoipreader, config: config}
	os.MkdirAll(a.tmpDir, 0700)
	return a
}

type ISP struct {
	Name         string
	Organization string
}

type Location struct {
	IsoCode string
	Name    string
}

type IndexableLogFile struct {
	Id            string            `json:"_id,omitempty"`
	Timestamp     string            `json:"@timestamp"`
	Host          string            `json:"host,omitempty"`
	IP            string            `json:"ip,omitempty"`
	Path          string            `json:"path,omitempty"`
	Query         map[string]string `json:"query,omitempty"`
	Verb          string            `json:"verb,omitempty"`
	Status        int               `json:"status"`
	RequestBytes  int               `json:"request_bytes"`
	ResponseBytes int               `json:"response_bytes"`
	ResponseTime  int               `json:"response_time"`
	UserAgent     string            `json:"user_agent,omitempty"`
	City          string            `json:"city, omitempty"`
	Country       Location          `json:"country, omitempty"`
	Continent     Location          `json:"continent, omitempty"`
	Location      interface{}       `json:"location, omitempty"`
	ISP           ISP               `json:"isp, omitempty"`
	Coordinates   string            `json:"coordinates,omitempty"`
}

type RawAccessLogLine struct {
	Host           string `json:"host,omitempty"`
	ForwardedFor   string `json:"http_x_forwarded_for"`
	LocalTime      string `json:"time_local"`
	Request        string `json:"request"`
	StatusCode     string `json:"status"`
	RequestLength  string `json:"request_length"`
	ResponseLength string `json:"bytes_sent"`
	UserAgent      string `json:"user_agent"`
	ReponseTime    string `json:"request_time"`
}

func (line *RawAccessLogLine) ToIndexable(id string, georeader *geoip2.Reader) (*IndexableLogFile, error) {

	timestamp, err := time.Parse("02/Jan/2006:15:04:05 -0700", line.LocalTime)
	if err != nil {
		errLogger.Printf("%v\n", err)
		panic(err)
	}

	s := strings.Split(line.Request, " ")
	verb := ""
	path := ""
	query := ""
	if len(s) > 2 {
		verb = s[0]
		path = s[1]
	} else {
		path = s[0]
	}

	if strings.Index(path, "?") >= 0 {
		query = path[strings.Index(path, "?")+1:]
		path = path[0:strings.Index(path, "?")]
	}

	status, err := strconv.Atoi(line.StatusCode)
	if err != nil {
		return nil, err
	}

	reqBytes, err := strconv.Atoi(line.RequestLength)
	if err != nil {
		return nil, err
	}

	respBytes, err := strconv.Atoi(line.ResponseLength)
	if err != nil {
		return nil, err
	}

	respTimef, err := strconv.ParseFloat(line.ReponseTime, 64)
	if err != nil {
		return nil, err
	}
	respTime := int(respTimef * 1000)

	ips := strings.Split(line.ForwardedFor, ",")

	if len(ips) <= 0 {
		return nil, fmt.Errorf("no ip found: %v", line)
	}

	ip := strings.Trim(ips[len(ips)-1], " ")

	//log.Printf("IP: %s", ip)

	ispinfo, _ := georeader.ISP(net.ParseIP(ip))
	location, _ := georeader.City(net.ParseIP(ip))

	queryMap := parseQueryString(query)

	getOrDefault := func(m map[string]string, key string, def string) string {
		if val, exists := m["en"]; exists {
			return val
		} else {
			return def
		}
	}

	return &IndexableLogFile{
		Id:            id,
		Coordinates:   fmt.Sprintf("%v,%v", location.Location.Latitude, location.Location.Longitude),
		ISP:           ISP{ispinfo.ISP, ispinfo.Organization},
		Continent:     Location{location.Continent.Code, getOrDefault(location.Continent.Names, "en", "unknown")},
		Country:       Location{location.Country.IsoCode, getOrDefault(location.Country.Names, "en", "unknown")},
		City:          getOrDefault(location.City.Names, "en", "unknown"),
		Location:      location.Location,
		Host:          strings.ToLower(line.Host),
		IP:            strings.ToLower(line.ForwardedFor),
		Timestamp:     timestamp.UTC().Format(time.RFC3339),
		Path:          strings.ToLower(path),
		Query:         queryMap,
		Verb:          strings.ToUpper(verb),
		Status:        status,
		RequestBytes:  reqBytes,
		ResponseBytes: respBytes,
		ResponseTime:  respTime,
		UserAgent:     strings.ToLower(line.UserAgent),
	}, nil

}

func (parser *LogFileParser) Watch(fileChannel chan string) {

	threads := make(chan int, 8)
	run := func(file string, done chan int, parser *LogFileParser) {
		time.Sleep(2 * time.Second)
		p := NewLogFileParser(parser.Output, parser.GeoipReader, parser.config)
		defer func() {
			p.Flush()
			if err := os.Remove(file); err != nil {
				errLogger.Printf("unable to delete file: %v, error: %v", file, err)
			}
			<-done
		}()
		if err := p.ParseFile(file); err != nil {
			errLogger.Printf("unbale to parse file: %v, error: %v", file, err)
		}
	}
	for {
		select {
		case file := <-fileChannel:
			threads <- 1
			infoLogger.Printf("got file %s", file)
			go run(file, threads, parser)
			break
		}
	}
}

func (parser *LogFileParser) ParseFile(filePath string) error {

	infoLogger.Printf("parsing file %s", filePath)

	file, err := os.Open(filePath)
	if err != nil {
		errLogger.Printf("opening file: %s, error: %v", filePath, err)
		return err
	}
	defer file.Close()

	linenumber := 0
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {

		linenumber++

		if err := scanner.Err(); err != nil {
			errLogger.Printf("reading file: %s, error: %v", filePath, err)
			return err
		}

		line := scanner.Text()

		data, err := parser.ParseLine(line, filePath, linenumber)
		if err != nil {
			errLogger.Printf("parsing line: %s, error: %v", line, err)
			continue
		}

		data.Host = strings.ToLower(strings.TrimSpace(data.Host))

		parser.Store(data)

		//fmt.Println(scanner.Text()) // Println will add back the final '\n'
	}

	return nil
}

func (parser *LogFileParser) Flush() {
	infoLogger.Printf("flushing: %v", parser.Id)
	for _, value := range parser.tmpHostFiles {
		infoLogger.Printf("flushing file '%s'", value.Path)
		value.Flush()
		parser.Output <- value
	}
}

func (parser *LogFileParser) Store(logfile *IndexableLogFile) error {
	jsonBytes, err := json.Marshal(logfile)
	if err != nil {
		return nil
	}

	if v, exists := parser.tmpHostFiles[logfile.Index()]; !exists {
		parser.tmpHostFiles[logfile.Index()] = parser.NewTmpFile(logfile)
	} else if v.Lines > 20000 {
		infoLogger.Printf("purging file '%s'", v.Path)
		v.Flush()
		parser.Output <- v
		parser.tmpHostFiles[logfile.Index()] = parser.NewTmpFile(logfile)
	}

	line1map := map[string]interface{}{
		"index": map[string]interface{}{
			"_id": logfile.Id,
		},
	}
	line1mapbytes, _ := json.Marshal(line1map)
	if err := parser.tmpHostFiles[logfile.Index()].Append(string(line1mapbytes)); err != nil {
		errLogger.Println(err.Error())
		return err
	}
	if err := parser.tmpHostFiles[logfile.Index()].Append(string(jsonBytes)); err != nil {
		errLogger.Println(err.Error())
		return err
	}
	return nil
}

func (appender *HostLogFile) Append(line string) error {
	appender.Buffer = append(appender.Buffer, line)
	appender.Lines++
	if len(appender.Buffer) > 20000 {
		appender.Flush()
	}
	return nil
}

func (appender *HostLogFile) Flush() error {

	infoLogger.Printf("flushing %v lines -> %s", len(appender.Buffer), appender.Path)

	f, err := os.OpenFile(appender.Path, os.O_APPEND|os.O_CREATE|os.O_RDWR, 0600)
	if err != nil {
		return err
	}
	defer f.Close()

	defer func() {
		appender.Buffer = make([]string, 0)
	}()

	for _, line := range appender.Buffer {
		if _, err = f.WriteString(line + "\n"); err != nil {
			return err
		}
	}

	return nil
}

func (logfile *IndexableLogFile) Index() string {
	logtime, _ := time.Parse(time.RFC3339, logfile.Timestamp)
	index := fmt.Sprintf("accesslogs.%s", logtime.Format("2006.01.02"))
	return index
}

func (parser *LogFileParser) NewTmpFile(logfile *IndexableLogFile) *HostLogFile {
	file := path.Join(parser.tmpDir, fmt.Sprintf("%s_%s_%s_%v.log", strings.ToLower(strings.TrimSpace(logfile.Host)), logfile.Index(), parser.Id, time.Now().Unix()))
	return &HostLogFile{Path: file, Lines: 0, Created: time.Now(), Host: logfile.Host, Index: logfile.Index(), Buffer: []string{}}
}

func (parser *LogFileParser) ParseLine(line string, filename string, linenumber int) (*IndexableLogFile, error) {

	if strings.LastIndex(line, ",") == len(line)-1 {
		line = line[0 : len(line)-1]
	}

	rawLogEntry := &RawAccessLogLine{}
	err := json.Unmarshal([]byte(line), rawLogEntry)

	if err != nil {
		errLogger.Printf("unable to parse json '%s', err: %v", line, err)
		return nil, err
	}

	idRegexp := regexp.MustCompile("[^0-9a-zA-Z]+")
	id := idRegexp.ReplaceAllString(fmt.Sprintf("%v:%v", filename, linenumber), "_")
	converted, err := rawLogEntry.ToIndexable(id, parser.GeoipReader)

	if err != nil {
		errLogger.Printf("%s => %v", line, err)
		return nil, err
	}

	//index(converted)

	return converted, nil
}

func parseQueryString(str string) map[string]string {
	str = strings.Replace(str, "\u0026", "&", -1)
	list := strings.Split(str, "&")
	m := map[string]string{}
	for _, pair := range list {
		if strings.Index(pair, "=") < 0 {
			m[pair] = ""
			continue
		}
		key := pair[0:strings.Index(pair, "=")]
		value, _ := url.QueryUnescape(pair[strings.Index(pair, "=")+1:])
		m[key] = value
	}
	return m
}
<|endoftext|>


<commit_before>package logger

import (
	"runtime"
	"strings"

	"github.com/ttacon/chalk"
)

// ColorMap - Used to map a particular color to a cf status phrase - returns lowercase strings in color.
func (l *Logger) ColorMap(s string) string {

	// If Windows, disable colorS
	if runtime.GOOS == "windows" || *l.Colors {
		return strings.ToLower(s)
	}

	v := strings.Split(s, "_")[len(strings.Split(s, "_"))-1]

	var result string

	switch v {
	case "COMPLETE":
		result = chalk.Green.Color(s)
	case "PROGRESS":
		result = chalk.Yellow.Color(s)
	case "FAILED":
		result = chalk.Red.Color(s)
	case "SKIPPED":
		result = chalk.Blue.Color(s)
	default:
		// Unidentified, just returns the same string
		return strings.ToLower(s)
	}
	return strings.ToLower(result)
}

// ColorString - Returns colored string
func (l *Logger) ColorString(s, color string) string {

	// If Windows, disable colorS
	if runtime.GOOS == "windows" || *l.Colors {
		return s
	}

	var result string
	switch strings.ToLower(color) {
	case "green":
		result = chalk.Green.Color(s)
	case "yellow":
		result = chalk.Yellow.Color(s)
	case "red":
		result = chalk.Red.Color(s)
	case "magenta":
		result = chalk.Magenta.Color(s)
	default:
		// Unidentified, just returns the same string
		return s
	}

	return result
}
<commit_msg>added cyan to colorString<commit_after>package logger

import (
	"runtime"
	"strings"

	"github.com/ttacon/chalk"
)

// ColorMap - Used to map a particular color to a cf status phrase - returns lowercase strings in color.
func (l *Logger) ColorMap(s string) string {

	// If Windows, disable colorS
	if runtime.GOOS == "windows" || *l.Colors {
		return strings.ToLower(s)
	}

	v := strings.Split(s, "_")[len(strings.Split(s, "_"))-1]

	var result string

	switch v {
	case "COMPLETE":
		result = chalk.Green.Color(s)
	case "PROGRESS":
		result = chalk.Yellow.Color(s)
	case "FAILED":
		result = chalk.Red.Color(s)
	case "SKIPPED":
		result = chalk.Blue.Color(s)
	default:
		// Unidentified, just returns the same string
		return strings.ToLower(s)
	}
	return strings.ToLower(result)
}

// ColorString - Returns colored string
func (l *Logger) ColorString(s, color string) string {

	// If Windows, disable colorS
	if runtime.GOOS == "windows" || *l.Colors {
		return s
	}

	var result string
	switch strings.ToLower(color) {
	case "green":
		result = chalk.Green.Color(s)
	case "yellow":
		result = chalk.Yellow.Color(s)
	case "red":
		result = chalk.Red.Color(s)
	case "magenta":
		result = chalk.Magenta.Color(s)
	case "cyan":
		result = chalk.Cyan.Color(s)
	default:
		// Unidentified, just returns the same string
		return s
	}

	return result
}
<|endoftext|>


<commit_before>package logger

import (
	"io"
	"log"
	"os"
	"strings"

	kv "gopkg.in/Clever/kayvee-go.v5"
	"gopkg.in/Clever/kayvee-go.v5/router"
)

/////////////////////
//
//  Helper definitions
//
/////////////////////

// Formatter is a function type that takes a map and returns a formatted string with the contents of the map
type Formatter func(data map[string]interface{}) string

// M is a convenience type for passing data into a log message.
type M map[string]interface{}

// LogLevel is an enum is used to denote level of logging
type LogLevel int

// Constants used to define different LogLevels supported
const (
	Debug LogLevel = iota
	Info
	Warning
	Error
	Critical
)

var logLevelNames = map[LogLevel]string{
	Debug:    "debug",
	Info:     "info",
	Warning:  "warning",
	Error:    "error",
	Critical: "critical",
}

func (l LogLevel) String() string {
	switch l {
	case Debug:
		fallthrough
	case Info:
		fallthrough
	case Warning:
		fallthrough
	case Error:
		fallthrough
	case Critical:
		return logLevelNames[l]
	}
	return ""
}

/////////////////////////////
//
//	Logger
//
/////////////////////////////

// Logger is the default implementation of KayveeLogger.
// It provides customization of globals, default log level, formatting, and output destination.
type Logger struct {
	globals   map[string]interface{}
	logLvl    LogLevel
	formatter Formatter
	logWriter *log.Logger
	logRouter router.Router
}

// SetConfig allows configuration changes in one go
var reservedKeyNames = map[string]bool{
	"title":   true,
	"source":  true,
	"value":   true,
	"type":    true,
	"level":   true,
	"_kvmeta": true,
}

// SetConfig implements the method for the KayveeLogger interface.
func (l *Logger) SetConfig(source string, logLvl LogLevel, formatter Formatter, output io.Writer) {
	if l.globals == nil {
		l.globals = make(map[string]interface{})
	}
	l.globals["source"] = source
	l.logLvl = logLvl
	l.formatter = formatter
	l.logWriter = log.New(output, "", 0) // No prefixes
}

// AddContext implements the method for the KayveeLogger interface.
func (l *Logger) AddContext(key, val string) {
	updateContextMapIfNotReserved(l.globals, key, val)
}

// SetLogLevel implements the method for the KayveeLogger interface.
func (l *Logger) SetLogLevel(logLvl LogLevel) {
	l.logLvl = logLvl
}

// SetFormatter implements the method for the KayveeLogger interface.
func (l *Logger) SetFormatter(formatter Formatter) {
	l.formatter = formatter
}

// SetOutput implements the method for the KayveeLogger interface.
func (l *Logger) SetOutput(output io.Writer) {
	l.logWriter = log.New(output, "", 0) // No prefixes
}

// Debug implements the method for the KayveeLogger interface.
func (l *Logger) Debug(title string) {
	l.DebugD(title, M{})
}

// Info implements the method for the KayveeLogger interface.
func (l *Logger) Info(title string) {
	l.InfoD(title, M{})
}

// Warn implements the method for the KayveeLogger interface.
func (l *Logger) Warn(title string) {
	l.WarnD(title, M{})
}

// Error implements the method for the KayveeLogger interface.
func (l *Logger) Error(title string) {
	l.ErrorD(title, M{})
}

// Critical implements the method for the KayveeLogger interface.
func (l *Logger) Critical(title string) {
	l.CriticalD(title, M{})
}

// Counter implements the method for the KayveeLogger interface.
// Logs with type = gauge, and value = value
func (l *Logger) Counter(title string) {
	l.CounterD(title, 1, M{}) // Defaults to a value of 1
}

// GaugeInt implements the method for the KayveeLogger interface.
// Logs with type = gauge, and value = value
func (l *Logger) GaugeInt(title string, value int) {
	l.GaugeIntD(title, value, M{})
}

// GaugeFloat implements the method for the KayveeLogger interface.
// Logs with type = gauge, and value = value
func (l *Logger) GaugeFloat(title string, value float64) {
	l.GaugeFloatD(title, value, M{})
}

// DebugD implements the method for the KayveeLogger interface.
func (l *Logger) DebugD(title string, data map[string]interface{}) {
	data["title"] = title
	l.logWithLevel(Debug, data)
}

// InfoD implements the method for the KayveeLogger interface.
func (l *Logger) InfoD(title string, data map[string]interface{}) {
	data["title"] = title
	l.logWithLevel(Info, data)
}

// WarnD implements the method for the KayveeLogger interface.
func (l *Logger) WarnD(title string, data map[string]interface{}) {
	data["title"] = title
	l.logWithLevel(Warning, data)
}

// ErrorD implements the method for the KayveeLogger interface.
func (l *Logger) ErrorD(title string, data map[string]interface{}) {
	data["title"] = title
	l.logWithLevel(Error, data)
}

// CriticalD implements the method for the KayveeLogger interface.
func (l *Logger) CriticalD(title string, data map[string]interface{}) {
	data["title"] = title
	l.logWithLevel(Critical, data)
}

// CounterD implements the method for the KayveeLogger interface.
// Logs with type = gauge, and value = value
func (l *Logger) CounterD(title string, value int, data map[string]interface{}) {
	data["title"] = title
	data["value"] = value
	data["type"] = "counter"
	l.logWithLevel(Info, data)
}

// GaugeIntD implements the method for the KayveeLogger interface.
// Logs with type = gauge, and value = value
func (l *Logger) GaugeIntD(title string, value int, data map[string]interface{}) {
	l.gauge(title, value, data)
}

// GaugeFloatD implements the method for the KayveeLogger interface.
// Logs with type = gauge, and value = value
func (l *Logger) GaugeFloatD(title string, value float64, data map[string]interface{}) {
	l.gauge(title, value, data)
}

func (l *Logger) gauge(title string, value interface{}, data map[string]interface{}) {
	data["title"] = title
	data["value"] = value
	data["type"] = "gauge"
	l.logWithLevel(Info, data)
}

// Actual logging. Handles whether to output based on log level and
// unifies the passed in data with the stored globals
func (l *Logger) logWithLevel(logLvl LogLevel, data map[string]interface{}) {
	if logLvl < l.logLvl {
		// No log output
		return
	}
	data["level"] = logLvl.String()
	for key, value := range l.globals {
		if _, ok := data[key]; ok {
			// Values in the data map override the globals
			continue
		}
		data[key] = value
	}
	if l.logRouter != nil {
		data["_kvmeta"] = l.logRouter.Route(data)
	}

	logString := l.formatter(data)
	l.logWriter.Println(logString)
}

// updateContextMapIfNotReserved updates context[key] to val if key is not in the reserved list.
func updateContextMapIfNotReserved(context M, key string, val interface{}) {
	if reservedKeyNames[strings.ToLower(key)] {
		log.Printf("WARN: kayvee logger reserves '%s' from being set as context", key)
		return
	}
	context[key] = val
}

// WithRoutingConfig implements the method for the KayveeLogger interface.
func (l *Logger) WithRoutingConfig(filename string) (KayveeLogger, error) {
	routes, err := router.NewFromConfig(filename)
	if err != nil {
		return l, err
	}
	l.logRouter = routes
	return l, nil
}

// New creates a *logger.Logger. Default values are Debug LogLevel, kayvee Formatter, and std.err output.
func New(source string) *Logger {
	return NewWithContext(source, nil)
}

// NewWithContext creates a *logger.Logger. Default values are Debug LogLevel, kayvee Formatter, and std.err output.
func NewWithContext(source string, contextValues map[string]interface{}) *Logger {
	context := M{}
	for k, v := range contextValues {
		updateContextMapIfNotReserved(context, k, v)
	}
	if os.Getenv("_DEPLOY_ENV") != "" {
		context["deploy_env"] = os.Getenv("_DEPLOY_ENV")
	}

	logObj := Logger{
		globals: context,
	}

	var logLvl LogLevel
	strLogLvl := os.Getenv("KAYVEE_LOG_LEVEL")
	if strLogLvl == "" {
		logLvl = Debug
	} else {
		for key, val := range logLevelNames {
			if strings.ToLower(strLogLvl) == val {
				logLvl = key
				break
			}
		}
	}
	logObj.SetConfig(source, logLvl, kv.Format, os.Stderr)
	return &logObj
}
<commit_msg>Abstract format and  writing in Logger to allow for mocking<commit_after>package logger

import (
	"io"
	"log"
	"os"
	"strings"

	kv "gopkg.in/Clever/kayvee-go.v5"
	"gopkg.in/Clever/kayvee-go.v5/router"
)

/////////////////////
//
//  Helper definitions
//
/////////////////////

// Formatter is a function type that takes a map and returns a formatted string with the contents of the map
type Formatter func(data map[string]interface{}) string

// M is a convenience type for passing data into a log message.
type M map[string]interface{}

// LogLevel is an enum is used to denote level of logging
type LogLevel int

// Constants used to define different LogLevels supported
const (
	Debug LogLevel = iota
	Info
	Warning
	Error
	Critical
)

var logLevelNames = map[LogLevel]string{
	Debug:    "debug",
	Info:     "info",
	Warning:  "warning",
	Error:    "error",
	Critical: "critical",
}

func (l LogLevel) String() string {
	switch l {
	case Debug:
		fallthrough
	case Info:
		fallthrough
	case Warning:
		fallthrough
	case Error:
		fallthrough
	case Critical:
		return logLevelNames[l]
	}
	return ""
}

/////////////////////////////
//
//	Logger
//
/////////////////////////////

// Logger is the default implementation of KayveeLogger.
// It provides customization of globals, default log level, formatting, and output destination.
type Logger struct {
	globals   map[string]interface{}
	logLvl    LogLevel
	fLogger   formatLogger
	logRouter router.Router
}

// SetConfig allows configuration changes in one go
var reservedKeyNames = map[string]bool{
	"title":   true,
	"source":  true,
	"value":   true,
	"type":    true,
	"level":   true,
	"_kvmeta": true,
}

// SetConfig implements the method for the KayveeLogger interface.
func (l *Logger) SetConfig(source string, logLvl LogLevel, formatter Formatter, output io.Writer) {
	if l.globals == nil {
		l.globals = make(map[string]interface{})
	}
	l.globals["source"] = source
	l.logLvl = logLvl
	l.fLogger.setFormatter(formatter)
	l.fLogger.setOutput(output)
}

// AddContext implements the method for the KayveeLogger interface.
func (l *Logger) AddContext(key, val string) {
	updateContextMapIfNotReserved(l.globals, key, val)
}

// SetLogLevel implements the method for the KayveeLogger interface.
func (l *Logger) SetLogLevel(logLvl LogLevel) {
	l.logLvl = logLvl
}

// SetFormatter implements the method for the KayveeLogger interface.
func (l *Logger) SetFormatter(formatter Formatter) {
	l.fLogger.setFormatter(formatter)
}

// SetOutput implements the method for the KayveeLogger interface.
func (l *Logger) SetOutput(output io.Writer) {
	l.fLogger.setOutput(output)
}

// Debug implements the method for the KayveeLogger interface.
func (l *Logger) Debug(title string) {
	l.DebugD(title, M{})
}

// Info implements the method for the KayveeLogger interface.
func (l *Logger) Info(title string) {
	l.InfoD(title, M{})
}

// Warn implements the method for the KayveeLogger interface.
func (l *Logger) Warn(title string) {
	l.WarnD(title, M{})
}

// Error implements the method for the KayveeLogger interface.
func (l *Logger) Error(title string) {
	l.ErrorD(title, M{})
}

// Critical implements the method for the KayveeLogger interface.
func (l *Logger) Critical(title string) {
	l.CriticalD(title, M{})
}

// Counter implements the method for the KayveeLogger interface.
// Logs with type = gauge, and value = value
func (l *Logger) Counter(title string) {
	l.CounterD(title, 1, M{}) // Defaults to a value of 1
}

// GaugeInt implements the method for the KayveeLogger interface.
// Logs with type = gauge, and value = value
func (l *Logger) GaugeInt(title string, value int) {
	l.GaugeIntD(title, value, M{})
}

// GaugeFloat implements the method for the KayveeLogger interface.
// Logs with type = gauge, and value = value
func (l *Logger) GaugeFloat(title string, value float64) {
	l.GaugeFloatD(title, value, M{})
}

// DebugD implements the method for the KayveeLogger interface.
func (l *Logger) DebugD(title string, data map[string]interface{}) {
	data["title"] = title
	l.logWithLevel(Debug, data)
}

// InfoD implements the method for the KayveeLogger interface.
func (l *Logger) InfoD(title string, data map[string]interface{}) {
	data["title"] = title
	l.logWithLevel(Info, data)
}

// WarnD implements the method for the KayveeLogger interface.
func (l *Logger) WarnD(title string, data map[string]interface{}) {
	data["title"] = title
	l.logWithLevel(Warning, data)
}

// ErrorD implements the method for the KayveeLogger interface.
func (l *Logger) ErrorD(title string, data map[string]interface{}) {
	data["title"] = title
	l.logWithLevel(Error, data)
}

// CriticalD implements the method for the KayveeLogger interface.
func (l *Logger) CriticalD(title string, data map[string]interface{}) {
	data["title"] = title
	l.logWithLevel(Critical, data)
}

// CounterD implements the method for the KayveeLogger interface.
// Logs with type = gauge, and value = value
func (l *Logger) CounterD(title string, value int, data map[string]interface{}) {
	data["title"] = title
	data["value"] = value
	data["type"] = "counter"
	l.logWithLevel(Info, data)
}

// GaugeIntD implements the method for the KayveeLogger interface.
// Logs with type = gauge, and value = value
func (l *Logger) GaugeIntD(title string, value int, data map[string]interface{}) {
	l.gauge(title, value, data)
}

// GaugeFloatD implements the method for the KayveeLogger interface.
// Logs with type = gauge, and value = value
func (l *Logger) GaugeFloatD(title string, value float64, data map[string]interface{}) {
	l.gauge(title, value, data)
}

func (l *Logger) gauge(title string, value interface{}, data map[string]interface{}) {
	data["title"] = title
	data["value"] = value
	data["type"] = "gauge"
	l.logWithLevel(Info, data)
}

// Actual logging. Handles whether to output based on log level and
// unifies the passed in data with the stored globals
func (l *Logger) logWithLevel(logLvl LogLevel, data map[string]interface{}) {
	if logLvl < l.logLvl {
		// No log output
		return
	}
	data["level"] = logLvl.String()
	for key, value := range l.globals {
		if _, ok := data[key]; ok {
			// Values in the data map override the globals
			continue
		}
		data[key] = value
	}
	if l.logRouter != nil {
		data["_kvmeta"] = l.logRouter.Route(data)
	}

	l.fLogger.formatAndLog(data)
}

// updateContextMapIfNotReserved updates context[key] to val if key is not in the reserved list.
func updateContextMapIfNotReserved(context M, key string, val interface{}) {
	if reservedKeyNames[strings.ToLower(key)] {
		log.Printf("WARN: kayvee logger reserves '%s' from being set as context", key)
		return
	}
	context[key] = val
}

// WithRoutingConfig implements the method for the KayveeLogger interface.
func (l *Logger) WithRoutingConfig(filename string) (KayveeLogger, error) {
	routes, err := router.NewFromConfig(filename)
	if err != nil {
		return l, err
	}
	l.logRouter = routes
	return l, nil
}

// New creates a *logger.Logger. Default values are Debug LogLevel, kayvee Formatter, and std.err output.
func New(source string) *Logger {
	return NewWithContext(source, nil)
}

// NewWithContext creates a *logger.Logger. Default values are Debug LogLevel, kayvee Formatter, and std.err output.
func NewWithContext(source string, contextValues map[string]interface{}) *Logger {
	context := M{}
	for k, v := range contextValues {
		updateContextMapIfNotReserved(context, k, v)
	}
	if os.Getenv("_DEPLOY_ENV") != "" {
		context["deploy_env"] = os.Getenv("_DEPLOY_ENV")
	}

	logObj := Logger{
		globals: context,
	}

	fl := defaultFormatLogger{}
	logObj.fLogger = &fl

	var logLvl LogLevel
	strLogLvl := os.Getenv("KAYVEE_LOG_LEVEL")
	if strLogLvl == "" {
		logLvl = Debug
	} else {
		for key, val := range logLevelNames {
			if strings.ToLower(strLogLvl) == val {
				logLvl = key
				break
			}
		}
	}

	logObj.SetConfig(source, logLvl, kv.Format, os.Stderr)

	return &logObj
}

/////////////////////////////
//
//	FormatLogger
//
/////////////////////////////

// formatLogger is an interface that abstracts the last steps in submitting a log
// message to a Logger: formatting and log writing. It can be replaced for testing.
// This is not yet exported, but could be if clients want customization of the
// format and writing steps.
type formatLogger interface {
	// formatAndLog processes the given data map into a log line and writes it
	formatAndLog(data map[string]interface{})

	// setFormatter specifies the Formatter function to use in formatAndLog
	setFormatter(formatter Formatter)

	// setOutput specifies the output destination to use in formatAndLog
	setOutput(output io.Writer)
}

// defaultFormatLogger provides default implementation of a formatLogger.
type defaultFormatLogger struct {
	formatter Formatter
	logWriter *log.Logger
}

// formatAndLog implements the formatLogger interface for *defaultFormatLogger.
func (fl *defaultFormatLogger) formatAndLog(data map[string]interface{}) {
	logString := fl.formatter(data)
	fl.logWriter.Println(logString)
}

// setFormat implements the formatLogger interface for *defaultFormatLogger.
func (fl *defaultFormatLogger) setFormatter(formatter Formatter) {
	fl.formatter = formatter
}

// setOutput implements the formatLogger interface for *defaultFormatLogger.
func (fl *defaultFormatLogger) setOutput(output io.Writer) {
	fl.logWriter = log.New(output, "", 0) // No prefixes
}
<|endoftext|>


<commit_before>package logger

type Logger interface {
	Print(v ...interface{})
}

// LogWriter log writer interface
type LogWriter interface {
	Println(v ...interface{})
}
<commit_msg>[logger]: Combined LogWriter with Logger into Logger interface<commit_after>package logger

type Logger interface {
	Print(v ...interface{})
	Println(v ...interface{})
}
<|endoftext|>