<commit_before>// Alexandria
//
// Copyright (C) 2015-2016  Colin Benner
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package alexandria

import (
	"errors"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"strconv"
	"strings"
)

var NoSuchScrollError = errors.New("No such scroll")

type errTemplateReader struct {
	doc string
	err error
}

// Load a template file from disk and propagate errors
func (e *errTemplateReader) readTemplate(name string) {
	if e.err != nil {
		return
	}

	tmp, err := readTemplate(name)
	e.err = err
	e.doc += tmp
}

// Create a LaTeX file from the content of the given scroll together with all
// the appropriate templates.  The resulting file stored in the temp directory.
func scrollToLatex(id Id) error {
	var e errTemplateReader

	scroll, err := readScroll(id)
	if err != nil {
		if os.IsNotExist(err) {
			RemoveFromIndex(id)
			return NoSuchScrollError
		}
		LogError(err)
		return err
	}
	tags := ParseMetadata(scroll)

	e.readTemplate("header")
	e.readTemplate(DocumentType(tags) + "_header")
	e.doc += scroll
	e.readTemplate(DocumentType(tags) + "_footer")
	e.readTemplate("footer")

	if e.err != nil {
		return e.err
	}
	return writeTemp(id, e.doc)
}

// Compile a LaTeX file with the given id to produce a PDF file.  Both input
// and output files are in the temp directory.
func latexToPdf(id Id) error {
	msg, err := exec.Command("xelatex", "-interaction", "nonstopmode",
		"-output-directory", Config.TempDirectory,
		Config.TempDirectory+string(id)).CombinedOutput()
	if err != nil {
		log.Fatal(string(msg))
		return err
	}
	return nil
}

// Convert PDF to PNG, storing the result in the cache directory.  From there,
// it can be served by the web server or displayed to the user via some other
// user interface.
func pdfToPng(i Id) error {
	id := string(i)
	return exec.Command("convert", "-trim",
		"-quality", strconv.Itoa(Config.Quality),
		"-density", strconv.Itoa(Config.Dpi),
		Config.TempDirectory+id+".pdf", Config.CacheDirectory+id+".png").Run()
}

// Generate a PNG image from a given scroll, if there is no up-to-date image.
func ProcessScroll(id Id) error {
	if isUpToDate(id) {
		return nil
	}

	err := scrollToLatex(id)
	if err != nil {
		return err
	}

	err = latexToPdf(id)
	if err != nil {
		return err
	}

	return pdfToPng(id)
}

// Generate images for a list of scrolls.
func ProcessScrolls(ids []Scroll) int {
	numScrolls := 0

	for _, foo := range ids {
		id := foo.Id
		err := ProcessScroll(id)
		if err == NoSuchScrollError {
			continue
		} else if err != nil {
			log.Panic("An error ocurred when processing scroll ", id, ": ", err)
		} else {
			numScrolls += 1
		}
	}

	return numScrolls
}

func RenderAllScrolls() error {
	files, err := ioutil.ReadDir(Config.KnowledgeDirectory)
	if err != nil {
		return err
	}
	for _, file := range files {
		id := Id(strings.TrimSuffix(file.Name(), ".tex"))
		if err := ProcessScroll(id); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>Continue after errors, count number of scrolls<commit_after>// Alexandria
//
// Copyright (C) 2015-2016  Colin Benner
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package alexandria

import (
	"errors"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"strconv"
	"strings"
)

var NoSuchScrollError = errors.New("No such scroll")
const hashes = "############################################################"

type errTemplateReader struct {
	doc string
	err error
}

// Load a template file from disk and propagate errors
func (e *errTemplateReader) readTemplate(name string) {
	if e.err != nil {
		return
	}

	tmp, err := readTemplate(name)
	e.err = err
	e.doc += tmp
}

// Create a LaTeX file from the content of the given scroll together with all
// the appropriate templates.  The resulting file stored in the temp directory.
func scrollToLatex(id Id) error {
	var e errTemplateReader

	scroll, err := readScroll(id)
	if err != nil {
		if os.IsNotExist(err) {
			RemoveFromIndex(id)
			return NoSuchScrollError
		}
		LogError(err)
		return err
	}
	tags := ParseMetadata(scroll)

	e.readTemplate("header")
	e.readTemplate(DocumentType(tags) + "_header")
	e.doc += scroll
	e.readTemplate(DocumentType(tags) + "_footer")
	e.readTemplate("footer")

	if e.err != nil {
		return e.err
	}
	return writeTemp(id, e.doc)
}

// Compile a LaTeX file with the given id to produce a PDF file.  Both input
// and output files are in the temp directory.
func latexToPdf(id Id) error {
	msg, err := exec.Command("xelatex", "-interaction", "nonstopmode",
		"-output-directory", Config.TempDirectory,
		Config.TempDirectory+string(id)).CombinedOutput()
	if err != nil {
		log.Fatal(string(msg))
		return err
	}
	return nil
}

// Convert PDF to PNG, storing the result in the cache directory.  From there,
// it can be served by the web server or displayed to the user via some other
// user interface.
func pdfToPng(i Id) error {
	id := string(i)
	return exec.Command("convert", "-trim",
		"-quality", strconv.Itoa(Config.Quality),
		"-density", strconv.Itoa(Config.Dpi),
		Config.TempDirectory+id+".pdf", Config.CacheDirectory+id+".png").Run()
}

// Generate a PNG image from a given scroll, if there is no up-to-date image.
func ProcessScroll(id Id) error {
	if isUpToDate(id) {
		return nil
	}

	err := scrollToLatex(id)
	if err != nil {
		return err
	}

	err = latexToPdf(id)
	if err != nil {
		return err
	}

	return pdfToPng(id)
}

// Generate images for a list of scrolls.
func ProcessScrolls(ids []Scroll) int {
	numScrolls := 0

	for _, foo := range ids {
		id := foo.Id
		err := ProcessScroll(id)
		if err == NoSuchScrollError {
			continue
		} else if err != nil {
			log.Panic("An error ocurred when processing scroll ", id, ": ", err)
		} else {
			numScrolls += 1
		}
	}

	return numScrolls
}

func RenderAllScrolls() int {
	files, err := ioutil.ReadDir(Config.KnowledgeDirectory)
	if err != nil {
		panic(err)
	}
	counter := 0
	var errors []error
	for _, file := range files {
		id := Id(strings.TrimSuffix(file.Name(), ".tex"))
		if err := ProcessScroll(id); err != nil && err != NoSuchScrollError {
			log.Printf("%s\nERROR\n%s\n%v\n%s\n", hashes, hashes, err, hashes)
		}
		counter += 1
	}
	for _, err := range errors {
		log.Printf("Error: %v\n", err)
	}
	return counter
}
<|endoftext|>


<commit_before>package strconv

import "fmt"
import "math"
import "strings"
import "strconv"
import "testing"
import "github.com/stretchr/testify/assert"

var (
	nan  = math.NaN()
	pinf = math.Inf(+1)
	ninf = math.Inf(-1)
)

const (
	hiThres = 1.000000000000001
	loThres = 0.999999999999999
)

func atof(s string) float64 {
	f, err := strconv.ParseFloat(s, 64)
	if err != nil {
		panic(err)
	}
	return f
}

func signStr(f float64) string {
	if f >= 0 {
		return ""
	}
	return "-"
}

func stripDot(s string) string {
	if s == "." {
		return ""
	}
	return s
}

func addIEC(s string) string {
	if s == "" {
		return ""
	}
	return s + "i"
}

func normToAlt(s string) string {
	for _, ch := range s {
		if alt, ok := mapNormToAlt[ch]; ok {
			return string(alt)
		}
	}
	return s
}

func altToNorm(s string) string {
	for _, ch := range s {
		if norm, ok := mapAltToNorm[ch]; ok {
			return string(norm)
		}
	}
	return s
}

func split(s string) (string, string) {
	i := strings.IndexAny(s, parsePrefixes)
	if i < 0 {
		return s, ""
	}
	return s[:i], s[i:]
}

func TestPrefixExactSI(t *testing.T) {
	for _, sign := range []float64{-1, +1} {
		for i, f := range scaleSI {
			str := FormatPrefix(sign*f, SI, -1)
			flt, err := ParsePrefix(str, SI)

			pre := normToAlt(stripDot(string(prefixes[i])))
			assert.Equal(t, signStr(sign)+"1"+pre, str)
			assert.Equal(t, sign*f, flt)
			assert.Equal(t, nil, err)
		}
	}
}

func testPrefixExact(t *testing.T, minExp, mode int, wrap func(string) string) {
	var pres = prefixes[minExp+len(divPrefixes):]
	var str, pre string
	var flt float64
	var err error

	for _, sign := range []float64{-1, +1} {
		var f0 = sign * scaleIEC[minExp+len(divPrefixes)]

		str = FormatPrefix(f0/2, mode, -1)
		flt, err = ParsePrefix(str, mode)
		pre = wrap(string(pres[0]))
		assert.Equal(t, signStr(sign)+"0.5"+pre, str)
		assert.Equal(t, f0/2, flt)
		assert.Equal(t, nil, err)

		for i := 0; i < 10*len(pres); i++ {
			str = FormatPrefix(f0, mode, -1)
			flt, err = ParsePrefix(str, mode)

			pre := wrap(string(pres[(i / 10)]))
			ord := fmt.Sprintf("%d", 1<<uint(i%10))
			assert.Equal(t, signStr(sign)+ord+pre, str)
			assert.Equal(t, f0, flt)
			assert.Equal(t, nil, err)
			f0 *= 2
		}

		str = FormatPrefix(f0, mode, -1)
		flt, err = ParsePrefix(str, mode)
		pre = wrap(string(pres[len(pres)-1]))
		assert.Equal(t, signStr(sign)+"1024"+pre, str)
		assert.Equal(t, f0, flt)
		assert.Equal(t, nil, err)
	}
}

func TestPrefixExactBase1024(t *testing.T) {
	wrap := func(s string) string { return stripDot(s) }
	testPrefixExact(t, minExp, Base1024, wrap)
}

func TestPrefixExactIEC(t *testing.T) {
	wrap := func(s string) string { return addIEC(stripDot(s)) }
	testPrefixExact(t, 0, IEC, wrap)
}

func testPrefixBoundary(t *testing.T, scales []float64, prefixes string, mode int, wrap func(string) string) {
	var str, str1, str2, pre string
	var flt, fabs, fnum float64
	var err error

	base := 1024.0
	if mode == SI {
		base = 1000.0
	}

	for _, sign := range []float64{-1, +1} {
		for i, f := range scales {
			// Round towards zero.
			str = FormatPrefix(math.Nextafter(sign*f, sign*ninf), mode, -1)
			flt, err = ParsePrefix(str, mode)
			fabs = math.Abs(flt)

			pre = string(prefixes[0])
			if i > 0 {
				pre = string(prefixes[i-1])
			}
			pre = wrap(pre)
			str1, str2 = split(str)
			fnum = math.Abs(atof(str1))
			if i == 0 {
				assert.True(t, 1.0*loThres <= fnum && fnum <= 1.0)
			} else {
				assert.True(t, base*loThres <= fnum && fnum <= base)
			}
			assert.Equal(t, pre, str2)
			assert.True(t, f*loThres <= fabs && fabs <= f)
			assert.Equal(t, math.Signbit(flt), math.Signbit(sign))
			assert.Equal(t, nil, err)

			// Round away from zero.
			str = FormatPrefix(math.Nextafter(sign*f, sign*pinf), mode, -1)
			flt, err = ParsePrefix(str, mode)
			fabs = math.Abs(flt)

			pre = wrap(string(prefixes[i]))
			str1, str2 = split(str)
			fnum = math.Abs(atof(str1))
			assert.True(t, 1.0 <= fnum && fnum <= 1.0*hiThres)
			assert.Equal(t, pre, str2)
			assert.True(t, f <= fabs && fabs <= f*hiThres)
			assert.Equal(t, math.Signbit(flt), math.Signbit(sign))
			assert.Equal(t, nil, err)
		}
	}
}

func TestPrefixBoundarySI(t *testing.T) {
	wrap := func(s string) string { return normToAlt(stripDot(s)) }
	testPrefixBoundary(t, scaleSI, prefixes, SI, wrap)
}

func TestPrefixBoundaryBase1024(t *testing.T) {
	wrap := func(s string) string { return altToNorm(stripDot(s)) }
	testPrefixBoundary(t, scaleIEC, prefixes, Base1024, wrap)
}

func TestPrefixBoundaryIEC(t *testing.T) {
	idx := len(prefixes) / 2
	wrap := func(s string) string { return addIEC(altToNorm(stripDot(s))) }
	testPrefixBoundary(t, scaleIEC[idx:], prefixes[idx:], IEC, wrap)
}

func TestPrefixFailParse(t *testing.T) {
	for _, x := range []struct {
		str  string
		mode int
		ok   bool
		flt  float64
	}{
		{"", SI, false, 0},
		{"NaN1M", SI, false, 0},
		{"1", IEC, true, Unit},
		{"1 ", IEC, false, 0},
		{"1M", IEC, false, 0},
		{"1Mi", SI, false, 0},
		{"+1M", Base1024, true, +Mebi},
		{"-1Mi", Base1024, true, -Mebi},
		{"+1Mi", Base1024, true, +Mebi},
		{"1E-3", SI, false, 0},
		{"1e-3", SI, false, 0},
		{"1ki", SI, false, 0},
		{"1ki", IEC, false, 0},
		{"1ki", Base1024, true, Kibi},
		{"+1ki", Base1024, true, Kibi},
		{"1μi", SI, false, 0},
		{"1μi", IEC, false, 0},
		{"1μi", Base1024, false, 0},
		{"1k", SI, true, Kilo},
		{"1k", IEC, false, 0},
		{"1k", Base1024, true, Kibi},
		{"1μ", SI, true, Micro},
		{"1μ ", SI, false, 0},
		{" 1μ", SI, false, 0},
		{"+1μ", SI, true, Micro},
		{"1μ", IEC, false, 0},
		{"1μ", Base1024, true, 1.0 / Mebi},
		{"+1μ", Base1024, true, 1.0 / Mebi},
		{"1mi", IEC, false, 0},
		{"0.000001", SI, true, Micro},
		{"1000000u", SI, true, Unit},
		{"1048576", Base1024, true, Mebi},
		{"1048576Ki", IEC, true, Gibi},
		{"nAn", SI, true, nan},
		{"+nan", Base1024, false, 0},
		{"-NAN", IEC, false, 0},
		{"INF", SI, true, pinf},
		{"+iNf", Base1024, true, pinf},
		{"-inF", IEC, true, ninf},
		{"", AutoParse, false, 0},
		{"123", AutoParse, true, 123},
		{"123Ki", AutoParse, true, 123 * Kibi},
		{"123k", AutoParse, true, 123 * Kilo},
		{"123K", AutoParse, true, 123 * Kilo},
		{"3Mi", AutoParse, true, 3 * Mebi},
		{"3M", AutoParse, true, 3 * Mega},
		{"3E-3", AutoParse, true, 3E-3},
		{"2E2", AutoParse, true, 2E2},
	} {
		flt, err := ParsePrefix(x.str, x.mode)
		if x.ok {
			assert.Nil(t, err)
			if !math.IsNaN(x.flt) || !math.IsNaN(flt) {
				assert.Equal(t, x.flt, flt)
			}
		} else {
			assert.NotNil(t, err)
		}
	}
}

func TestPrefix(t *testing.T) {
	// Test for zero, NaN, -Inf, and +Inf.
	for _, mode := range []int{SI, Base1024, IEC} {
		for _, prec := range []int{-1, 0, +1} {
			for _, f := range []float64{-0.0, +0.0, nan, ninf, pinf} {
				str := FormatPrefix(f, mode, prec)
				flt, err := ParsePrefix(str, mode)

				assert.Equal(t, str, strconv.FormatFloat(f, 'f', prec, 64))
				if !math.IsNaN(f) || !math.IsNaN(flt) {
					assert.Equal(t, f, flt)
				}
				assert.Equal(t, nil, err)
			}
		}
	}

	// Test for a huge range of values.
	for _, mode := range []int{SI, Base1024, IEC} {
		for _, prec := range []int{-1, 0, +1, +2} {
			for i := -100; i <= +100; i++ {
				f := 1.234567890123456 * math.Pow(10, float64(i))
				str := FormatPrefix(f, mode, prec)
				flt, err := ParsePrefix(str, mode)
				str1, _ := split(str)
				fnum := math.Abs(atof(str1))

				// Ensure that we maintain decent precision.
				if prec < 0 {
					assert.True(t, f*loThres <= flt && flt <= f*hiThres)
				} else if flt != 0 {
					assert.True(t, f*0.5 <= flt && flt <= f*2)
				}

				// Ensure that we choose the best scale if possible.
				var base, minScale, maxScale float64
				switch mode {
				case SI:
					base = 1000.0
					minScale, maxScale = scaleSI[0], scaleSI[len(scaleSI)-1]
				case Base1024:
					base = 1024.0
					minScale, maxScale = scaleIEC[0], scaleIEC[len(scaleIEC)-1]
				case IEC:
					base = 1024.0
					minScale, maxScale = 1, scaleIEC[len(scaleIEC)-1]
				}
				if minScale <= f && f <= maxScale {
					assert.True(t, 1.0 <= fnum && fnum <= base)
				}

				assert.Equal(t, nil, err)
			}
		}
	}
}
<commit_msg>Trivial reorganization<commit_after>package strconv

import "fmt"
import "math"
import "strings"
import "strconv"
import "testing"
import "github.com/stretchr/testify/assert"

var (
	nan  = math.NaN()
	pinf = math.Inf(+1)
	ninf = math.Inf(-1)
)

const (
	hiThres = 1.000000000000001
	loThres = 0.999999999999999
)

func atof(s string) float64 {
	f, err := strconv.ParseFloat(s, 64)
	if err != nil {
		panic(err)
	}
	return f
}

func signStr(f float64) string {
	if f >= 0 {
		return ""
	}
	return "-"
}

func stripDot(s string) string {
	if s == "." {
		return ""
	}
	return s
}

func addIEC(s string) string {
	if s == "" {
		return ""
	}
	return s + "i"
}

func normToAlt(s string) string {
	for _, ch := range s {
		if alt, ok := mapNormToAlt[ch]; ok {
			return string(alt)
		}
	}
	return s
}

func altToNorm(s string) string {
	for _, ch := range s {
		if norm, ok := mapAltToNorm[ch]; ok {
			return string(norm)
		}
	}
	return s
}

func split(s string) (string, string) {
	if i := strings.IndexAny(s, parsePrefixes); i >= 0 {
		return s[:i], s[i:]
	}
	return s, ""
}

func TestPrefixExactSI(t *testing.T) {
	for _, sign := range []float64{-1, +1} {
		for i, f := range scaleSI {
			str := FormatPrefix(sign*f, SI, -1)
			flt, err := ParsePrefix(str, SI)

			pre := normToAlt(stripDot(string(prefixes[i])))
			assert.Equal(t, signStr(sign)+"1"+pre, str)
			assert.Equal(t, sign*f, flt)
			assert.Equal(t, nil, err)
		}
	}
}

func testPrefixExact(t *testing.T, minExp, mode int, wrap func(string) string) {
	var pres = prefixes[minExp+len(divPrefixes):]
	var str, pre string
	var flt float64
	var err error

	for _, sign := range []float64{-1, +1} {
		var f0 = sign * scaleIEC[minExp+len(divPrefixes)]

		str = FormatPrefix(f0/2, mode, -1)
		flt, err = ParsePrefix(str, mode)
		pre = wrap(string(pres[0]))
		assert.Equal(t, signStr(sign)+"0.5"+pre, str)
		assert.Equal(t, f0/2, flt)
		assert.Equal(t, nil, err)

		for i := 0; i < 10*len(pres); i++ {
			str = FormatPrefix(f0, mode, -1)
			flt, err = ParsePrefix(str, mode)

			pre := wrap(string(pres[(i / 10)]))
			ord := fmt.Sprintf("%d", 1<<uint(i%10))
			assert.Equal(t, signStr(sign)+ord+pre, str)
			assert.Equal(t, f0, flt)
			assert.Equal(t, nil, err)
			f0 *= 2
		}

		str = FormatPrefix(f0, mode, -1)
		flt, err = ParsePrefix(str, mode)
		pre = wrap(string(pres[len(pres)-1]))
		assert.Equal(t, signStr(sign)+"1024"+pre, str)
		assert.Equal(t, f0, flt)
		assert.Equal(t, nil, err)
	}
}

func TestPrefixExactBase1024(t *testing.T) {
	wrap := func(s string) string { return stripDot(s) }
	testPrefixExact(t, minExp, Base1024, wrap)
}

func TestPrefixExactIEC(t *testing.T) {
	wrap := func(s string) string { return addIEC(stripDot(s)) }
	testPrefixExact(t, 0, IEC, wrap)
}

func testPrefixBoundary(t *testing.T, scales []float64, prefixes string, mode int, wrap func(string) string) {
	var str, str1, str2, pre string
	var flt, fabs, fnum float64
	var err error

	base := 1024.0
	if mode == SI {
		base = 1000.0
	}

	for _, sign := range []float64{-1, +1} {
		for i, f := range scales {
			// Round towards zero.
			str = FormatPrefix(math.Nextafter(sign*f, sign*ninf), mode, -1)
			flt, err = ParsePrefix(str, mode)
			fabs = math.Abs(flt)

			pre = string(prefixes[0])
			if i > 0 {
				pre = string(prefixes[i-1])
			}
			pre = wrap(pre)
			str1, str2 = split(str)
			fnum = math.Abs(atof(str1))
			if i == 0 {
				assert.True(t, 1.0*loThres <= fnum && fnum <= 1.0)
			} else {
				assert.True(t, base*loThres <= fnum && fnum <= base)
			}
			assert.Equal(t, pre, str2)
			assert.True(t, f*loThres <= fabs && fabs <= f)
			assert.Equal(t, math.Signbit(flt), math.Signbit(sign))
			assert.Equal(t, nil, err)

			// Round away from zero.
			str = FormatPrefix(math.Nextafter(sign*f, sign*pinf), mode, -1)
			flt, err = ParsePrefix(str, mode)
			fabs = math.Abs(flt)

			pre = wrap(string(prefixes[i]))
			str1, str2 = split(str)
			fnum = math.Abs(atof(str1))
			assert.True(t, 1.0 <= fnum && fnum <= 1.0*hiThres)
			assert.Equal(t, pre, str2)
			assert.True(t, f <= fabs && fabs <= f*hiThres)
			assert.Equal(t, math.Signbit(flt), math.Signbit(sign))
			assert.Equal(t, nil, err)
		}
	}
}

func TestPrefixBoundarySI(t *testing.T) {
	wrap := func(s string) string { return normToAlt(stripDot(s)) }
	testPrefixBoundary(t, scaleSI, prefixes, SI, wrap)
}

func TestPrefixBoundaryBase1024(t *testing.T) {
	wrap := func(s string) string { return altToNorm(stripDot(s)) }
	testPrefixBoundary(t, scaleIEC, prefixes, Base1024, wrap)
}

func TestPrefixBoundaryIEC(t *testing.T) {
	idx := len(prefixes) / 2
	wrap := func(s string) string { return addIEC(altToNorm(stripDot(s))) }
	testPrefixBoundary(t, scaleIEC[idx:], prefixes[idx:], IEC, wrap)
}

func TestPrefixFailParse(t *testing.T) {
	for _, x := range []struct {
		str  string
		mode int
		ok   bool
		flt  float64
	}{
		{"", SI, false, 0},
		{"NaN1M", SI, false, 0},
		{"1", IEC, true, Unit},
		{"1 ", IEC, false, 0},
		{"1M", IEC, false, 0},
		{"1Mi", SI, false, 0},
		{"+1M", Base1024, true, +Mebi},
		{"-1Mi", Base1024, true, -Mebi},
		{"+1Mi", Base1024, true, +Mebi},
		{"1E-3", SI, false, 0},
		{"1e-3", SI, false, 0},
		{"1ki", SI, false, 0},
		{"1ki", IEC, false, 0},
		{"1ki", Base1024, true, Kibi},
		{"+1ki", Base1024, true, Kibi},
		{"1μi", SI, false, 0},
		{"1μi", IEC, false, 0},
		{"1μi", Base1024, false, 0},
		{"1k", SI, true, Kilo},
		{"1k", IEC, false, 0},
		{"1k", Base1024, true, Kibi},
		{"1μ", SI, true, Micro},
		{"1μ ", SI, false, 0},
		{" 1μ", SI, false, 0},
		{"+1μ", SI, true, Micro},
		{"1μ", IEC, false, 0},
		{"1μ", Base1024, true, 1.0 / Mebi},
		{"+1μ", Base1024, true, 1.0 / Mebi},
		{"1mi", IEC, false, 0},
		{"0.000001", SI, true, Micro},
		{"1000000u", SI, true, Unit},
		{"1048576", Base1024, true, Mebi},
		{"1048576Ki", IEC, true, Gibi},
		{"nAn", SI, true, nan},
		{"+nan", Base1024, false, 0},
		{"-NAN", IEC, false, 0},
		{"INF", SI, true, pinf},
		{"+iNf", Base1024, true, pinf},
		{"-inF", IEC, true, ninf},
		{"", AutoParse, false, 0},
		{"123", AutoParse, true, 123},
		{"123Ki", AutoParse, true, 123 * Kibi},
		{"123k", AutoParse, true, 123 * Kilo},
		{"123K", AutoParse, true, 123 * Kilo},
		{"3Mi", AutoParse, true, 3 * Mebi},
		{"3M", AutoParse, true, 3 * Mega},
		{"3E-3", AutoParse, true, 3E-3},
		{"2E2", AutoParse, true, 2E2},
	} {
		flt, err := ParsePrefix(x.str, x.mode)
		if x.ok {
			assert.Nil(t, err)
			if !math.IsNaN(x.flt) || !math.IsNaN(flt) {
				assert.Equal(t, x.flt, flt)
			}
		} else {
			assert.NotNil(t, err)
		}
	}
}

func TestPrefix(t *testing.T) {
	// Test for zero, NaN, -Inf, and +Inf.
	for _, mode := range []int{SI, Base1024, IEC} {
		for _, prec := range []int{-1, 0, +1} {
			for _, f := range []float64{-0.0, +0.0, nan, ninf, pinf} {
				str := FormatPrefix(f, mode, prec)
				flt, err := ParsePrefix(str, mode)

				assert.Equal(t, str, strconv.FormatFloat(f, 'f', prec, 64))
				if !math.IsNaN(f) || !math.IsNaN(flt) {
					assert.Equal(t, f, flt)
				}
				assert.Equal(t, nil, err)
			}
		}
	}

	// Test for a huge range of values.
	for _, mode := range []int{SI, Base1024, IEC} {
		for _, prec := range []int{-1, 0, +1, +2} {
			for i := -100; i <= +100; i++ {
				f := 1.234567890123456 * math.Pow(10, float64(i))
				str := FormatPrefix(f, mode, prec)
				flt, err := ParsePrefix(str, mode)
				str1, _ := split(str)
				fnum := math.Abs(atof(str1))

				// Ensure that we maintain decent precision.
				if prec < 0 {
					assert.True(t, f*loThres <= flt && flt <= f*hiThres)
				} else if flt != 0 {
					assert.True(t, f*0.5 <= flt && flt <= f*2)
				}

				// Ensure that we choose the best scale if possible.
				var base, minScale, maxScale float64
				switch mode {
				case SI:
					base = 1000.0
					minScale, maxScale = scaleSI[0], scaleSI[len(scaleSI)-1]
				case Base1024:
					base = 1024.0
					minScale, maxScale = scaleIEC[0], scaleIEC[len(scaleIEC)-1]
				case IEC:
					base = 1024.0
					minScale, maxScale = 1, scaleIEC[len(scaleIEC)-1]
				}
				if minScale <= f && f <= maxScale {
					assert.True(t, 1.0 <= fnum && fnum <= base)
				}

				assert.Equal(t, nil, err)
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"regexp"
	"strings"

	"github.com/mizkei/rundeck-cli/rundeck"
)

var (
	re  = regexp.MustCompile(` +`)
	re2 = regexp.MustCompile(`^ +`)
)

func listHasPrefix(pre string, ls []string) []string {
	list := make([]string, 0, len(ls))

	for _, s := range ls {
		if strings.HasPrefix(s, pre) {
			list = append(list, s)
		}
	}

	return list
}

type completer struct {
	cmds    []string
	subCmds []string
	jobs    []string
}

func (c *completer) completeCmd(line string, pos int) (string, []string, string) {
	pre, ls := line[:pos], line[pos:]
	pre = re2.ReplaceAllString(re.ReplaceAllString(pre, " "), "")

	ss := strings.Split(pre, " ")

	switch len(ss) {
	case 1:
		target := ss[0]
		if target == "" {
			return "", c.cmds, ls
		}
		return "", listHasPrefix(target, c.cmds), ls
	case 2:
		target := ss[1]
		if ss[0] == rundeck.CmdRun {
			return ss[0] + " ", listHasPrefix(target, c.jobs), ls
		}

		return ss[0] + " ", listHasPrefix(target, c.subCmds), ls
	case 3:
		pre := strings.Join(ss[:2], " ")
		target := ss[2]
		if ss[1] == rundeck.SubCmdJob {
			return pre + " ", listHasPrefix(target, c.jobs), ls
		}

		return pre + " ", nil, ls
	default:
		return pre + " ", nil, ls
	}
}
<commit_msg>switchの中では代入のみをおこなう<commit_after>package main

import (
	"regexp"
	"strings"

	"github.com/mizkei/rundeck-cli/rundeck"
)

var (
	re  = regexp.MustCompile(` +`)
	re2 = regexp.MustCompile(`^ +`)
)

func listHasPrefix(pre string, ls []string) []string {
	list := make([]string, 0, len(ls))

	for _, s := range ls {
		if strings.HasPrefix(s, pre) {
			list = append(list, s)
		}
	}

	return list
}

type completer struct {
	cmds    []string
	subCmds []string
	jobs    []string
}

func (c *completer) completeCmd(line string, pos int) (string, []string, string) {
	pre, ls := line[:pos], line[pos:]
	pre = re2.ReplaceAllString(re.ReplaceAllString(pre, " "), "")

	newPre := pre + " "
	var list []string

	switch ss := strings.Split(pre, " "); len(ss) {
	case 1:
		target := ss[0]
		newPre = ""
		if target == "" {
			list = c.cmds
			break
		}
		list = listHasPrefix(target, c.cmds)
	case 2:
		target := ss[1]
		newPre = ss[0] + " "
		if ss[0] == rundeck.CmdRun {
			list = listHasPrefix(target, c.jobs)
			break
		}
		list = listHasPrefix(target, c.subCmds)
	case 3:
		target := ss[2]
		newPre = strings.Join(ss[:2], " ") + " "
		if ss[1] == rundeck.SubCmdJob {
			list = listHasPrefix(target, c.jobs)
		}
	}

	return newPre, list, ls
}
<|endoftext|>


<commit_before>package ttlstore

import (
	"fmt"
	"testing"
	"time"
)

func TestNewDeleter(t *testing.T) {
	ex := []*actionTimestamp{
		{"1", 10},
		{"2", 20},
		{"3", 15},
		{"4", 25},
		{"5", 5},
		{"6", 17},
		{"7", 14},
		{"8", time.Now().UnixNano()},
	}
	deleted := make(map[interface{}]bool)
	cnt := 0
	var dh DeleteHook = func(key interface{}) {
		deleted[key] = true
		cnt++
	}
	d := NewDeleter("", dh, time.Minute, time.Hour, 3, 3)
	go d.Process()
	for _, v := range ex {
		d.c <- v
	}
	close(d.c)
	<-d.finished
	if l := len(d.lastActionTimestamp); l != 8 {
		t.Errorf("want: 8, got: %d", l)
	}
	d.deleteUntilBelowMaxUuid()

	if cnt != 5 {
		t.Errorf("deleted, want: 5, got: %d", cnt)
		t.Errorf("%v", deleted)
	}
	if l := len(d.lastActionTimestamp); l != 3 {
		t.Errorf("want: 3, got: %d", l)
	}

	d.DeleteTooOld()
	if cnt != 7 {
		t.Errorf("deleted, want: 7, got: %d", cnt)
		t.Errorf("%v", deleted)
	}
	if l := len(d.lastActionTimestamp); l != 1 {
		t.Errorf("want: 1, got: %d", l)
	}
}

func TestFullDeleter(t *testing.T) {
	ex := []*actionTimestamp{
		{"1", 10},
		{"2", 20},
		{"3", 15},
		{"4", 25},
		{"5", 5},
		{"6", 17},
		{"7", 14},
		{"8", time.Now().UnixNano()},
	}
	d := NewDeleter("", nil, time.Minute, time.Second, 3, 3)
	go d.Start()
	for _, v := range ex {
		d.c <- v
	}
	<-time.NewTimer(time.Second * 3).C

	defer d.Stop()
	d.m.Lock()
	defer d.m.Unlock()
	if l := len(d.lastActionTimestamp); l != 1 {
		t.Errorf("want: 1, got: %d", l)
	}
}

func benchForUsers(n, a, x, y int, b *testing.B) {
	ids := [][]byte{}
	for i := 0; i < n; i++ {
		ids = append(ids, []byte(fmt.Sprint(i)))
	}

	for i := 0; i < b.N; i++ {
		d := NewDeleter("", nil, time.Minute, 5*time.Second, x, y)
		go d.Start()
		for _, id := range ids {
			for j := 0; j < a; j++ {
				d.c <- &actionTimestamp{id, time.Now().UnixNano()}
			}
		}
		d.Stop()
	}
	b.Logf("added %d", len(ids)*b.N)
}

// TODO think about better performance test
func BenchmarkDel70000(b *testing.B) { benchForUsers(70000, 10, 1000, 50000, b) }

func BenchmarkDel70000x100(b *testing.B) { benchForUsers(70000, 100, 1000, 50000, b) }
<commit_msg>add TestActionTimestampHeap<commit_after>package ttlstore

import (
	"container/heap"
	"fmt"
	"testing"
	"time"
)

func TestNewDeleter(t *testing.T) {
	ex := []*actionTimestamp{
		{"1", 10},
		{"2", 20},
		{"3", 15},
		{"4", 25},
		{"5", 5},
		{"6", 17},
		{"7", 14},
		{"8", time.Now().UnixNano()},
	}
	deleted := make(map[interface{}]bool)
	cnt := 0
	var dh DeleteHook = func(key interface{}) {
		deleted[key] = true
		cnt++
	}
	d := NewDeleter("", dh, time.Minute, time.Hour, 3, 3)
	go d.Process()
	for _, v := range ex {
		d.c <- v
	}
	close(d.c)
	<-d.finished
	if l := len(d.lastActionTimestamp); l != 8 {
		t.Errorf("want: 8, got: %d", l)
	}
	d.deleteUntilBelowMaxUuid()

	if cnt != 5 {
		t.Errorf("deleted, want: 5, got: %d", cnt)
		t.Errorf("%v", deleted)
	}
	if l := len(d.lastActionTimestamp); l != 3 {
		t.Errorf("want: 3, got: %d", l)
	}

	d.DeleteTooOld()
	if cnt != 7 {
		t.Errorf("deleted, want: 7, got: %d", cnt)
		t.Errorf("%v", deleted)
	}
	if l := len(d.lastActionTimestamp); l != 1 {
		t.Errorf("want: 1, got: %d", l)
	}
}

func TestFullDeleter(t *testing.T) {
	ex := []*actionTimestamp{
		{"1", 10},
		{"2", 20},
		{"3", 15},
		{"4", 25},
		{"5", 5},
		{"6", 17},
		{"7", 14},
		{"8", time.Now().UnixNano()},
	}
	d := NewDeleter("", nil, time.Minute, time.Second, 3, 3)
	go d.Start()
	for _, v := range ex {
		d.c <- v
	}
	<-time.NewTimer(time.Second * 3).C

	defer d.Stop()
	d.m.Lock()
	defer d.m.Unlock()
	if l := len(d.lastActionTimestamp); l != 1 {
		t.Errorf("want: 1, got: %d", l)
	}
}

func benchForUsers(n, a, x, y int, b *testing.B) {
	ids := [][]byte{}
	for i := 0; i < n; i++ {
		ids = append(ids, []byte(fmt.Sprint(i)))
	}

	for i := 0; i < b.N; i++ {
		d := NewDeleter("", nil, time.Minute, 5*time.Second, x, y)
		go d.Start()
		for _, id := range ids {
			for j := 0; j < a; j++ {
				d.c <- &actionTimestamp{id, time.Now().UnixNano()}
			}
		}
		d.Stop()
	}
	b.Logf("added %d", len(ids)*b.N)
}

func TestActionTimestmapHeap(t *testing.T) {
	h := &ActionTimestmapHeap{}

	for _, v := range []*actionTimestamp{
		{"1", 10},
		{"2", 20},
		{"3", 15},
		{"4", 25},
		{"5", 5},
		{"6", 17},
		{"7", 14},
	} {
		heap.Push(h, v)
	}
	a := heap.Pop(h).(*actionTimestamp)
	if uid := a.Key.(string); uid != "4" {
		t.Errorf("want 5, got %s", uid)
	}
}

// TODO think about better performance test
func BenchmarkDel70000(b *testing.B) { benchForUsers(70000, 10, 1000, 50000, b) }

func BenchmarkDel70000x100(b *testing.B) { benchForUsers(70000, 100, 1000, 50000, b) }
<|endoftext|>


<commit_before>package drivers

import (
	"database/sql"
	"fmt"
	"net/url"
	"strings"

	_ "github.com/denisenkom/go-mssqldb"
	"github.com/pkg/errors"
	"github.com/vattle/sqlboiler/bdb"
)

// MSSQLDriver holds the database connection string and a handle
// to the database connection.
type MSSQLDriver struct {
	connStr string
	dbConn  *sql.DB
}

// NewMSSQLDriver takes the database connection details as parameters and
// returns a pointer to a MSSQLDriver object. Note that it is required to
// call MSSQLDriver.Open() and MSSQLDriver.Close() to open and close
// the database connection once an object has been obtained.
func NewMSSQLDriver(user, pass, dbname, host string, port int, sslmode string) *MSSQLDriver {
	driver := MSSQLDriver{
		connStr: MSSQLBuildQueryString(user, pass, dbname, host, port, sslmode),
	}

	return &driver
}

// MSSQLBuildQueryString builds a query string for MSSQL.
func MSSQLBuildQueryString(user, pass, dbname, host string, port int, sslmode string) string {

	query := url.Values{}
	query.Add("database", dbname)
	query.Add("encrypt", sslmode)

	u := &url.URL{
		Scheme: "sqlserver",
		User:   url.UserPassword(user, pass),
		Host:   fmt.Sprintf("%s:%d", host, port),
		// Path:  instance, // if connecting to an instance instead of a port
		RawQuery: query.Encode(),
	}

	return u.String()
}

// Open opens the database connection using the connection string
func (m *MSSQLDriver) Open() error {
	var err error
	m.dbConn, err = sql.Open("mssql", m.connStr)
	if err != nil {
		return err
	}

	return nil
}

// Close closes the database connection
func (m *MSSQLDriver) Close() {
	m.dbConn.Close()
}

// UseLastInsertID returns false for mssql
func (m *MSSQLDriver) UseLastInsertID() bool {
	return false
}

// UseTopClause returns true to indicate MS SQL supports SQL TOP clause
func (m *MSSQLDriver) UseTopClause() bool {
	return true
}

// TableNames connects to the postgres database and
// retrieves all table names from the information_schema where the
// table schema is public.
func (m *MSSQLDriver) TableNames(schema string, whitelist, blacklist []string) ([]string, error) {
	var names []string

	query := fmt.Sprintf(`select table_name from information_schema.tables where table_schema = ? and table_type = 'BASE TABLE'`)
	args := []interface{}{schema}
	if len(whitelist) > 0 {
		query += fmt.Sprintf(" and table_name in (%s);", strings.Repeat(",?", len(whitelist))[1:])
		for _, w := range whitelist {
			args = append(args, w)
		}
	} else if len(blacklist) > 0 {
		query += fmt.Sprintf(" and table_name not in (%s);", strings.Repeat(",?", len(blacklist))[1:])
		for _, b := range blacklist {
			args = append(args, b)
		}
	}

	rows, err := m.dbConn.Query(query, args...)

	if err != nil {
		return nil, err
	}

	defer rows.Close()
	for rows.Next() {
		var name string
		if err := rows.Scan(&name); err != nil {
			return nil, err
		}
		names = append(names, name)
	}

	return names, nil
}

// Columns takes a table name and attempts to retrieve the table information
// from the database information_schema.columns. It retrieves the column names
// and column types and returns those as a []Column after TranslateColumnType()
// converts the SQL types to Go types, for example: "varchar" to "string"
func (m *MSSQLDriver) Columns(schema, tableName string) ([]bdb.Column, error) {
	var columns []bdb.Column

	rows, err := m.dbConn.Query(`
	SELECT column_name,
       CASE
         WHEN CHARACTER_MAXIMUM_LENGTH IS NULL THEN data_type
         ELSE data_type + '(' + CAST(CHARACTER_MAXIMUM_LENGTH AS VARCHAR) + ')'
       END AS full_type,
       data_type,
	   column_default,
       CASE
         WHEN IS_NULLABLE = 'YES' THEN 1
         ELSE 0
       END AS is_nullable,
       CASE
         WHEN EXISTS (SELECT c.column_name
                      FROM information_schema.table_constraints tc
                        INNER JOIN information_schema.key_column_usage kcu
                                ON tc.constraint_name = kcu.constraint_name
                               AND tc.table_name = kcu.table_name
                               AND tc.table_schema = kcu.table_schema
                      WHERE c.column_name = kcu.column_name
                      AND   tc.table_name = c.table_name
                      AND   (tc.constraint_type = 'PRIMARY KEY' OR tc.constraint_type = 'UNIQUE')
                      AND   (SELECT COUNT(*)
                             FROM information_schema.key_column_usage
                             WHERE table_schema = kcu.table_schema
                             AND   table_name = tc.table_name
                             AND   constraint_name = tc.constraint_name) = 1) THEN 1
         ELSE 0
       END AS is_unique
	FROM INFORMATION_SCHEMA.columns c
	where table_name = ? and table_schema = ?;
	`, tableName, schema)

	if err != nil {
		return nil, err
	}
	defer rows.Close()

	for rows.Next() {
		var colName, colType, colFullType string
		var nullable, unique bool
		var defaultValue *string
		if err := rows.Scan(&colName, &colFullType, &colType, &defaultValue, &nullable, &unique); err != nil {
			return nil, errors.Wrapf(err, "unable to scan for table %s", tableName)
		}

		column := bdb.Column{
			Name:       colName,
			FullDBType: colFullType, // example: tinyint(1) instead of tinyint
			DBType:     colType,
			Nullable:   nullable,
			Unique:     unique,
		}

		if defaultValue != nil && *defaultValue != "NULL" {
			column.Default = *defaultValue
		}

		columns = append(columns, column)
	}

	return columns, nil
}

// PrimaryKeyInfo looks up the primary key for a table.
func (m *MSSQLDriver) PrimaryKeyInfo(schema, tableName string) (*bdb.PrimaryKey, error) {
	pkey := &bdb.PrimaryKey{}
	var err error

	query := `
	select tc.constraint_name
	from information_schema.table_constraints as tc
	where tc.table_name = ? and tc.constraint_type = 'PRIMARY KEY' and tc.table_schema = ?;`

	row := m.dbConn.QueryRow(query, tableName, schema)
	if err = row.Scan(&pkey.Name); err != nil {
		if err == sql.ErrNoRows {
			return nil, nil
		}
		return nil, err
	}

	queryColumns := `
	select kcu.column_name
	from   information_schema.key_column_usage as kcu
	where  table_name = ? and constraint_name = ? and table_schema = ?;`

	var rows *sql.Rows
	if rows, err = m.dbConn.Query(queryColumns, tableName, pkey.Name, schema); err != nil {
		return nil, err
	}
	defer rows.Close()

	var columns []string
	for rows.Next() {
		var column string

		err = rows.Scan(&column)
		if err != nil {
			return nil, err
		}

		columns = append(columns, column)
	}

	if err = rows.Err(); err != nil {
		return nil, err
	}

	pkey.Columns = columns

	return pkey, nil
}

// ForeignKeyInfo retrieves the foreign keys for a given table name.
func (m *MSSQLDriver) ForeignKeyInfo(schema, tableName string) ([]bdb.ForeignKey, error) {
	var fkeys []bdb.ForeignKey

	query := `
	SELECT
		ccu.constraint_name AS SourceConstraint
		,ccu.table_name AS SourceTable
		,ccu.column_name AS SourceColumn
		,kcu.table_name AS TargetTable
		,kcu.column_name AS TargetColumn
	FROM INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE ccu
    INNER JOIN INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS rc
        ON ccu.CONSTRAINT_NAME = rc.CONSTRAINT_NAME
    INNER JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE kcu
        ON kcu.CONSTRAINT_NAME = rc.UNIQUE_CONSTRAINT_NAME
	where ccu.table_schema = ? and ccu.constraint_schema = ? and ccu.table_name = ?
	`

	var rows *sql.Rows
	var err error
	if rows, err = m.dbConn.Query(query, schema, schema, tableName); err != nil {
		return nil, err
	}

	for rows.Next() {
		var fkey bdb.ForeignKey
		var sourceTable string

		fkey.Table = tableName
		err = rows.Scan(&fkey.Name, &sourceTable, &fkey.Column, &fkey.ForeignTable, &fkey.ForeignColumn)
		if err != nil {
			return nil, err
		}

		fkeys = append(fkeys, fkey)
	}

	if err = rows.Err(); err != nil {
		return nil, err
	}

	return fkeys, nil
}

// TranslateColumnType converts postgres database types to Go types, for example
// "varchar" to "string" and "bigint" to "int64". It returns this parsed data
// as a Column object.
func (m *MSSQLDriver) TranslateColumnType(c bdb.Column) bdb.Column {
	if c.Nullable {
		switch c.DBType {
		case "bit":
			c.Type = "bool"
		case "tinyint":
			c.Type = "int8"
		case "smallint":
			c.Type = "null.Int16"
		case "mediumint":
			c.Type = "null.Int32"
		case "int", "integer":
			c.Type = "null.Int"
		case "bigint":
			c.Type = "null.Int64"
		case "float":
			c.Type = "null.Float32"
		case "double", "double precision", "real":
			c.Type = "null.Float64"
		case "boolean", "bool":
			c.Type = "null.Bool"
		case "date", "datetime", "timestamp", "time":
			c.Type = "null.Time"
		case "binary", "varbinary", "tinyblob", "blob", "mediumblob", "longblob":
			c.Type = "null.Bytes"
		case "json":
			c.Type = "types.JSON"
		default:
			c.Type = "null.String"
		}
	} else {
		switch c.DBType {
		case "bit":
			c.Type = "bool"
		case "tinyint":
			c.Type = "int8"
		case "smallint":
			c.Type = "int16"
		case "mediumint":
			c.Type = "int32"
		case "int", "integer":
			c.Type = "int"
		case "bigint":
			c.Type = "int64"
		case "float":
			c.Type = "float32"
		case "double", "double precision", "real":
			c.Type = "float64"
		case "boolean", "bool":
			c.Type = "bool"
		case "date", "datetime", "timestamp", "time":
			c.Type = "time.Time"
		case "binary", "varbinary", "tinyblob", "blob", "mediumblob", "longblob":
			c.Type = "[]byte"
		case "json":
			c.Type = "types.JSON"
		default:
			c.Type = "string"
		}
	}

	return c
}

// RightQuote is the quoting character for the right side of the identifier
func (m *MSSQLDriver) RightQuote() byte {
	return '"'
}

// LeftQuote is the quoting character for the left side of the identifier
func (m *MSSQLDriver) LeftQuote() byte {
	return '"'
}

// IndexPlaceholders returns false to indicate MS SQL doesnt support indexed placeholders
func (m *MSSQLDriver) IndexPlaceholders() bool {
	return false
}
<commit_msg>Allow MS SQL driver to use indexed placeholders<commit_after>package drivers

import (
	"database/sql"
	"fmt"
	"net/url"
	"strings"

	_ "github.com/denisenkom/go-mssqldb"
	"github.com/pkg/errors"
	"github.com/vattle/sqlboiler/bdb"
)

// MSSQLDriver holds the database connection string and a handle
// to the database connection.
type MSSQLDriver struct {
	connStr string
	dbConn  *sql.DB
}

// NewMSSQLDriver takes the database connection details as parameters and
// returns a pointer to a MSSQLDriver object. Note that it is required to
// call MSSQLDriver.Open() and MSSQLDriver.Close() to open and close
// the database connection once an object has been obtained.
func NewMSSQLDriver(user, pass, dbname, host string, port int, sslmode string) *MSSQLDriver {
	driver := MSSQLDriver{
		connStr: MSSQLBuildQueryString(user, pass, dbname, host, port, sslmode),
	}

	return &driver
}

// MSSQLBuildQueryString builds a query string for MSSQL.
func MSSQLBuildQueryString(user, pass, dbname, host string, port int, sslmode string) string {

	query := url.Values{}
	query.Add("database", dbname)
	query.Add("encrypt", sslmode)

	u := &url.URL{
		Scheme: "sqlserver",
		User:   url.UserPassword(user, pass),
		Host:   fmt.Sprintf("%s:%d", host, port),
		// Path:  instance, // if connecting to an instance instead of a port
		RawQuery: query.Encode(),
	}

	return u.String()
}

// Open opens the database connection using the connection string
func (m *MSSQLDriver) Open() error {
	var err error
	m.dbConn, err = sql.Open("mssql", m.connStr)
	if err != nil {
		return err
	}

	return nil
}

// Close closes the database connection
func (m *MSSQLDriver) Close() {
	m.dbConn.Close()
}

// UseLastInsertID returns false for mssql
func (m *MSSQLDriver) UseLastInsertID() bool {
	return false
}

// UseTopClause returns true to indicate MS SQL supports SQL TOP clause
func (m *MSSQLDriver) UseTopClause() bool {
	return true
}

// TableNames connects to the postgres database and
// retrieves all table names from the information_schema where the
// table schema is public.
func (m *MSSQLDriver) TableNames(schema string, whitelist, blacklist []string) ([]string, error) {
	var names []string

	query := fmt.Sprintf(`select table_name from information_schema.tables where table_schema = ? and table_type = 'BASE TABLE'`)
	args := []interface{}{schema}
	if len(whitelist) > 0 {
		query += fmt.Sprintf(" and table_name in (%s);", strings.Repeat(",?", len(whitelist))[1:])
		for _, w := range whitelist {
			args = append(args, w)
		}
	} else if len(blacklist) > 0 {
		query += fmt.Sprintf(" and table_name not in (%s);", strings.Repeat(",?", len(blacklist))[1:])
		for _, b := range blacklist {
			args = append(args, b)
		}
	}

	rows, err := m.dbConn.Query(query, args...)

	if err != nil {
		return nil, err
	}

	defer rows.Close()
	for rows.Next() {
		var name string
		if err := rows.Scan(&name); err != nil {
			return nil, err
		}
		names = append(names, name)
	}

	return names, nil
}

// Columns takes a table name and attempts to retrieve the table information
// from the database information_schema.columns. It retrieves the column names
// and column types and returns those as a []Column after TranslateColumnType()
// converts the SQL types to Go types, for example: "varchar" to "string"
func (m *MSSQLDriver) Columns(schema, tableName string) ([]bdb.Column, error) {
	var columns []bdb.Column

	rows, err := m.dbConn.Query(`
	SELECT column_name,
       CASE
         WHEN CHARACTER_MAXIMUM_LENGTH IS NULL THEN data_type
         ELSE data_type + '(' + CAST(CHARACTER_MAXIMUM_LENGTH AS VARCHAR) + ')'
       END AS full_type,
       data_type,
	   column_default,
       CASE
         WHEN IS_NULLABLE = 'YES' THEN 1
         ELSE 0
       END AS is_nullable,
       CASE
         WHEN EXISTS (SELECT c.column_name
                      FROM information_schema.table_constraints tc
                        INNER JOIN information_schema.key_column_usage kcu
                                ON tc.constraint_name = kcu.constraint_name
                               AND tc.table_name = kcu.table_name
                               AND tc.table_schema = kcu.table_schema
                      WHERE c.column_name = kcu.column_name
                      AND   tc.table_name = c.table_name
                      AND   (tc.constraint_type = 'PRIMARY KEY' OR tc.constraint_type = 'UNIQUE')
                      AND   (SELECT COUNT(*)
                             FROM information_schema.key_column_usage
                             WHERE table_schema = kcu.table_schema
                             AND   table_name = tc.table_name
                             AND   constraint_name = tc.constraint_name) = 1) THEN 1
         ELSE 0
       END AS is_unique
	FROM INFORMATION_SCHEMA.columns c
	where table_name = ? and table_schema = ?;
	`, tableName, schema)

	if err != nil {
		return nil, err
	}
	defer rows.Close()

	for rows.Next() {
		var colName, colType, colFullType string
		var nullable, unique bool
		var defaultValue *string
		if err := rows.Scan(&colName, &colFullType, &colType, &defaultValue, &nullable, &unique); err != nil {
			return nil, errors.Wrapf(err, "unable to scan for table %s", tableName)
		}

		column := bdb.Column{
			Name:       colName,
			FullDBType: colFullType, // example: tinyint(1) instead of tinyint
			DBType:     colType,
			Nullable:   nullable,
			Unique:     unique,
		}

		if defaultValue != nil && *defaultValue != "NULL" {
			column.Default = *defaultValue
		}

		columns = append(columns, column)
	}

	return columns, nil
}

// PrimaryKeyInfo looks up the primary key for a table.
func (m *MSSQLDriver) PrimaryKeyInfo(schema, tableName string) (*bdb.PrimaryKey, error) {
	pkey := &bdb.PrimaryKey{}
	var err error

	query := `
	select tc.constraint_name
	from information_schema.table_constraints as tc
	where tc.table_name = ? and tc.constraint_type = 'PRIMARY KEY' and tc.table_schema = ?;`

	row := m.dbConn.QueryRow(query, tableName, schema)
	if err = row.Scan(&pkey.Name); err != nil {
		if err == sql.ErrNoRows {
			return nil, nil
		}
		return nil, err
	}

	queryColumns := `
	select kcu.column_name
	from   information_schema.key_column_usage as kcu
	where  table_name = ? and constraint_name = ? and table_schema = ?;`

	var rows *sql.Rows
	if rows, err = m.dbConn.Query(queryColumns, tableName, pkey.Name, schema); err != nil {
		return nil, err
	}
	defer rows.Close()

	var columns []string
	for rows.Next() {
		var column string

		err = rows.Scan(&column)
		if err != nil {
			return nil, err
		}

		columns = append(columns, column)
	}

	if err = rows.Err(); err != nil {
		return nil, err
	}

	pkey.Columns = columns

	return pkey, nil
}

// ForeignKeyInfo retrieves the foreign keys for a given table name.
func (m *MSSQLDriver) ForeignKeyInfo(schema, tableName string) ([]bdb.ForeignKey, error) {
	var fkeys []bdb.ForeignKey

	query := `
	SELECT
		ccu.constraint_name AS SourceConstraint
		,ccu.table_name AS SourceTable
		,ccu.column_name AS SourceColumn
		,kcu.table_name AS TargetTable
		,kcu.column_name AS TargetColumn
	FROM INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE ccu
    INNER JOIN INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS rc
        ON ccu.CONSTRAINT_NAME = rc.CONSTRAINT_NAME
    INNER JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE kcu
        ON kcu.CONSTRAINT_NAME = rc.UNIQUE_CONSTRAINT_NAME
	where ccu.table_schema = ? and ccu.constraint_schema = ? and ccu.table_name = ?
	`

	var rows *sql.Rows
	var err error
	if rows, err = m.dbConn.Query(query, schema, schema, tableName); err != nil {
		return nil, err
	}

	for rows.Next() {
		var fkey bdb.ForeignKey
		var sourceTable string

		fkey.Table = tableName
		err = rows.Scan(&fkey.Name, &sourceTable, &fkey.Column, &fkey.ForeignTable, &fkey.ForeignColumn)
		if err != nil {
			return nil, err
		}

		fkeys = append(fkeys, fkey)
	}

	if err = rows.Err(); err != nil {
		return nil, err
	}

	return fkeys, nil
}

// TranslateColumnType converts postgres database types to Go types, for example
// "varchar" to "string" and "bigint" to "int64". It returns this parsed data
// as a Column object.
func (m *MSSQLDriver) TranslateColumnType(c bdb.Column) bdb.Column {
	if c.Nullable {
		switch c.DBType {
		case "bit":
			c.Type = "bool"
		case "tinyint":
			c.Type = "int8"
		case "smallint":
			c.Type = "null.Int16"
		case "mediumint":
			c.Type = "null.Int32"
		case "int", "integer":
			c.Type = "null.Int"
		case "bigint":
			c.Type = "null.Int64"
		case "float":
			c.Type = "null.Float32"
		case "double", "double precision", "real":
			c.Type = "null.Float64"
		case "boolean", "bool":
			c.Type = "null.Bool"
		case "date", "datetime", "timestamp", "time":
			c.Type = "null.Time"
		case "binary", "varbinary", "tinyblob", "blob", "mediumblob", "longblob":
			c.Type = "null.Bytes"
		case "json":
			c.Type = "types.JSON"
		default:
			c.Type = "null.String"
		}
	} else {
		switch c.DBType {
		case "bit":
			c.Type = "bool"
		case "tinyint":
			c.Type = "int8"
		case "smallint":
			c.Type = "int16"
		case "mediumint":
			c.Type = "int32"
		case "int", "integer":
			c.Type = "int"
		case "bigint":
			c.Type = "int64"
		case "float":
			c.Type = "float32"
		case "double", "double precision", "real":
			c.Type = "float64"
		case "boolean", "bool":
			c.Type = "bool"
		case "date", "datetime", "timestamp", "time":
			c.Type = "time.Time"
		case "binary", "varbinary", "tinyblob", "blob", "mediumblob", "longblob":
			c.Type = "[]byte"
		case "json":
			c.Type = "types.JSON"
		default:
			c.Type = "string"
		}
	}

	return c
}

// RightQuote is the quoting character for the right side of the identifier
func (m *MSSQLDriver) RightQuote() byte {
	return '"'
}

// LeftQuote is the quoting character for the left side of the identifier
func (m *MSSQLDriver) LeftQuote() byte {
	return '"'
}

// IndexPlaceholders returns true to indicate MS SQL supports indexed placeholders
func (m *MSSQLDriver) IndexPlaceholders() bool {
	return true
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"os"
	"time"

	"github.com/aclements/go-moremath/stats"
)

var cmdPlotFlags = flag.NewFlagSet(os.Args[0]+" plot", flag.ExitOnError)

var (
	plotBaseline string
)

// Currently I'm plotting this using gnuplot:
//
// plot for [i=3:50] 'data' using 0:i index "ns/op" with l title columnhead

// TODO: HTML output using Google Charts?

// TODO: Low-pass filter?

func init() {
	f := cmdPlotFlags
	f.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s plot [flags] <revision range>\n", os.Args[0])
		f.PrintDefaults()
	}
	f.StringVar(&gitDir, "C", "", "run git in `dir`")
	f.StringVar(&plotBaseline, "baseline", "", "normalize to `revision`; revision may be \"first\" or \"last\"")
	registerSubcommand("plot", "[flags] <revision range> - print benchmark results", cmdPlot, f)
}

func cmdPlot() {
	if cmdPlotFlags.NArg() < 1 {
		cmdPlotFlags.Usage()
		os.Exit(2)
	}

	commits := getCommits(cmdPlotFlags.Args())

	// Put commits in forward chronological order.
	for i, j := 0, len(commits)-1; i < j; i, j = i+1, j-1 {
		commits[i], commits[j] = commits[j], commits[i]
	}

	// Load benchmark statistics.
	logFiles := []string{}
	for _, commit := range commits {
		if exists(commit.logPath) {
			logFiles = append(logFiles, commit.logPath)
		}
	}
	c := readFiles(logFiles...)
	for _, stat := range c.Stats {
		stat.ComputeStats()
	}

	// Get baseline commit.
	var baselineCommit *commitInfo
	if plotBaseline == "first" || plotBaseline == "last" {
		// Find the first/last commit with benchmark data.
		for _, commit := range commits {
			if c.ConfigSet[commit.logPath] {
				baselineCommit = commit
				if plotBaseline == "first" {
					break
				}
			}
		}
	} else if plotBaseline != "" {
		hash := trimNL(git("rev-parse", "--", plotBaseline))
		for _, commit := range commits {
			if hash == commit.hash {
				baselineCommit = commit
				break
			}
		}
		if baselineCommit == nil {
			fmt.Fprintf(os.Stderr, "baseline revision %s not found in revision range\n", hash)
			os.Exit(2)
		}
	}
	if baselineCommit != nil && !c.ConfigSet[baselineCommit.logPath] {
		fmt.Fprintf(os.Stderr, "no benchmark data for baseline commit\n")
		os.Exit(2)
	}

	// Build tables.
	var tables []*Table
	var key BenchKey
	baseline := make([]float64, 0)
	means := make([]float64, 0)
	for _, unit := range c.Units {
		key.Unit = unit
		table := &Table{Unit: unit}
		tables = append(tables, table)

		// Print table of commit vs. benchmark mean.
		subc := c.Filter(BenchKey{Unit: unit})
		table.Rows = [][]interface{}{{"date", "commit", "geomean"}}
		for _, bench := range subc.Benchmarks {
			table.Rows[0] = append(table.Rows[0], bench)
		}

		// Get baseline numbers.
		baseline = baseline[:0]
		if baselineCommit == nil {
			for range subc.Benchmarks {
				baseline = append(baseline, 1)
			}
		} else {
			key.Config = baselineCommit.logPath
			for _, bench := range subc.Benchmarks {
				key.Benchmark = bench
				baseline = append(baseline, subc.Stats[key].Mean)
			}
		}

		for _, commit := range commits {
			key.Config = commit.logPath
			if !subc.ConfigSet[commit.logPath] {
				continue
			}

			means = means[:0]
			for _, bench := range subc.Benchmarks {
				key.Benchmark = bench
				means = append(means, subc.Stats[key].Mean)
			}

			row := []interface{}{commit.commitDate.Format(time.RFC3339), commit.hash[:7], stats.GeoMean(means) / stats.GeoMean(baseline)}
			for i, bench := range subc.Benchmarks {
				key.Benchmark = bench
				row = append(row, subc.Stats[key].Mean/baseline[i])
			}
			table.Rows = append(table.Rows, row)
		}
	}

	// Print tables.
	for i, table := range tables {
		if i > 0 {
			fmt.Printf("\n\n")
		}
		fmt.Printf("# %s\n", table.Unit)

		for _, row := range table.Rows {
			for i, val := range row {
				if i > 0 {
					fmt.Printf(" ")
				}
				switch val := val.(type) {
				case float64:
					fmt.Printf("%g", val)
				case float32:
					fmt.Printf("%g", val)
				default:
					fmt.Printf("%s", val)
				}
			}
			fmt.Printf("\n")
		}
	}
}

type Table struct {
	Unit string
	Rows [][]interface{}
}
<commit_msg>benchmany: add JSON output<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"
	"time"

	"github.com/aclements/go-moremath/stats"
)

var cmdPlotFlags = flag.NewFlagSet(os.Args[0]+" plot", flag.ExitOnError)

var (
	plotBaseline string
	plotJSON     bool
)

// Currently I'm plotting this using gnuplot:
//
// plot for [i=3:50] 'data' using 0:i index "ns/op" with l title columnhead

// TODO: HTML output using Google Charts?

// TODO: Low-pass filter?

func init() {
	f := cmdPlotFlags
	f.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s plot [flags] <revision range>\n", os.Args[0])
		f.PrintDefaults()
	}
	f.StringVar(&gitDir, "C", "", "run git in `dir`")
	f.StringVar(&plotBaseline, "baseline", "", "normalize to `revision`; revision may be \"first\" or \"last\"")
	f.BoolVar(&plotJSON, "json", false, "emit data in JSON")
	registerSubcommand("plot", "[flags] <revision range> - print benchmark results", cmdPlot, f)
}

func cmdPlot() {
	if cmdPlotFlags.NArg() < 1 {
		cmdPlotFlags.Usage()
		os.Exit(2)
	}

	commits := getCommits(cmdPlotFlags.Args())

	// Put commits in forward chronological order.
	for i, j := 0, len(commits)-1; i < j; i, j = i+1, j-1 {
		commits[i], commits[j] = commits[j], commits[i]
	}

	// Load benchmark statistics.
	logFiles := []string{}
	for _, commit := range commits {
		if exists(commit.logPath) {
			logFiles = append(logFiles, commit.logPath)
		}
	}
	c := readFiles(logFiles...)
	for _, stat := range c.Stats {
		stat.ComputeStats()
	}

	// Get baseline commit.
	var baselineCommit *commitInfo
	if plotBaseline == "first" || plotBaseline == "last" {
		// Find the first/last commit with benchmark data.
		for _, commit := range commits {
			if c.ConfigSet[commit.logPath] {
				baselineCommit = commit
				if plotBaseline == "first" {
					break
				}
			}
		}
	} else if plotBaseline != "" {
		hash := trimNL(git("rev-parse", "--", plotBaseline))
		for _, commit := range commits {
			if hash == commit.hash {
				baselineCommit = commit
				break
			}
		}
		if baselineCommit == nil {
			fmt.Fprintf(os.Stderr, "baseline revision %s not found in revision range\n", hash)
			os.Exit(2)
		}
	}
	if baselineCommit != nil && !c.ConfigSet[baselineCommit.logPath] {
		fmt.Fprintf(os.Stderr, "no benchmark data for baseline commit\n")
		os.Exit(2)
	}

	// Build tables.
	var tables []*Table
	var key BenchKey
	baseline := make([]float64, 0)
	means := make([]float64, 0)
	for _, unit := range c.Units {
		key.Unit = unit
		table := &Table{Unit: unit}
		tables = append(tables, table)

		// Print table of commit vs. benchmark mean.
		subc := c.Filter(BenchKey{Unit: unit})
		table.Rows = [][]interface{}{{"date", "commit", "geomean"}}
		for _, bench := range subc.Benchmarks {
			table.Rows[0] = append(table.Rows[0], bench)
		}

		// Get baseline numbers.
		baseline = baseline[:0]
		if baselineCommit == nil {
			for range subc.Benchmarks {
				baseline = append(baseline, 1)
			}
		} else {
			key.Config = baselineCommit.logPath
			for _, bench := range subc.Benchmarks {
				key.Benchmark = bench
				baseline = append(baseline, subc.Stats[key].Mean)
			}
		}

		for _, commit := range commits {
			key.Config = commit.logPath
			if !subc.ConfigSet[commit.logPath] {
				continue
			}

			means = means[:0]
			for _, bench := range subc.Benchmarks {
				key.Benchmark = bench
				means = append(means, subc.Stats[key].Mean)
			}

			row := []interface{}{commit.commitDate.Format(time.RFC3339), commit.hash[:7], stats.GeoMean(means) / stats.GeoMean(baseline)}
			for i, bench := range subc.Benchmarks {
				key.Benchmark = bench
				row = append(row, subc.Stats[key].Mean/baseline[i])
			}
			table.Rows = append(table.Rows, row)
		}
	}

	if plotJSON {
		if err := json.NewEncoder(os.Stdout).Encode(tables); err != nil {
			log.Fatal(err)
		}
	} else {
		// Print tables.
		for i, table := range tables {
			if i > 0 {
				fmt.Printf("\n\n")
			}
			fmt.Printf("# %s\n", table.Unit)

			for _, row := range table.Rows {
				for i, val := range row {
					if i > 0 {
						fmt.Printf(" ")
					}
					switch val := val.(type) {
					case float64:
						fmt.Printf("%g", val)
					case float32:
						fmt.Printf("%g", val)
					default:
						fmt.Printf("%s", val)
					}
				}
				fmt.Printf("\n")
			}
		}
	}
}

type Table struct {
	Unit string
	Rows [][]interface{}
}
<|endoftext|>


<commit_before>package commands

import (
	"bufio"
	"cred-alert/inflator"
	"cred-alert/kolsch"
	"cred-alert/mimetype"
	"cred-alert/scanners"
	"cred-alert/scanners/diffscanner"
	"cred-alert/scanners/dirscanner"
	"cred-alert/scanners/filescanner"
	"cred-alert/sniff"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"time"

	"code.cloudfoundry.org/lager"
)

type ScanCommand struct {
	File string `short:"f" long:"file" description:"the file to scan" value-name:"FILE"`
	Diff bool   `long:"diff" description:"content to be scanned is a git diff"`
}

func (command *ScanCommand) Execute(args []string) error {
	logger := kolsch.NewLogger()
	sniffer := sniff.NewDefaultSniffer()
	inflate := inflator.New()

	exitFuncs := []func(){
		func() { inflate.Close() },
	}

	var credsFound int
	handler := func(logger lager.Logger, line scanners.Line) error {
		credsFound++
		fmt.Printf("%s %s:%d\n", red("[CRED]"), line.Path, line.LineNumber)

		return nil
	}

	if command.File != "" {
		fh, err := os.Open(command.File)
		if err != nil {
			log.Fatalln(err.Error())
		}

		br := bufio.NewReader(fh)
		if mime, isArchive := mimetype.IsArchive(logger, br); isArchive {
			inflateDir, err := ioutil.TempDir("", "cred-alert-cli")
			if err != nil {
				log.Fatalln(err.Error())
			}
			exitFuncs = append(exitFuncs, func() { os.RemoveAll(inflateDir) })

			violationsDir, err := ioutil.TempDir("", "cred-alert-cli-violations")
			if err != nil {
				log.Fatalln(err.Error())
			}

			archiveViolationHandler := func(logger lager.Logger, line scanners.Line) error {
				credsFound++
				relPath, err := filepath.Rel(inflateDir, line.Path)
				if err != nil {
					return err
				}

				destPath := filepath.Join(violationsDir, relPath)
				err = os.MkdirAll(filepath.Dir(destPath), os.ModePerm)
				if err != nil {
					return err
				}

				err = persistFile(line.Path, destPath)
				if err != nil {
					return err
				}

				fmt.Printf("%s %s:%d\n", red("[CRED]"), destPath, line.LineNumber)

				return nil
			}

			inflateStart := time.Now()
			fmt.Printf("Inflating archive... ")
			err = inflate.Inflate(logger, mime, command.File, inflateDir)
			if err != nil {
				fmt.Printf("%s\n", red("FAILED"))
				log.Fatalln(err.Error())
			}
			fmt.Printf("%s (%s)\n", green("DONE"), time.Since(inflateStart))

			scanStart := time.Now()
			dirScanner := dirscanner.New(archiveViolationHandler, sniffer)
			err = dirScanner.Scan(logger, inflateDir)
			if err != nil {
				log.Fatalln(err.Error())
			}

			fmt.Println()
			fmt.Println("Scan complete!")
			fmt.Println()
			fmt.Println("Time taken:", time.Since(scanStart))
			fmt.Println("Credentials found:", credsFound)
			fmt.Println()
			fmt.Println("Any archive inflation errors can be found in: ", inflate.LogPath())
		} else {
			scanFile(logger, handler, sniffer, br, command.File)
		}
	} else if command.Diff {
		handleDiff(logger, handler)

	} else {
		scanFile(logger, handler, sniffer, os.Stdin, "STDIN")
	}

	if credsFound > 0 {
		exitFuncs = append(exitFuncs, func() {
			os.Exit(3)
		})
	}

	for _, f := range exitFuncs {
		f()
	}

	return nil
}

func persistFile(srcPath, destPath string) error {
	destFile, err := os.Create(destPath)
	if err != nil {
		return err
	}
	defer destFile.Close()

	srcFile, err := os.Open(srcPath)
	if err != nil {
		return err
	}
	defer srcFile.Close()

	_, err = io.Copy(destFile, srcFile)
	return err
}

func scanFile(
	logger lager.Logger,
	handler sniff.ViolationHandlerFunc,
	sniffer sniff.Sniffer,
	f io.Reader,
	name string,
) {
	scanner := filescanner.New(f, name)
	sniffer.Sniff(logger, scanner, handler)
}

func handleDiff(logger lager.Logger, handler sniff.ViolationHandlerFunc) {
	logger.Session("handle-diff")
	scanner := diffscanner.NewDiffScanner(os.Stdin)
	sniffer := sniff.NewDefaultSniffer()

	sniffer.Sniff(logger, scanner, handler)
}
<commit_msg>Clean up when being interrupted<commit_after>package commands

import (
	"bufio"
	"cred-alert/inflator"
	"cred-alert/kolsch"
	"cred-alert/mimetype"
	"cred-alert/scanners"
	"cred-alert/scanners/diffscanner"
	"cred-alert/scanners/dirscanner"
	"cred-alert/scanners/filescanner"
	"cred-alert/sniff"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"path/filepath"
	"time"

	"code.cloudfoundry.org/lager"
)

type ScanCommand struct {
	File string `short:"f" long:"file" description:"the file to scan" value-name:"FILE"`
	Diff bool   `long:"diff" description:"content to be scanned is a git diff"`
}

func (command *ScanCommand) Execute(args []string) error {
	logger := kolsch.NewLogger()
	sniffer := sniff.NewDefaultSniffer()
	inflate := inflator.New()

	exitFuncs := []func(){
		func() { inflate.Close() },
	}

	signalsCh := make(chan os.Signal, 1)
	signal.Notify(signalsCh, os.Interrupt)

	go func() {
		for {
			select {
			case <-signalsCh:
				for _, f := range exitFuncs {
					f()
				}
				os.Exit(1)
			}
		}
	}()

	var credsFound int
	handler := func(logger lager.Logger, line scanners.Line) error {
		credsFound++
		fmt.Printf("%s %s:%d\n", red("[CRED]"), line.Path, line.LineNumber)

		return nil
	}

	if command.File != "" {
		fh, err := os.Open(command.File)
		if err != nil {
			log.Fatalln(err.Error())
		}

		br := bufio.NewReader(fh)
		if mime, isArchive := mimetype.IsArchive(logger, br); isArchive {
			inflateDir, err := ioutil.TempDir("", "cred-alert-cli")
			if err != nil {
				log.Fatalln(err.Error())
			}
			exitFuncs = append(exitFuncs, func() { os.RemoveAll(inflateDir) })

			violationsDir, err := ioutil.TempDir("", "cred-alert-cli-violations")
			if err != nil {
				log.Fatalln(err.Error())
			}

			archiveViolationHandler := func(logger lager.Logger, line scanners.Line) error {
				credsFound++
				relPath, err := filepath.Rel(inflateDir, line.Path)
				if err != nil {
					return err
				}

				destPath := filepath.Join(violationsDir, relPath)
				err = os.MkdirAll(filepath.Dir(destPath), os.ModePerm)
				if err != nil {
					return err
				}

				err = persistFile(line.Path, destPath)
				if err != nil {
					return err
				}

				fmt.Printf("%s %s:%d\n", red("[CRED]"), destPath, line.LineNumber)

				return nil
			}

			inflateStart := time.Now()
			fmt.Printf("Inflating archive... ")
			err = inflate.Inflate(logger, mime, command.File, inflateDir)
			if err != nil {
				fmt.Printf("%s\n", red("FAILED"))
				log.Fatalln(err.Error())
			}
			fmt.Printf("%s (%s)\n", green("DONE"), time.Since(inflateStart))

			scanStart := time.Now()
			dirScanner := dirscanner.New(archiveViolationHandler, sniffer)
			err = dirScanner.Scan(logger, inflateDir)
			if err != nil {
				log.Fatalln(err.Error())
			}

			fmt.Println()
			fmt.Println("Scan complete!")
			fmt.Println()
			fmt.Println("Time taken:", time.Since(scanStart))
			fmt.Println("Credentials found:", credsFound)
			fmt.Println()
			fmt.Println("Any archive inflation errors can be found in: ", inflate.LogPath())
		} else {
			scanFile(logger, handler, sniffer, br, command.File)
		}
	} else if command.Diff {
		handleDiff(logger, handler)

	} else {
		scanFile(logger, handler, sniffer, os.Stdin, "STDIN")
	}

	if credsFound > 0 {
		exitFuncs = append(exitFuncs, func() {
			os.Exit(3)
		})
	}

	for _, f := range exitFuncs {
		f()
	}

	return nil
}

func persistFile(srcPath, destPath string) error {
	destFile, err := os.Create(destPath)
	if err != nil {
		return err
	}
	defer destFile.Close()

	srcFile, err := os.Open(srcPath)
	if err != nil {
		return err
	}
	defer srcFile.Close()

	_, err = io.Copy(destFile, srcFile)
	return err
}

func scanFile(
	logger lager.Logger,
	handler sniff.ViolationHandlerFunc,
	sniffer sniff.Sniffer,
	f io.Reader,
	name string,
) {
	scanner := filescanner.New(f, name)
	sniffer.Sniff(logger, scanner, handler)
}

func handleDiff(logger lager.Logger, handler sniff.ViolationHandlerFunc) {
	logger.Session("handle-diff")
	scanner := diffscanner.NewDiffScanner(os.Stdin)
	sniffer := sniff.NewDefaultSniffer()

	sniffer.Sniff(logger, scanner, handler)
}
<|endoftext|>


<commit_before>package parser

import (
	"github.com/yuin/goldmark/ast"
	"github.com/yuin/goldmark/text"
	"github.com/yuin/goldmark/util"
)

type codeBlockParser struct {
}

// CodeBlockParser is a BlockParser implementation that parses indented code blocks.
var defaultCodeBlockParser = &codeBlockParser{}

// NewCodeBlockParser returns a new BlockParser that
// parses code blocks.
func NewCodeBlockParser() BlockParser {
	return defaultCodeBlockParser
}

func (b *codeBlockParser) Trigger() []byte {
	return nil
}

func (b *codeBlockParser) Open(parent ast.Node, reader text.Reader, pc Context) (ast.Node, State) {
	line, segment := reader.PeekLine()
	pos, padding := util.IndentPosition(line, reader.LineOffset(), 4)
	if pos < 0 || util.IsBlank(line) {
		return nil, NoChildren
	}
	node := ast.NewCodeBlock()
	reader.AdvanceAndSetPadding(pos, padding)
	_, segment = reader.PeekLine()
	node.Lines().Append(segment)
	reader.Advance(segment.Len() - 1)
	return node, NoChildren

}

func (b *codeBlockParser) Continue(node ast.Node, reader text.Reader, pc Context) State {
	line, segment := reader.PeekLine()
	if util.IsBlank(line) {
		node.Lines().Append(segment.TrimLeftSpaceWidth(4, reader.Source()))
		return Continue | NoChildren
	}
	pos, padding := util.IndentPosition(line, reader.LineOffset(), 4)
	if pos < 0 {
		return Close
	}
	reader.AdvanceAndSetPadding(pos, padding)
	_, segment = reader.PeekLine()

	// if code block line starts with a tab, keep a tab as it is.
	if segment.Padding != 0 {
		preserveLeadingTabInCodeBlock(&segment, reader, 0)
	}

	node.Lines().Append(segment)
	reader.Advance(segment.Len() - 1)
	return Continue | NoChildren
}

func (b *codeBlockParser) Close(node ast.Node, reader text.Reader, pc Context) {
	// trim trailing blank lines
	lines := node.Lines()
	length := lines.Len() - 1
	source := reader.Source()
	for length >= 0 {
		line := lines.At(length)
		if util.IsBlank(line.Value(source)) {
			length--
		} else {
			break
		}
	}
	lines.SetSliced(0, length+1)
}

func (b *codeBlockParser) CanInterruptParagraph() bool {
	return false
}

func (b *codeBlockParser) CanAcceptIndentedLine() bool {
	return true
}

func preserveLeadingTabInCodeBlock(segment *text.Segment, reader text.Reader, indent int) {
	offsetWithPadding := reader.LineOffset() + indent
	sl, ss := reader.Position()
	reader.SetPosition(sl, text.NewSegment(ss.Start-1, ss.Stop))
	if offsetWithPadding == reader.LineOffset() {
		println("here")
		segment.Padding = 0
		segment.Start--
	}
	reader.SetPosition(sl, ss)
}
<commit_msg>Remove debug print<commit_after>package parser

import (
	"github.com/yuin/goldmark/ast"
	"github.com/yuin/goldmark/text"
	"github.com/yuin/goldmark/util"
)

type codeBlockParser struct {
}

// CodeBlockParser is a BlockParser implementation that parses indented code blocks.
var defaultCodeBlockParser = &codeBlockParser{}

// NewCodeBlockParser returns a new BlockParser that
// parses code blocks.
func NewCodeBlockParser() BlockParser {
	return defaultCodeBlockParser
}

func (b *codeBlockParser) Trigger() []byte {
	return nil
}

func (b *codeBlockParser) Open(parent ast.Node, reader text.Reader, pc Context) (ast.Node, State) {
	line, segment := reader.PeekLine()
	pos, padding := util.IndentPosition(line, reader.LineOffset(), 4)
	if pos < 0 || util.IsBlank(line) {
		return nil, NoChildren
	}
	node := ast.NewCodeBlock()
	reader.AdvanceAndSetPadding(pos, padding)
	_, segment = reader.PeekLine()
	node.Lines().Append(segment)
	reader.Advance(segment.Len() - 1)
	return node, NoChildren

}

func (b *codeBlockParser) Continue(node ast.Node, reader text.Reader, pc Context) State {
	line, segment := reader.PeekLine()
	if util.IsBlank(line) {
		node.Lines().Append(segment.TrimLeftSpaceWidth(4, reader.Source()))
		return Continue | NoChildren
	}
	pos, padding := util.IndentPosition(line, reader.LineOffset(), 4)
	if pos < 0 {
		return Close
	}
	reader.AdvanceAndSetPadding(pos, padding)
	_, segment = reader.PeekLine()

	// if code block line starts with a tab, keep a tab as it is.
	if segment.Padding != 0 {
		preserveLeadingTabInCodeBlock(&segment, reader, 0)
	}

	node.Lines().Append(segment)
	reader.Advance(segment.Len() - 1)
	return Continue | NoChildren
}

func (b *codeBlockParser) Close(node ast.Node, reader text.Reader, pc Context) {
	// trim trailing blank lines
	lines := node.Lines()
	length := lines.Len() - 1
	source := reader.Source()
	for length >= 0 {
		line := lines.At(length)
		if util.IsBlank(line.Value(source)) {
			length--
		} else {
			break
		}
	}
	lines.SetSliced(0, length+1)
}

func (b *codeBlockParser) CanInterruptParagraph() bool {
	return false
}

func (b *codeBlockParser) CanAcceptIndentedLine() bool {
	return true
}

func preserveLeadingTabInCodeBlock(segment *text.Segment, reader text.Reader, indent int) {
	offsetWithPadding := reader.LineOffset() + indent
	sl, ss := reader.Position()
	reader.SetPosition(sl, text.NewSegment(ss.Start-1, ss.Stop))
	if offsetWithPadding == reader.LineOffset() {
		segment.Padding = 0
		segment.Start--
	}
	reader.SetPosition(sl, ss)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"errors"
	"fmt"
	"os"
	"path"
	"strings"
	"sync"
)

type PuppetFile struct {
	*os.File
	wg       *sync.WaitGroup
	filename string
}

func NewPuppetFile(puppetfile string) *PuppetFile {
	f, err := os.Open(puppetfile)
	if err != nil {
		return nil
	}

	return &PuppetFile{File: f, wg: &sync.WaitGroup{}, filename: puppetfile}
}

func (m *PuppetFile) Filename() string         { return m.filename }
func (p *PuppetFile) Close()                   { p.File.Close() }
func (p *PuppetFile) moduleProcessedCallback() { p.wg.Done() }

func (p *PuppetFile) parseParameter(line string) string {
	if strings.Contains(line, "=>") {
		return strings.Trim(strings.Split(line, "=>")[1], " \"'")
	} else {
		return strings.Trim(strings.SplitN(line, ":", 3)[2], " \"'")
	}
}

func (p *PuppetFile) parseModule(line string) (PuppetModule, error) {
	var name, repoUrl, repoName, moduleType, installPath, targetFolder, version string
	var tag, ref, branch = "", "", ""

	line = strings.TrimSpace(line)
	if !strings.HasPrefix(line, "mod") {
		return &GitModule{}, errors.New("Error: Module definition not starting with mod")
	}

	for index, part := range strings.Split(line, ",") {
		part = strings.TrimSpace(part)
		switch {
		case strings.HasPrefix(part, "mod"):
			name = strings.FieldsFunc(part, func(r rune) bool {
				return r == '\'' || r == '"'
			})[1]

		// A line will contain : if it's in the form :tag: value or :tag => value
		// if not then it must be a version string, and no further parameter is allowed
		case index == 1 && !strings.Contains(part, "=>") && part != ":latest" && !strings.Contains(part, ":"):
			moduleType = "forge"
			version = strings.Trim(part, " \"'")

		case strings.HasPrefix(part, ":github_tarball"):
			moduleType = "github_tarball"
			repoName = p.parseParameter(part)

		case strings.HasPrefix(part, ":git"):
			moduleType = "git"
			repoUrl = p.parseParameter(part)

		case strings.HasPrefix(part, ":install_path"):
			installPath = p.parseParameter(part)

		case strings.HasPrefix(part, ":tag"):
			tag = p.parseParameter(part)

		case strings.HasPrefix(part, ":ref"):
			ref = p.parseParameter(part)

		case strings.HasPrefix(part, ":branch"):
			branch = p.parseParameter(part)

		default:
			fmt.Println("Unsupported parameter: " + part)
		}
	}

	switch {
	case moduleType == "git":
		return &GitModule{
			name:        name,
			repoUrl:     repoUrl,
			installPath: installPath,
			processed:   p.moduleProcessedCallback,
			want: struct {
				ref    string
				tag    string
				branch string
			}{
				ref,
				tag,
				branch,
			},
			targetFolder: targetFolder,
			cacheFolder:  ""}, nil

	case moduleType == "github_tarball":
		return &GithubTarballModule{
			name:         name,
			repoName:     repoName,
			version:      version,
			targetFolder: targetFolder,
			processed:    p.moduleProcessedCallback,
			cacheFolder:  "",
		}, nil

	default:
		return &ForgeModule{name: name, version: version, processed: p.moduleProcessedCallback}, nil
	}
}

func (p *PuppetFile) parsePuppetFile(s *bufio.Scanner) ([]PuppetModule, map[string]string) {
	opts := make(map[string]string)
	modules := make([]PuppetModule, 0, 5)

	for block := ""; s.Scan(); {
		line := s.Text()
		line = strings.Split(s.Text(), "#")[0] // Remove comments

		if len(line) == 0 {
			continue
		}

		block += line

		optionValue := func(block string) string {
			return strings.FieldsFunc(block, func(r rune) bool {
				return r == '\'' || r == '"'
			})[1]
		}

		if !strings.HasSuffix(line, ",") { // Full Block
			switch {
			case strings.HasPrefix(block, "forge"):
				opts["forge"] = optionValue(block)

			case strings.HasPrefix(block, "exclusion"):
				opts[""] = optionValue(block)

			case strings.HasPrefix(block, "moduledir"):
				opts["moduledir"] = optionValue(block)

			case strings.HasPrefix(block, "mod"):
				module, _ := p.parseModule(block)
				modules = append(modules, module)
			}
			block = ""
		}
	}

	return modules, opts
}

func (p *PuppetFile) Process(modules chan<- PuppetModule, done func()) error {
	parsedModules, opts := p.parsePuppetFile(bufio.NewScanner(p.File))
	modulePath, ok := opts["moduledir"]
	if !ok {
		modulePath = "modules"
	}

	for _, module := range parsedModules {
		splitPath := strings.Split(module.Name(), "/")
		folderName := splitPath[len(splitPath)-1]
		module.SetTargetFolder(path.Join(modulePath, folderName))

		p.wg.Add(1)
		modules <- module
	}

	go func() {
		p.wg.Wait()
		done()
	}()

	return nil
}
<commit_msg>Fail if Puppetfile can not be found<commit_after>package main

import (
	"bufio"
	"errors"
	"fmt"
	"log"
	"os"
	"path"
	"strings"
	"sync"
)

type PuppetFile struct {
	*os.File
	wg       *sync.WaitGroup
	filename string
}

func NewPuppetFile(puppetfile string) *PuppetFile {
	f, err := os.Open(puppetfile)
	if err != nil {
		log.Fatalf("could not open %s: %v", puppetfile, err)
	}

	return &PuppetFile{File: f, wg: &sync.WaitGroup{}, filename: puppetfile}
}

func (m *PuppetFile) Filename() string         { return m.filename }
func (p *PuppetFile) Close()                   { p.File.Close() }
func (p *PuppetFile) moduleProcessedCallback() { p.wg.Done() }

func (p *PuppetFile) parseParameter(line string) string {
	if strings.Contains(line, "=>") {
		return strings.Trim(strings.Split(line, "=>")[1], " \"'")
	} else {
		return strings.Trim(strings.SplitN(line, ":", 3)[2], " \"'")
	}
}

func (p *PuppetFile) parseModule(line string) (PuppetModule, error) {
	var name, repoUrl, repoName, moduleType, installPath, targetFolder, version string
	var tag, ref, branch = "", "", ""

	line = strings.TrimSpace(line)
	if !strings.HasPrefix(line, "mod") {
		return &GitModule{}, errors.New("Error: Module definition not starting with mod")
	}

	for index, part := range strings.Split(line, ",") {
		part = strings.TrimSpace(part)
		switch {
		case strings.HasPrefix(part, "mod"):
			name = strings.FieldsFunc(part, func(r rune) bool {
				return r == '\'' || r == '"'
			})[1]

		// A line will contain : if it's in the form :tag: value or :tag => value
		// if not then it must be a version string, and no further parameter is allowed
		case index == 1 && !strings.Contains(part, "=>") && part != ":latest" && !strings.Contains(part, ":"):
			moduleType = "forge"
			version = strings.Trim(part, " \"'")

		case strings.HasPrefix(part, ":github_tarball"):
			moduleType = "github_tarball"
			repoName = p.parseParameter(part)

		case strings.HasPrefix(part, ":git"):
			moduleType = "git"
			repoUrl = p.parseParameter(part)

		case strings.HasPrefix(part, ":install_path"):
			installPath = p.parseParameter(part)

		case strings.HasPrefix(part, ":tag"):
			tag = p.parseParameter(part)

		case strings.HasPrefix(part, ":ref"):
			ref = p.parseParameter(part)

		case strings.HasPrefix(part, ":branch"):
			branch = p.parseParameter(part)

		default:
			fmt.Println("Unsupported parameter: " + part)
		}
	}

	switch {
	case moduleType == "git":
		return &GitModule{
			name:        name,
			repoUrl:     repoUrl,
			installPath: installPath,
			processed:   p.moduleProcessedCallback,
			want: struct {
				ref    string
				tag    string
				branch string
			}{
				ref,
				tag,
				branch,
			},
			targetFolder: targetFolder,
			cacheFolder:  ""}, nil

	case moduleType == "github_tarball":
		return &GithubTarballModule{
			name:         name,
			repoName:     repoName,
			version:      version,
			targetFolder: targetFolder,
			processed:    p.moduleProcessedCallback,
			cacheFolder:  "",
		}, nil

	default:
		return &ForgeModule{name: name, version: version, processed: p.moduleProcessedCallback}, nil
	}
}

func (p *PuppetFile) parsePuppetFile(s *bufio.Scanner) ([]PuppetModule, map[string]string) {
	opts := make(map[string]string)
	modules := make([]PuppetModule, 0, 5)

	for block := ""; s.Scan(); {
		line := s.Text()
		line = strings.Split(s.Text(), "#")[0] // Remove comments

		if len(line) == 0 {
			continue
		}

		block += line

		optionValue := func(block string) string {
			return strings.FieldsFunc(block, func(r rune) bool {
				return r == '\'' || r == '"'
			})[1]
		}

		if !strings.HasSuffix(line, ",") { // Full Block
			switch {
			case strings.HasPrefix(block, "forge"):
				opts["forge"] = optionValue(block)

			case strings.HasPrefix(block, "exclusion"):
				opts[""] = optionValue(block)

			case strings.HasPrefix(block, "moduledir"):
				opts["moduledir"] = optionValue(block)

			case strings.HasPrefix(block, "mod"):
				module, _ := p.parseModule(block)
				modules = append(modules, module)
			}
			block = ""
		}
	}

	return modules, opts
}

func (p *PuppetFile) Process(modules chan<- PuppetModule, done func()) error {
	parsedModules, opts := p.parsePuppetFile(bufio.NewScanner(p.File))
	modulePath, ok := opts["moduledir"]
	if !ok {
		modulePath = "modules"
	}

	for _, module := range parsedModules {
		splitPath := strings.Split(module.Name(), "/")
		folderName := splitPath[len(splitPath)-1]
		module.SetTargetFolder(path.Join(modulePath, folderName))

		p.wg.Add(1)
		modules <- module
	}

	go func() {
		p.wg.Wait()
		done()
	}()

	return nil
}
<|endoftext|>