<commit_before>// +build ignore

package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"runtime"
	"strings"
)

var (
	verbose    bool
	keepGopath bool
	runTests   bool
	enableCGO  bool
)

var config = struct {
	Name      string
	Namespace string
	Main      string
	Tests     []string
}{
	Name:      "rest-server",                             // name of the program executable and directory
	Namespace: "github.com/restic/rest-server",           // subdir of GOPATH, e.g. "github.com/foo/bar"
	Main:      "github.com/restic/rest-server",           // package name for the main package
	Tests:     []string{"github.com/restic/rest-server"}, // tests to run
}

// specialDir returns true if the file begins with a special character ('.' or '_').
func specialDir(name string) bool {
	if name == "." {
		return false
	}

	base := filepath.Base(name)
	if base[0] == '_' || base[0] == '.' {
		return true
	}

	return false
}

// excludePath returns true if the file should not be copied to the new GOPATH.
func excludePath(name string) bool {
	ext := path.Ext(name)
	if ext == ".go" || ext == ".s" || ext == ".h" {
		return false
	}

	parentDir := filepath.Base(filepath.Dir(name))
	if parentDir == "testdata" {
		return false
	}

	return true
}

// updateGopath builds a valid GOPATH at dst, with all Go files in src/ copied to dst/prefix/, so calling
//
//   updateGopath("/tmp/gopath", "/home/u/rest-server", "github.com/restic/rest-server")
//
// with "/home/u/restic" containing the file "foo.go" yields the following tree at "/tmp/gopath":
//
//   /tmp/gopath
//   └── src
//       └── github.com
//           └── restic
//               └── rest-server
//                   └── foo.go
func updateGopath(dst, src, prefix string) error {
	return filepath.Walk(src, func(name string, fi os.FileInfo, err error) error {
		if specialDir(name) {
			if fi.IsDir() {
				return filepath.SkipDir
			}

			return nil
		}

		if fi.IsDir() {
			return nil
		}

		if excludePath(name) {
			return nil
		}

		intermediatePath, err := filepath.Rel(src, name)
		if err != nil {
			return err
		}

		fileSrc := filepath.Join(src, intermediatePath)
		fileDst := filepath.Join(dst, "src", prefix, intermediatePath)

		return copyFile(fileDst, fileSrc)
	})
}

func directoryExists(dirname string) bool {
	stat, err := os.Stat(dirname)
	if err != nil && os.IsNotExist(err) {
		return false
	}

	return stat.IsDir()
}

// copyFile creates dst from src, preserving file attributes and timestamps.
func copyFile(dst, src string) error {
	fi, err := os.Stat(src)
	if err != nil {
		return err
	}

	fsrc, err := os.Open(src)
	if err != nil {
		return err
	}
	defer fsrc.Close()

	if err = os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
		fmt.Printf("MkdirAll(%v)\n", filepath.Dir(dst))
		return err
	}

	fdst, err := os.Create(dst)
	if err != nil {
		return err
	}
	defer fdst.Close()

	_, err = io.Copy(fdst, fsrc)
	if err == nil {
		err = os.Chmod(dst, fi.Mode())
	}
	if err == nil {
		err = os.Chtimes(dst, fi.ModTime(), fi.ModTime())
	}

	return err
}

func showUsage(output io.Writer) {
	fmt.Fprintf(output, "USAGE: go run build.go OPTIONS\n")
	fmt.Fprintf(output, "\n")
	fmt.Fprintf(output, "OPTIONS:\n")
	fmt.Fprintf(output, "  -v     --verbose       output more messages\n")
	fmt.Fprintf(output, "  -t     --tags          specify additional build tags\n")
	fmt.Fprintf(output, "  -k     --keep-gopath   do not remove the GOPATH after build\n")
	fmt.Fprintf(output, "  -T     --test          run tests\n")
	fmt.Fprintf(output, "  -o     --output        set output file name\n")
	fmt.Fprintf(output, "         --enable-cgo    use CGO to link against libc\n")
	fmt.Fprintf(output, "         --goos value    set GOOS for cross-compilation\n")
	fmt.Fprintf(output, "         --goarch value  set GOARCH for cross-compilation\n")
}

func verbosePrintf(message string, args ...interface{}) {
	if !verbose {
		return
	}

	fmt.Printf("build: "+message, args...)
}

// cleanEnv returns a clean environment with GOPATH and GOBIN removed (if present).
func cleanEnv() (env []string) {
	for _, v := range os.Environ() {
		if strings.HasPrefix(v, "GOPATH=") || strings.HasPrefix(v, "GOBIN=") {
			continue
		}

		env = append(env, v)
	}

	return env
}

// build runs "go build args..." with GOPATH set to gopath.
func build(cwd, goos, goarch, gopath string, args ...string) error {
	a := []string{"build"}
	a = append(a, "-asmflags", fmt.Sprintf("-trimpath=%s", gopath))
	a = append(a, "-gcflags", fmt.Sprintf("-trimpath=%s", gopath))
	a = append(a, args...)
	cmd := exec.Command("go", a...)
	cmd.Env = append(cleanEnv(), "GOPATH="+gopath, "GOARCH="+goarch, "GOOS="+goos)
	if !enableCGO {
		cmd.Env = append(cmd.Env, "CGO_ENABLED=0")
	}

	cmd.Dir = cwd
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	verbosePrintf("go %s\n", args)

	return cmd.Run()
}

// test runs "go test args..." with GOPATH set to gopath.
func test(cwd, gopath string, args ...string) error {
	args = append([]string{"test"}, args...)
	cmd := exec.Command("go", args...)
	cmd.Env = append(cleanEnv(), "GOPATH="+gopath)
	cmd.Dir = cwd
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	verbosePrintf("go %s\n", args)

	return cmd.Run()
}

// getVersion returns the version string from the file VERSION in the current directory.
func getVersionFromFile() string {
	buf, err := ioutil.ReadFile("VERSION")
	if err != nil {
		verbosePrintf("error reading file VERSION: %v\n", err)
		return ""
	}

	return strings.TrimSpace(string(buf))
}

// getVersion returns a version string which is a combination of the contents of the file VERSION in the current
// directory and the version from git (if available).
func getVersion() string {
	versionFile := getVersionFromFile()
	versionGit := getVersionFromGit()

	verbosePrintf("version from file 'VERSION' is %q, version from git %q\n",
		versionFile, versionGit)

	switch {
	case versionFile == "":
		return versionGit
	case versionGit == "":
		return versionFile
	}

	return fmt.Sprintf("%s (%s)", versionFile, versionGit)
}

// getVersionFromGit returns a version string that identifies the currently checked out git commit.
func getVersionFromGit() string {
	cmd := exec.Command("git", "describe",
		"--long", "--tags", "--dirty", "--always")
	out, err := cmd.Output()
	if err != nil {
		verbosePrintf("git describe returned error: %v\n", err)
		return ""
	}

	version := strings.TrimSpace(string(out))
	verbosePrintf("git version is %s\n", version)
	return version
}

// Constants represents a set of constants that are set in the final binary to the given value via compiler flags.
type Constants map[string]string

// LDFlags returns the string that can be passed to go build's `-ldflags`.
func (cs Constants) LDFlags() string {
	l := make([]string, 0, len(cs))

	for k, v := range cs {
		l = append(l, fmt.Sprintf(`-X "%s=%s"`, k, v))
	}

	return strings.Join(l, " ")
}

func main() {
	log.SetFlags(0)

	ver := runtime.Version()
	if strings.HasPrefix(ver, "go1") && ver < "go1.7" {
		log.Fatalf("Go version %s detected, rest-server requires at least Go 1.7\n", ver)
	}

	buildTags := []string{}

	skipNext := false
	params := os.Args[1:]

	targetGOOS := runtime.GOOS
	targetGOARCH := runtime.GOARCH

	var outputFilename string

	for i, arg := range params {
		if skipNext {
			skipNext = false
			continue
		}

		switch arg {
		case "-v", "--verbose":
			verbose = true
		case "-k", "--keep-gopath":
			keepGopath = true
		case "-t", "-tags", "--tags":
			if i+1 >= len(params) {
				log.Fatal("-t given but no tag specified")
			}
			skipNext = true
			buildTags = strings.Split(params[i+1], " ")
		case "-o", "--output":
			skipNext = true
			outputFilename = params[i+1]
		case "-T", "--test":
			runTests = true
		case "--enable-cgo":
			enableCGO = true
		case "--goos":
			skipNext = true
			targetGOOS = params[i+1]
		case "--goarch":
			skipNext = true
			targetGOARCH = params[i+1]
		case "-h":
			showUsage(os.Stdout)
			return
		default:
			log.Printf("Error: unknown option %q\n\n", arg)
			showUsage(os.Stderr)
			os.Exit(1)
		}
	}

	if len(buildTags) == 0 {
		verbosePrintf("adding build-tag release\n")
		buildTags = []string{"release"}
	}

	for i := range buildTags {
		buildTags[i] = strings.TrimSpace(buildTags[i])
	}

	verbosePrintf("build tags: %s\n", buildTags)

	root, err := os.Getwd()
	if err != nil {
		log.Fatalf("Getwd(): %v\n", err)
	}

	gopath, err := ioutil.TempDir("", fmt.Sprintf("%v-build-", config.Name))
	if err != nil {
		log.Fatalf("TempDir(): %v\n", err)
	}

	verbosePrintf("create GOPATH at %v\n", gopath)
	if err = updateGopath(gopath, root, config.Namespace); err != nil {
		log.Fatalf("copying files from %v/src to %v/src failed: %v\n", root, gopath, err)
	}

	vendor := filepath.Join(root, "vendor")
	if directoryExists(vendor) {
		if err = updateGopath(gopath, vendor, ""); err != nil {
			log.Fatalf("copying files from %v to %v failed: %v\n", root, gopath, err)
		}
	}

	defer func() {
		if !keepGopath {
			verbosePrintf("remove %v\n", gopath)
			if err = os.RemoveAll(gopath); err != nil {
				log.Fatalf("remove GOPATH at %s failed: %v\n", gopath, err)
			}
		} else {
			verbosePrintf("leaving temporary GOPATH at %v\n", gopath)
		}
	}()

	if outputFilename == "" {
		outputFilename = config.Name
		if targetGOOS == "windows" {
			outputFilename += ".exe"
		}
	}

	cwd, err := os.Getwd()
	if err != nil {
		log.Fatalf("Getwd() returned %v\n", err)
	}
	output := filepath.Join(cwd, outputFilename)

	version := getVersion()
	constants := Constants{}
	if version != "" {
		constants["main.version"] = version
	}
	ldflags := "-s -w " + constants.LDFlags()
	verbosePrintf("ldflags: %s\n", ldflags)

	args := []string{
		"-tags", strings.Join(buildTags, " "),
		"-ldflags", ldflags,
		"-o", output, config.Main,
	}

	err = build(filepath.Join(gopath, "src"), targetGOOS, targetGOARCH, gopath, args...)
	if err != nil {
		log.Fatalf("build failed: %v\n", err)
	}

	if runTests {
		verbosePrintf("running tests\n")

		err = test(cwd, gopath, config.Tests...)
		if err != nil {
			log.Fatalf("running tests failed: %v\n", err)
		}
	}
}
<commit_msg>build.go: Fix path for new cmd/ subdir<commit_after>// +build ignore

package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"runtime"
	"strings"
)

var (
	verbose    bool
	keepGopath bool
	runTests   bool
	enableCGO  bool
)

var config = struct {
	Name      string
	Namespace string
	Main      string
	Tests     []string
}{
	Name:      "rest-server",                                   // name of the program executable and directory
	Namespace: "github.com/restic/rest-server",                 // subdir of GOPATH, e.g. "github.com/foo/bar"
	Main:      "github.com/restic/rest-server/cmd/rest-server", // package name for the main package
	Tests:     []string{"github.com/restic/rest-server"},       // tests to run
}

// specialDir returns true if the file begins with a special character ('.' or '_').
func specialDir(name string) bool {
	if name == "." {
		return false
	}

	base := filepath.Base(name)
	if base[0] == '_' || base[0] == '.' {
		return true
	}

	return false
}

// excludePath returns true if the file should not be copied to the new GOPATH.
func excludePath(name string) bool {
	ext := path.Ext(name)
	if ext == ".go" || ext == ".s" || ext == ".h" {
		return false
	}

	parentDir := filepath.Base(filepath.Dir(name))
	if parentDir == "testdata" {
		return false
	}

	return true
}

// updateGopath builds a valid GOPATH at dst, with all Go files in src/ copied to dst/prefix/, so calling
//
//   updateGopath("/tmp/gopath", "/home/u/rest-server", "github.com/restic/rest-server")
//
// with "/home/u/restic" containing the file "foo.go" yields the following tree at "/tmp/gopath":
//
//   /tmp/gopath
//   └── src
//       └── github.com
//           └── restic
//               └── rest-server
//                   └── foo.go
func updateGopath(dst, src, prefix string) error {
	return filepath.Walk(src, func(name string, fi os.FileInfo, err error) error {
		if specialDir(name) {
			if fi.IsDir() {
				return filepath.SkipDir
			}

			return nil
		}

		if fi.IsDir() {
			return nil
		}

		if excludePath(name) {
			return nil
		}

		intermediatePath, err := filepath.Rel(src, name)
		if err != nil {
			return err
		}

		fileSrc := filepath.Join(src, intermediatePath)
		fileDst := filepath.Join(dst, "src", prefix, intermediatePath)

		return copyFile(fileDst, fileSrc)
	})
}

func directoryExists(dirname string) bool {
	stat, err := os.Stat(dirname)
	if err != nil && os.IsNotExist(err) {
		return false
	}

	return stat.IsDir()
}

// copyFile creates dst from src, preserving file attributes and timestamps.
func copyFile(dst, src string) error {
	fi, err := os.Stat(src)
	if err != nil {
		return err
	}

	fsrc, err := os.Open(src)
	if err != nil {
		return err
	}
	defer fsrc.Close()

	if err = os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
		fmt.Printf("MkdirAll(%v)\n", filepath.Dir(dst))
		return err
	}

	fdst, err := os.Create(dst)
	if err != nil {
		return err
	}
	defer fdst.Close()

	_, err = io.Copy(fdst, fsrc)
	if err == nil {
		err = os.Chmod(dst, fi.Mode())
	}
	if err == nil {
		err = os.Chtimes(dst, fi.ModTime(), fi.ModTime())
	}

	return err
}

func showUsage(output io.Writer) {
	fmt.Fprintf(output, "USAGE: go run build.go OPTIONS\n")
	fmt.Fprintf(output, "\n")
	fmt.Fprintf(output, "OPTIONS:\n")
	fmt.Fprintf(output, "  -v     --verbose       output more messages\n")
	fmt.Fprintf(output, "  -t     --tags          specify additional build tags\n")
	fmt.Fprintf(output, "  -k     --keep-gopath   do not remove the GOPATH after build\n")
	fmt.Fprintf(output, "  -T     --test          run tests\n")
	fmt.Fprintf(output, "  -o     --output        set output file name\n")
	fmt.Fprintf(output, "         --enable-cgo    use CGO to link against libc\n")
	fmt.Fprintf(output, "         --goos value    set GOOS for cross-compilation\n")
	fmt.Fprintf(output, "         --goarch value  set GOARCH for cross-compilation\n")
}

func verbosePrintf(message string, args ...interface{}) {
	if !verbose {
		return
	}

	fmt.Printf("build: "+message, args...)
}

// cleanEnv returns a clean environment with GOPATH and GOBIN removed (if present).
func cleanEnv() (env []string) {
	for _, v := range os.Environ() {
		if strings.HasPrefix(v, "GOPATH=") || strings.HasPrefix(v, "GOBIN=") {
			continue
		}

		env = append(env, v)
	}

	return env
}

// build runs "go build args..." with GOPATH set to gopath.
func build(cwd, goos, goarch, gopath string, args ...string) error {
	a := []string{"build"}
	a = append(a, "-asmflags", fmt.Sprintf("-trimpath=%s", gopath))
	a = append(a, "-gcflags", fmt.Sprintf("-trimpath=%s", gopath))
	a = append(a, args...)
	cmd := exec.Command("go", a...)
	cmd.Env = append(cleanEnv(), "GOPATH="+gopath, "GOARCH="+goarch, "GOOS="+goos)
	if !enableCGO {
		cmd.Env = append(cmd.Env, "CGO_ENABLED=0")
	}

	cmd.Dir = cwd
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	verbosePrintf("go %s\n", args)

	return cmd.Run()
}

// test runs "go test args..." with GOPATH set to gopath.
func test(cwd, gopath string, args ...string) error {
	args = append([]string{"test"}, args...)
	cmd := exec.Command("go", args...)
	cmd.Env = append(cleanEnv(), "GOPATH="+gopath)
	cmd.Dir = cwd
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	verbosePrintf("go %s\n", args)

	return cmd.Run()
}

// getVersion returns the version string from the file VERSION in the current directory.
func getVersionFromFile() string {
	buf, err := ioutil.ReadFile("VERSION")
	if err != nil {
		verbosePrintf("error reading file VERSION: %v\n", err)
		return ""
	}

	return strings.TrimSpace(string(buf))
}

// getVersion returns a version string which is a combination of the contents of the file VERSION in the current
// directory and the version from git (if available).
func getVersion() string {
	versionFile := getVersionFromFile()
	versionGit := getVersionFromGit()

	verbosePrintf("version from file 'VERSION' is %q, version from git %q\n",
		versionFile, versionGit)

	switch {
	case versionFile == "":
		return versionGit
	case versionGit == "":
		return versionFile
	}

	return fmt.Sprintf("%s (%s)", versionFile, versionGit)
}

// getVersionFromGit returns a version string that identifies the currently checked out git commit.
func getVersionFromGit() string {
	cmd := exec.Command("git", "describe",
		"--long", "--tags", "--dirty", "--always")
	out, err := cmd.Output()
	if err != nil {
		verbosePrintf("git describe returned error: %v\n", err)
		return ""
	}

	version := strings.TrimSpace(string(out))
	verbosePrintf("git version is %s\n", version)
	return version
}

// Constants represents a set of constants that are set in the final binary to the given value via compiler flags.
type Constants map[string]string

// LDFlags returns the string that can be passed to go build's `-ldflags`.
func (cs Constants) LDFlags() string {
	l := make([]string, 0, len(cs))

	for k, v := range cs {
		l = append(l, fmt.Sprintf(`-X "%s=%s"`, k, v))
	}

	return strings.Join(l, " ")
}

func main() {
	log.SetFlags(0)

	ver := runtime.Version()
	if strings.HasPrefix(ver, "go1") && ver < "go1.7" {
		log.Fatalf("Go version %s detected, rest-server requires at least Go 1.7\n", ver)
	}

	buildTags := []string{}

	skipNext := false
	params := os.Args[1:]

	targetGOOS := runtime.GOOS
	targetGOARCH := runtime.GOARCH

	var outputFilename string

	for i, arg := range params {
		if skipNext {
			skipNext = false
			continue
		}

		switch arg {
		case "-v", "--verbose":
			verbose = true
		case "-k", "--keep-gopath":
			keepGopath = true
		case "-t", "-tags", "--tags":
			if i+1 >= len(params) {
				log.Fatal("-t given but no tag specified")
			}
			skipNext = true
			buildTags = strings.Split(params[i+1], " ")
		case "-o", "--output":
			skipNext = true
			outputFilename = params[i+1]
		case "-T", "--test":
			runTests = true
		case "--enable-cgo":
			enableCGO = true
		case "--goos":
			skipNext = true
			targetGOOS = params[i+1]
		case "--goarch":
			skipNext = true
			targetGOARCH = params[i+1]
		case "-h":
			showUsage(os.Stdout)
			return
		default:
			log.Printf("Error: unknown option %q\n\n", arg)
			showUsage(os.Stderr)
			os.Exit(1)
		}
	}

	if len(buildTags) == 0 {
		verbosePrintf("adding build-tag release\n")
		buildTags = []string{"release"}
	}

	for i := range buildTags {
		buildTags[i] = strings.TrimSpace(buildTags[i])
	}

	verbosePrintf("build tags: %s\n", buildTags)

	root, err := os.Getwd()
	if err != nil {
		log.Fatalf("Getwd(): %v\n", err)
	}

	gopath, err := ioutil.TempDir("", fmt.Sprintf("%v-build-", config.Name))
	if err != nil {
		log.Fatalf("TempDir(): %v\n", err)
	}

	verbosePrintf("create GOPATH at %v\n", gopath)
	if err = updateGopath(gopath, root, config.Namespace); err != nil {
		log.Fatalf("copying files from %v/src to %v/src failed: %v\n", root, gopath, err)
	}

	vendor := filepath.Join(root, "vendor")
	if directoryExists(vendor) {
		if err = updateGopath(gopath, vendor, ""); err != nil {
			log.Fatalf("copying files from %v to %v failed: %v\n", root, gopath, err)
		}
	}

	defer func() {
		if !keepGopath {
			verbosePrintf("remove %v\n", gopath)
			if err = os.RemoveAll(gopath); err != nil {
				log.Fatalf("remove GOPATH at %s failed: %v\n", gopath, err)
			}
		} else {
			verbosePrintf("leaving temporary GOPATH at %v\n", gopath)
		}
	}()

	if outputFilename == "" {
		outputFilename = config.Name
		if targetGOOS == "windows" {
			outputFilename += ".exe"
		}
	}

	cwd, err := os.Getwd()
	if err != nil {
		log.Fatalf("Getwd() returned %v\n", err)
	}
	output := filepath.Join(cwd, outputFilename)

	version := getVersion()
	constants := Constants{}
	if version != "" {
		constants["main.version"] = version
	}
	ldflags := "-s -w " + constants.LDFlags()
	verbosePrintf("ldflags: %s\n", ldflags)

	args := []string{
		"-tags", strings.Join(buildTags, " "),
		"-ldflags", ldflags,
		"-o", output, config.Main,
	}

	err = build(filepath.Join(gopath, "src"), targetGOOS, targetGOARCH, gopath, args...)
	if err != nil {
		log.Fatalf("build failed: %v\n", err)
	}

	if runTests {
		verbosePrintf("running tests\n")

		err = test(cwd, gopath, config.Tests...)
		if err != nil {
			log.Fatalf("running tests failed: %v\n", err)
		}
	}
}
<|endoftext|>


<commit_before>/* Copyright (c) 2017 Jason Ish
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

package rules

import (
	"github.com/fsnotify/fsnotify"
	"github.com/jasonish/evebox/eve"
	"github.com/jasonish/evebox/log"
	"github.com/jasonish/go-idsrules"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"
)

type RuleMap struct {
	paths   []string
	rules   map[uint64]idsrules.Rule
	watcher *fsnotify.Watcher
	lock    sync.RWMutex
}

func NewRuleMap(paths []string) *RuleMap {

	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		log.Warning("Failed to initialize file watcher, rules won't be automatically reloaded: %v", err)
		watcher = nil
	}

	rulemap := &RuleMap{
		paths:   paths,
		watcher: watcher,
	}

	rulemap.reload()

	go rulemap.watchFiles()

	return rulemap
}

func (r *RuleMap) watchFiles() {

	doReload := false
	lastMod := time.Now()

	timer := time.Tick(1 * time.Second)

	for {
		select {
		case <-r.watcher.Events:
			doReload = true
			lastMod = time.Now()
		case err := <-r.watcher.Errors:
			log.Warning("File watch error: %v", err)
		case <-timer:
			if doReload && time.Now().Sub(lastMod).Seconds() > 1 {
				log.Info("Reloading rules...")
				r.reload()
				doReload = false
			}
		}
	}
}

func (r *RuleMap) reload() {
	rules := make(map[uint64]idsrules.Rule)
	filenames := findRuleFilenames(r.paths)
	for _, filename := range filenames {
		if err := loadRulesFromFile(&rules, filename); err != nil {
			log.Warning("Failed to load rules from %s: %v", filename, err)
		}
		if err := r.watcher.Add(filename); err != nil {
			log.Warning("Failed to add watch for %s: %v", filename, err)
		}
	}
	r.lock.Lock()
	r.rules = rules
	r.lock.Unlock()
	log.Info("Loaded %d rules", len(rules))
}

func (r *RuleMap) FindById(id uint64) *idsrules.Rule {
	r.lock.RLock()
	defer r.lock.RUnlock()
	if r == nil || r.rules == nil {
		return nil
	}
	if rule, ok := r.rules[id]; ok {
		return &rule
	}
	return nil
}

// Filter implements eve.EveFilter for RuleMap.
func (r *RuleMap) Filter(event eve.EveEvent) {
	ruleId, ok := event.GetAlertSignatureId()
	if ok {
		rule := r.FindById(ruleId)
		if rule != nil {
			event["rule"] = rule.Raw
		}
	}
}

func loadRulesFromFile(ruleMap *map[uint64]idsrules.Rule, filename string) error {
	file, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer file.Close()

	ruleReader := idsrules.NewRuleReader(file)

	count := 0

	for {
		rule, err := ruleReader.Next()
		if err != nil {
			if err == io.EOF {
				break
			}
			log.Warning("Rule parser error: %v", err)
			continue
		}

		if _, ok := (*ruleMap)[rule.Sid]; ok {
			log.Warning("A rule with ID %d already exists.", rule.Sid)
		} else {
			count++
			(*ruleMap)[rule.Sid] = rule
		}

	}

	log.Debug("Loaded %d rules from %s", count, filename)

	return nil
}

// findRuleFilenames returns a list of full file names from the
// paths/patterns provided.
func findRuleFilenames(paths []string) []string {
	filenames := make([]string, 0)

	for _, path := range paths {
		fileInfo, err := os.Stat(path)
		if err != nil {
			// Load as glob.
			matches, err := filepath.Glob(path)
			if err != nil {
				log.Warning("No matches for %s: %v", path, err)
				continue
			}
			for _, m := range matches {
				filenames = append(filenames, m)
			}
		} else if fileInfo.IsDir() {
			infos, err := ioutil.ReadDir(path)
			if err != nil {
				log.Warning("Failed to read %s: %v", fileInfo.Name(), err)
				continue
			}
			for _, info := range infos {
				if !strings.HasSuffix(info.Name(), ".rules") {
					continue
				}
				fullFilename := filepath.Join(path, info.Name())
				filenames = append(filenames, fullFilename)
			}
		} else {
			filenames = append(filenames, path)
		}
	}

	return filenames
}
<commit_msg>rules: log a warning when glob doesn't match any rules<commit_after>/* Copyright (c) 2017 Jason Ish
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

package rules

import (
	"github.com/fsnotify/fsnotify"
	"github.com/jasonish/evebox/eve"
	"github.com/jasonish/evebox/log"
	"github.com/jasonish/go-idsrules"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"
)

type RuleMap struct {
	paths   []string
	rules   map[uint64]idsrules.Rule
	watcher *fsnotify.Watcher
	lock    sync.RWMutex
}

func NewRuleMap(paths []string) *RuleMap {

	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		log.Warning("Failed to initialize file watcher, rules won't be automatically reloaded: %v", err)
		watcher = nil
	}

	rulemap := &RuleMap{
		paths:   paths,
		watcher: watcher,
	}

	rulemap.reload()

	go rulemap.watchFiles()

	return rulemap
}

func (r *RuleMap) watchFiles() {

	doReload := false
	lastMod := time.Now()

	timer := time.Tick(1 * time.Second)

	for {
		select {
		case <-r.watcher.Events:
			doReload = true
			lastMod = time.Now()
		case err := <-r.watcher.Errors:
			log.Warning("File watch error: %v", err)
		case <-timer:
			if doReload && time.Now().Sub(lastMod).Seconds() > 1 {
				log.Info("Reloading rules...")
				r.reload()
				doReload = false
			}
		}
	}
}

func (r *RuleMap) reload() {
	rules := make(map[uint64]idsrules.Rule)
	filenames := findRuleFilenames(r.paths)
	for _, filename := range filenames {
		if err := loadRulesFromFile(&rules, filename); err != nil {
			log.Warning("Failed to load rules from %s: %v", filename, err)
		}
		if err := r.watcher.Add(filename); err != nil {
			log.Warning("Failed to add watch for %s: %v", filename, err)
		}
	}
	r.lock.Lock()
	r.rules = rules
	r.lock.Unlock()
	log.Info("Loaded %d rules", len(rules))
}

func (r *RuleMap) FindById(id uint64) *idsrules.Rule {
	r.lock.RLock()
	defer r.lock.RUnlock()
	if r == nil || r.rules == nil {
		return nil
	}
	if rule, ok := r.rules[id]; ok {
		return &rule
	}
	return nil
}

// Filter implements eve.EveFilter for RuleMap.
func (r *RuleMap) Filter(event eve.EveEvent) {
	ruleId, ok := event.GetAlertSignatureId()
	if ok {
		rule := r.FindById(ruleId)
		if rule != nil {
			event["rule"] = rule.Raw
		}
	}
}

func loadRulesFromFile(ruleMap *map[uint64]idsrules.Rule, filename string) error {
	file, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer file.Close()

	ruleReader := idsrules.NewRuleReader(file)

	count := 0

	for {
		rule, err := ruleReader.Next()
		if err != nil {
			if err == io.EOF {
				break
			}
			log.Warning("Rule parser error: %v", err)
			continue
		}

		if _, ok := (*ruleMap)[rule.Sid]; ok {
			log.Warning("A rule with ID %d already exists.", rule.Sid)
		} else {
			count++
			(*ruleMap)[rule.Sid] = rule
		}

	}

	log.Debug("Loaded %d rules from %s", count, filename)

	return nil
}

// findRuleFilenames returns a list of full file names from the
// paths/patterns provided.
func findRuleFilenames(paths []string) []string {
	filenames := make([]string, 0)

	for _, path := range paths {
		fileInfo, err := os.Stat(path)
		if err != nil {
			// Load as glob.
			matches, err := filepath.Glob(path)
			if err != nil {
				log.Warning("Failed to load glob %s: %v", path, err)
			} else if len(matches) == 0 {
				log.Warning("No files matched glob %s.", path)
			} else {
				for _, m := range matches {
					filenames = append(filenames, m)
				}
			}
		} else if fileInfo.IsDir() {
			infos, err := ioutil.ReadDir(path)
			if err != nil {
				log.Warning("Failed to read %s: %v", fileInfo.Name(), err)
				continue
			}
			for _, info := range infos {
				if !strings.HasSuffix(info.Name(), ".rules") {
					continue
				}
				fullFilename := filepath.Join(path, info.Name())
				filenames = append(filenames, fullFilename)
			}
		} else {
			filenames = append(filenames, path)
		}
	}

	return filenames
}
<|endoftext|>


<commit_before>package runner

import (
	"encoding/json"
	"fmt"
	"os"
	"os/exec"
	"os/signal"
	"strings"
	"sync"
	"time"

	"github.com/pkg/errors"
	"github.com/urfave/cli"
	"github.com/yext/edward/commandline"
	"github.com/yext/edward/config"
	"github.com/yext/edward/services"
)

var Command = cli.Command{
	Name:   "run",
	Hidden: true,
	Action: run,
	Flags: []cli.Flag{
		cli.BoolFlag{
			Name:        "no-watch",
			Usage:       "Disable autorestart",
			Destination: &(noWatch),
		},
	},
}

var runningService *services.ServiceConfig
var runningCommand *exec.Cmd
var logFile *os.File
var messageLog *RunnerLog

var commandWait sync.WaitGroup

var noWatch bool

type Logger interface {
	Printf(format string, a ...interface{})
}

func run(c *cli.Context) error {
	args := c.Args()
	if len(args) < 1 {
		return errors.New("a service name is required")
	}

	var ok bool
	runningService, ok = config.GetServiceMap()[args[0]]
	if !ok {
		return errors.New("service not found")
	}

	logLocation := runningService.GetRunLog()
	os.Remove(logLocation)

	var err error
	logFile, err = os.Create(logLocation)
	if err != nil {
		return errors.WithStack(err)
	}

	messageLog = &RunnerLog{
		file:   logFile,
		name:   runningService.Name,
		stream: "messages",
	}

	defer messageLog.Printf("Service stopped\n")

	commandWait.Add(1)
	err = startService()
	if err != nil {
		return errors.WithStack(err)
	}

	if !noWatch {
		closeWatchers, err := BeginWatch(runningService, restartService, messageLog)
		if err != nil {
			messageLog.Printf("Could not enable auto-restart: %v\n", err)
		} else {
			messageLog.Printf("Auto-restart enabled. This service will restart when files in its watch directories are edited.\nThis can be disabled using the --no-watch flag.\n")
			defer closeWatchers()
		}
	}

	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, os.Interrupt)
	go func() {
		for _ = range signalChan {
			_ = stopService()
		}
	}()

	commandWait.Wait()
	return nil
}

func restartService() error {
	messageLog.Printf("Restarting service\n")

	// Increment the counter to prevent exiting unexpectedly
	commandWait.Add(1)

	err := stopService()
	if err != nil {
		return errors.WithStack(err)
	}
	err = startService()
	if err != nil {
		return errors.WithStack(err)
	}
	return nil
}

func stopService() error {
	command, err := runningService.GetCommand()
	if err != nil {
		messageLog.Printf("Could not get service command: %v\n", err)
	}

	var scriptErr error
	var scriptOutput []byte
	if runningService.Commands.Stop != "" {
		messageLog.Printf("Running stop script for %v.\n", runningService.Name)
		scriptOutput, scriptErr = command.RunStopScript()
		if scriptErr != nil {
			messageLog.Printf("%v\n", string(scriptOutput))
			messageLog.Printf("Stop script failed: %v\n", scriptErr)
		}
		if waitForCompletionWithTimeout(1 * time.Second) {
			return nil
		}
		messageLog.Printf("Stop script did not effectively stop service, sending interrupt\n")
	}

	err = runningCommand.Process.Signal(os.Interrupt)
	if err != nil {
		return errors.WithStack(err)
	}

	if waitForCompletionWithTimeout(1 * time.Second) {
		return nil
	}
	messageLog.Printf("Stop script did not effectively stop service, sending kill\n")

	err = runningCommand.Process.Kill()
	if err != nil {
		return errors.WithStack(err)
	}

	if waitForCompletionWithTimeout(1 * time.Second) {
		return nil
	}
	return errors.New("kill did not stop service")
}

func waitForCompletionWithTimeout(timeout time.Duration) bool {
	var completed = make(chan struct{})
	defer close(completed)
	timer := time.NewTimer(timeout)
	defer timer.Stop()

	go func() {
		runningCommand.Wait()
		completed <- struct{}{}
	}()

	select {
	case <-completed:
		return true
	case <-timer.C:
		return false
	}
}

func startService() error {
	messageLog.Printf("Service starting\n")
	command, cmdArgs, err := commandline.ParseCommand(os.ExpandEnv(runningService.Commands.Launch))
	if err != nil {
		return errors.WithStack(err)
	}

	standardLog := &RunnerLog{
		file:   logFile,
		name:   runningService.Name,
		stream: "stdout",
	}
	errorLog := &RunnerLog{
		file:   logFile,
		name:   runningService.Name,
		stream: "stderr",
	}

	cmd := exec.Command(command, cmdArgs...)
	if runningService.Path != nil {
		cmd.Dir = os.ExpandEnv(*runningService.Path)
	}
	cmd.Stdout = standardLog
	cmd.Stderr = errorLog

	runningCommand = cmd

	go func() {
		cmd.Run()
		commandWait.Done()
	}()
	return nil
}

// RunnerLog provides the io.Writer interface to publish service logs to file
type RunnerLog struct {
	file   *os.File
	name   string
	stream string
}

func (r *RunnerLog) Printf(format string, a ...interface{}) {
	fmt.Printf(format, a...)
	msg := fmt.Sprintf(format, a...)
	r.Write([]byte(msg))
}

func (r *RunnerLog) Write(p []byte) (int, error) {
	lineData := LogLine{
		Name:    r.name,
		Time:    time.Now(),
		Stream:  r.stream,
		Message: strings.TrimSpace(string(p)),
	}

	jsonContent, err := json.Marshal(lineData)
	if err != nil {
		return 0, errors.Wrap(err, "could not prepare log line")
	}

	line := fmt.Sprintln(string(jsonContent))
	count, err := r.file.Write([]byte(line))
	if err != nil {
		fmt.Println("Error")
		return count, errors.Wrap(err, "could not write log line")
	}
	return len(p), nil
}
<commit_msg>Refactor runner to embed everything into a single struct.<commit_after>package runner

import (
	"encoding/json"
	"fmt"
	"os"
	"os/exec"
	"os/signal"
	"strings"
	"sync"
	"time"

	"github.com/pkg/errors"
	"github.com/urfave/cli"
	"github.com/yext/edward/commandline"
	"github.com/yext/edward/config"
	"github.com/yext/edward/services"
)

var runnerInstance = &Runner{}

var Command = cli.Command{
	Name:   "run",
	Hidden: true,
	Action: runnerInstance.run,
	Flags: []cli.Flag{
		cli.BoolFlag{
			Name:        "no-watch",
			Usage:       "Disable autorestart",
			Destination: &(runnerInstance.noWatch),
		},
	},
}

type Runner struct {
	service    *services.ServiceConfig
	command    *exec.Cmd
	logFile    *os.File
	messageLog *RunnerLog

	commandWait sync.WaitGroup
	noWatch     bool
}

type Logger interface {
	Printf(format string, a ...interface{})
}

func (r *Runner) run(c *cli.Context) error {
	args := c.Args()
	if len(args) < 1 {
		return errors.New("a service name is required")
	}

	var ok bool
	r.service, ok = config.GetServiceMap()[args[0]]
	if !ok {
		return errors.New("service not found")
	}

	logLocation := r.service.GetRunLog()
	os.Remove(logLocation)

	var err error
	r.logFile, err = os.Create(logLocation)
	if err != nil {
		return errors.WithStack(err)
	}

	r.messageLog = &RunnerLog{
		file:   r.logFile,
		name:   r.service.Name,
		stream: "messages",
	}
	defer r.messageLog.Printf("Service stopped\n")

	r.commandWait.Add(1)
	err = r.startService()
	if err != nil {
		return errors.WithStack(err)
	}

	if !r.noWatch {
		closeWatchers, err := BeginWatch(r.service, r.restartService, r.messageLog)
		if err != nil {
			r.messageLog.Printf("Could not enable auto-restart: %v\n", err)
		} else {
			r.messageLog.Printf("Auto-restart enabled. This service will restart when files in its watch directories are edited.\nThis can be disabled using the --no-watch flag.\n")
			defer closeWatchers()
		}
	}

	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, os.Interrupt)
	go func() {
		for _ = range signalChan {
			r.messageLog.Printf("Received interrupt\n")
			err := r.stopService()
			if err != nil {
				r.messageLog.Printf("Could not stop service: %v", err)
			}
		}
	}()

	r.commandWait.Wait()
	return nil
}

func (r *Runner) restartService() error {
	r.messageLog.Printf("Restarting service\n")

	// Increment the counter to prevent exiting unexpectedly
	r.commandWait.Add(1)

	err := r.stopService()
	if err != nil {
		return errors.WithStack(err)
	}
	err = r.startService()
	if err != nil {
		return errors.WithStack(err)
	}
	return nil
}

func (r *Runner) stopService() error {
	r.messageLog.Printf("stopService")
	command, err := r.service.GetCommand()
	if err != nil {
		r.messageLog.Printf("Could not get service command: %v\n", err)
	}

	var scriptErr error
	var scriptOutput []byte
	if r.service.Commands.Stop != "" {
		r.messageLog.Printf("Running stop script for %v.\n", r.service.Name)
		scriptOutput, scriptErr = command.RunStopScript()
		if scriptErr != nil {
			r.messageLog.Printf("%v\n", string(scriptOutput))
			r.messageLog.Printf("Stop script failed: %v\n", scriptErr)
		}
		if r.waitForCompletionWithTimeout(1 * time.Second) {
			return nil
		}
		r.messageLog.Printf("Stop script did not effectively stop service, sending interrupt\n")
	}

	err = r.command.Process.Signal(os.Interrupt)
	if err != nil {
		return errors.WithStack(err)
	}

	if r.waitForCompletionWithTimeout(1 * time.Second) {
		r.messageLog.Printf("Interrupt succeeded")
		return nil
	}
	r.messageLog.Printf("Stop script did not effectively stop service, sending kill\n")

	err = r.command.Process.Kill()
	if err != nil {
		return errors.WithStack(err)
	}

	if r.waitForCompletionWithTimeout(1 * time.Second) {
		return nil
	}
	return errors.New("kill did not stop service")
}

func (r *Runner) waitForCompletionWithTimeout(timeout time.Duration) bool {
	var completed = make(chan struct{})
	defer close(completed)
	timer := time.NewTimer(timeout)
	defer timer.Stop()

	go func() {
		r.command.Wait()
		r.messageLog.Printf("Command completed")
		completed <- struct{}{}
	}()

	select {
	case <-completed:
		return true
	case <-timer.C:
		return false
	}
}

func (r *Runner) startService() error {
	r.messageLog.Printf("Service starting\n")
	command, cmdArgs, err := commandline.ParseCommand(os.ExpandEnv(r.service.Commands.Launch))
	if err != nil {
		return errors.WithStack(err)
	}

	standardLog := &RunnerLog{
		file:   r.logFile,
		name:   r.service.Name,
		stream: "stdout",
	}
	errorLog := &RunnerLog{
		file:   r.logFile,
		name:   r.service.Name,
		stream: "stderr",
	}

	cmd := exec.Command(command, cmdArgs...)
	if r.service.Path != nil {
		cmd.Dir = os.ExpandEnv(*r.service.Path)
	}
	cmd.Stdout = standardLog
	cmd.Stderr = errorLog

	r.command = cmd

	go func() {
		cmd.Run()
		r.messageLog.Printf("command run finished, sending done")
		r.commandWait.Done()
	}()
	return nil
}

// RunnerLog provides the io.Writer interface to publish service logs to file
type RunnerLog struct {
	file   *os.File
	name   string
	stream string
}

func (r *RunnerLog) Printf(format string, a ...interface{}) {
	msg := fmt.Sprintf(format, a...)
	r.Write([]byte(msg))
}

func (r *RunnerLog) Write(p []byte) (int, error) {
	fmt.Print(string(p))
	lineData := LogLine{
		Name:    r.name,
		Time:    time.Now(),
		Stream:  r.stream,
		Message: strings.TrimSpace(string(p)),
	}

	jsonContent, err := json.Marshal(lineData)
	if err != nil {
		return 0, errors.Wrap(err, "could not prepare log line")
	}

	line := fmt.Sprintln(string(jsonContent))
	count, err := r.file.Write([]byte(line))
	if err != nil {
		fmt.Println("Error")
		return count, errors.Wrap(err, "could not write log line")
	}
	return len(p), nil
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/go-gl/cl/v1.2/cl"
	"log"
	"math"
	"math/rand"
	"unsafe"
)

const (
	DATA_SIZE = 1024
)

var KernelSource string = `
__kernel void square(
   __global float* input,
   __global float* output,
   const unsigned int count)
{
   int i = get_global_id(0);
   if(i < count)
	   output[i] = input[i] * input[i];
}` + "\x00"

func main() {

	data := make([]float32, DATA_SIZE)
	for x := 0; x < len(data); x++ {
		data[x] = rand.Float32()*99 + 1
	}

	//Get Device
	var device cl.DeviceId
	err := cl.GetDeviceIDs(nil, cl.DEVICE_TYPE_GPU, 1, &device, nil)
	if err != cl.SUCCESS {
		log.Fatal("Failed to create device group")
	}
	var errptr *int32

	//Create Computer Context
	context := cl.CreateContext(nil, 1, &device, nil, nil, errptr)
	if errptr != nil && cl.ErrorCode(*errptr) != cl.SUCCESS {
		log.Fatal("couldnt create context")
	}

	//Create Command Queue
	cq := cl.CreateCommandQueue(*context, device, 0, errptr)
	if errptr != nil && cl.ErrorCode(*errptr) != cl.SUCCESS {
		log.Fatal("couldnt create command queue")
	}

	//Create program
	srcptr := cl.Str(KernelSource)
	program := cl.CreateProgramWithSource(*context, 1, &srcptr, nil, errptr)
	if errptr != nil && cl.ErrorCode(*errptr) != cl.SUCCESS {
		log.Fatal("couldnt create program")
	}

	err = cl.BuildProgram(program, 1, &device, nil, nil, nil)
	if err != cl.SUCCESS {
		var length uint64
		buffer := make([]byte, DATA_SIZE)

		log.Println("Error: Failed to build program executable!")
		cl.GetProgramBuildInfo(program, device, cl.PROGRAM_BUILD_LOG, uint64(len(buffer)), unsafe.Pointer(&buffer[0]), &length)
		log.Fatal(string(buffer[0:length]))
	}

	//Get Kernel
	kernel := cl.CreateKernel(program, cl.Str("square"+"\x00"), errptr)
	if errptr != nil && cl.ErrorCode(*errptr) != cl.SUCCESS {
		log.Fatal("couldnt create compute kernel")
	}

	//Create buffers
	input := cl.CreateBuffer(*context, cl.MEM_READ_ONLY, 4*DATA_SIZE, nil, errptr)
	if errptr != nil && cl.ErrorCode(*errptr) != cl.SUCCESS {
		log.Fatal("couldnt create input buffer")
	}

	output := cl.CreateBuffer(*context, cl.MEM_WRITE_ONLY, 4*DATA_SIZE, nil, errptr)
	if errptr != nil && cl.ErrorCode(*errptr) != cl.SUCCESS {
		log.Fatal("couldnt create output buffer")
	}

	//Write data
	err = cl.EnqueueWriteBuffer(cq, input, cl.TRUE, 0, 4*DATA_SIZE, unsafe.Pointer(&data[0]), 0, nil, nil)
	if err != cl.SUCCESS {
		log.Fatal("Failed to write to source array")
	}

	//Set kernel args
	count := uint32(DATA_SIZE)
	err = cl.SetKernelArg(kernel, 0, 8, unsafe.Pointer(&input))
	if err != cl.SUCCESS {
		log.Fatal("Failed to write kernel arg 0")
	}
	err = cl.SetKernelArg(kernel, 1, 8, unsafe.Pointer(&output))
	if err != cl.SUCCESS {
		log.Fatal("Failed to write kernel arg 1")
	}
	err = cl.SetKernelArg(kernel, 2, 4, unsafe.Pointer(&count))
	if err != cl.SUCCESS {
		log.Fatal("Failed to write kernel arg 2")
	}

	local := uint64(0)
	err = cl.GetKernelWorkGroupInfo(kernel, device, cl.KERNEL_WORK_GROUP_SIZE, 8, unsafe.Pointer(&local), nil)
	if err != cl.SUCCESS {
		log.Fatal("Failed to get kernel work group info")
	}

	global := local
	err = cl.EnqueueNDRangeKernel(cq, kernel, 1, nil, &global, &local, 0, nil, nil)
	if err != cl.SUCCESS {
		log.Fatal("Failed to execute kernel!")
	}

	cl.Finish(cq)

	results := make([]float32, DATA_SIZE)
	err = cl.EnqueueReadBuffer(cq, output, cl.TRUE, 0, 4*1024, unsafe.Pointer(&results[0]), 0, nil, nil)
	if err != cl.SUCCESS {
		log.Fatal("Failed to read buffer!")
	}

	success := 0
	notzero := 0
	for i, x := range data {
		if math.Abs(float64(x*x-results[i])) < 0.5 {
			success++
		}
		if results[i] > 0 {
			notzero++
		}
		log.Printf("I/O: %f\t%f", x, results[i])
	}

	log.Printf("%d/%d success", success, DATA_SIZE)
	log.Printf("values not zero: %d", notzero)

}
<commit_msg>add defer Releases in sample<commit_after>package main

import (
	"github.com/go-gl/cl/v1.2/cl"
	"log"
	"math"
	"math/rand"
	"unsafe"
)

const (
	DATA_SIZE = 1024
)

var KernelSource string = `
__kernel void square(
   __global float* input,
   __global float* output,
   const unsigned int count)
{
   int i = get_global_id(0);
   if(i < count)
	   output[i] = input[i] * input[i];
}` + "\x00"

func main() {

	data := make([]float32, DATA_SIZE)
	for x := 0; x < len(data); x++ {
		data[x] = rand.Float32()*99 + 1
	}

	//Get Device
	var device cl.DeviceId
	err := cl.GetDeviceIDs(nil, cl.DEVICE_TYPE_GPU, 1, &device, nil)
	if err != cl.SUCCESS {
		log.Fatal("Failed to create device group")
	}
	var errptr *int32

	//Create Computer Context
	context := cl.CreateContext(nil, 1, &device, nil, nil, errptr)
	if errptr != nil && cl.ErrorCode(*errptr) != cl.SUCCESS {
		log.Fatal("couldnt create context")
	}
	defer cl.ReleaseContext(context)

	//Create Command Queue
	cq := cl.CreateCommandQueue(*context, device, 0, errptr)
	if errptr != nil && cl.ErrorCode(*errptr) != cl.SUCCESS {
		log.Fatal("couldnt create command queue")
	}
	defer cl.ReleaseCommandQueue(cq)

	//Create program
	srcptr := cl.Str(KernelSource)
	program := cl.CreateProgramWithSource(*context, 1, &srcptr, nil, errptr)
	if errptr != nil && cl.ErrorCode(*errptr) != cl.SUCCESS {
		log.Fatal("couldnt create program")
	}
	defer cl.ReleaseProgram(program)

	err = cl.BuildProgram(program, 1, &device, nil, nil, nil)
	if err != cl.SUCCESS {
		var length uint64
		buffer := make([]byte, DATA_SIZE)

		log.Println("Error: Failed to build program executable!")
		cl.GetProgramBuildInfo(program, device, cl.PROGRAM_BUILD_LOG, uint64(len(buffer)), unsafe.Pointer(&buffer[0]), &length)
		log.Fatal(string(buffer[0:length]))
	}

	//Get Kernel
	kernel := cl.CreateKernel(program, cl.Str("square"+"\x00"), errptr)
	if errptr != nil && cl.ErrorCode(*errptr) != cl.SUCCESS {
		log.Fatal("couldnt create compute kernel")
	}
	defer cl.ReleaseKernel(kernel)

	//Create buffers
	input := cl.CreateBuffer(*context, cl.MEM_READ_ONLY, 4*DATA_SIZE, nil, errptr)
	if errptr != nil && cl.ErrorCode(*errptr) != cl.SUCCESS {
		log.Fatal("couldnt create input buffer")
	}
	defer cl.ReleaseMemObject(input)

	output := cl.CreateBuffer(*context, cl.MEM_WRITE_ONLY, 4*DATA_SIZE, nil, errptr)
	if errptr != nil && cl.ErrorCode(*errptr) != cl.SUCCESS {
		log.Fatal("couldnt create output buffer")
	}
	defer cl.ReleaseMemObject(output)

	//Write data
	err = cl.EnqueueWriteBuffer(cq, input, cl.TRUE, 0, 4*DATA_SIZE, unsafe.Pointer(&data[0]), 0, nil, nil)
	if err != cl.SUCCESS {
		log.Fatal("Failed to write to source array")
	}

	//Set kernel args
	count := uint32(DATA_SIZE)
	err = cl.SetKernelArg(kernel, 0, 8, unsafe.Pointer(&input))
	if err != cl.SUCCESS {
		log.Fatal("Failed to write kernel arg 0")
	}
	err = cl.SetKernelArg(kernel, 1, 8, unsafe.Pointer(&output))
	if err != cl.SUCCESS {
		log.Fatal("Failed to write kernel arg 1")
	}
	err = cl.SetKernelArg(kernel, 2, 4, unsafe.Pointer(&count))
	if err != cl.SUCCESS {
		log.Fatal("Failed to write kernel arg 2")
	}

	local := uint64(0)
	err = cl.GetKernelWorkGroupInfo(kernel, device, cl.KERNEL_WORK_GROUP_SIZE, 8, unsafe.Pointer(&local), nil)
	if err != cl.SUCCESS {
		log.Fatal("Failed to get kernel work group info")
	}

	global := local
	err = cl.EnqueueNDRangeKernel(cq, kernel, 1, nil, &global, &local, 0, nil, nil)
	if err != cl.SUCCESS {
		log.Fatal("Failed to execute kernel!")
	}

	cl.Finish(cq)

	results := make([]float32, DATA_SIZE)
	err = cl.EnqueueReadBuffer(cq, output, cl.TRUE, 0, 4*1024, unsafe.Pointer(&results[0]), 0, nil, nil)
	if err != cl.SUCCESS {
		log.Fatal("Failed to read buffer!")
	}

	success := 0
	notzero := 0
	for i, x := range data {
		if math.Abs(float64(x*x-results[i])) < 0.5 {
			success++
		}
		if results[i] > 0 {
			notzero++
		}
		log.Printf("I/O: %f\t%f", x, results[i])
	}

	log.Printf("%d/%d success", success, DATA_SIZE)
	log.Printf("values not zero: %d", notzero)
}
<|endoftext|>


<commit_before>package screen

import "github.com/elpinal/coco3/config"

type Screen interface {
	Start(*config.Config, bool, []rune, int, *Hi)
	Refresh(*config.Config, bool, []rune, int, *Hi)
	SetLastLine(string)
}

type Hi struct {
	Left, Right int
}

type TestScreen struct {
}

func (_ *TestScreen) Start(_ *config.Config, _ bool, _ []rune, _ int, _ *Hi) {
}

func (_ *TestScreen) Refresh(_ *config.Config, _ bool, _ []rune, _ int, _ *Hi) {
}

func (_ *TestScreen) SetLastLine(_ string) {
}
<commit_msg>Write comments to types declared in 'screen' package<commit_after>package screen

import "github.com/elpinal/coco3/config"

type Screen interface {
	Start(*config.Config, bool, []rune, int, *Hi)
	Refresh(*config.Config, bool, []rune, int, *Hi)
	SetLastLine(string)
}

// Hi represents a range for highlight.
type Hi struct {
	Left, Right int
}

// TestScreen is one of the simplest implementation for Screen. It does not
// anything. It is useful for testing.
type TestScreen struct{}

func (_ *TestScreen) Start(_ *config.Config, _ bool, _ []rune, _ int, _ *Hi) {
}

func (_ *TestScreen) Refresh(_ *config.Config, _ bool, _ []rune, _ int, _ *Hi) {
}

func (_ *TestScreen) SetLastLine(_ string) {
}
<|endoftext|>


<commit_before>package blocks

import (
	"../parser/"
	//"fmt"
)

// for more info see the ArduinoPWM.go file
type ArduinoPWMOutput struct {
	OutputBlockData
	address  string
	question ArduinoPWMPacket
}

func (b *ArduinoPWMOutput) Update() {
	numBytes := len(b.in)
	b.question.Header1.NumBytes = uint8(numBytes)

	for i, v := range b.in {
		b.question.Bytes[i] = byte(uint8(v))
	}

	//fmt.Println("sending message")
	_, err := SendReceiveArduinoPWM(b.address, b.question) // ORIG
	//SendArduinoPWM(b.address, b.question)

	if err == nil { // ORIG
		b.out = b.in
	} // ORIG
}

func ArduinoPWMOutputConstructor(name string, words []string) Block {
	var address string
	var bitRate int
	var pulseWidth int
	var clearCount int
	var numRepeat int

	positional := parser.PositionalArgs(&address, &bitRate, &pulseWidth, &clearCount, &numRepeat)
	optional := parser.OptionalArgs()

	parser.ParseArgs(words, positional, optional)

	// function implemented in ./Serial.go
	configId := 0
	MakeSerialPort(address, bitRate, configId)

	b := &ArduinoPWMOutput{
		address: address,
		question: ArduinoPWMPacket{
			Header1: ArduinoPWMHeader1{
				OpCode:     ARDUINOPWM_WRITEOP,
				PulseWidth: uint16(pulseWidth),
			},
			Header2: ArduinoPWMHeader2{
				ClearCount: uint8(clearCount),
				NumRepeat:  uint8(numRepeat),
			},
		},
	}

	return b
}

var ArduinoPWMOutputOk = AddConstructor("ArduinoPWMOutput", ArduinoPWMOutputConstructor)
<commit_msg>prepared arduinoPWMoutput for undefined states<commit_after>package blocks

import (
	"../parser/"
	//"fmt"
)

// for more info see the ArduinoPWM.go file
type ArduinoPWMOutput struct {
	OutputBlockData
	address  string
	question ArduinoPWMPacket
}

func (b *ArduinoPWMOutput) Update() {
	numBytes := len(b.in)
	b.question.Header1.NumBytes = uint8(numBytes)

	for i, v := range b.in {
		if v < 256.0 {
			b.question.Bytes[i] = byte(uint8(v))
		} else {
			b.question.Bytes[i] = 0
		}
	}

	//fmt.Println("sending message")
	_, err := SendReceiveArduinoPWM(b.address, b.question) // ORIG
	//SendArduinoPWM(b.address, b.question)

	if err == nil { // ORIG
		b.out = b.in
	} // ORIG
}

func ArduinoPWMOutputConstructor(name string, words []string) Block {
	var address string
	var bitRate int
	var pulseWidth int
	var clearCount int
	var numRepeat int

	positional := parser.PositionalArgs(&address, &bitRate, &pulseWidth, &clearCount, &numRepeat)
	optional := parser.OptionalArgs()

	parser.ParseArgs(words, positional, optional)

	// function implemented in ./Serial.go
	configId := 0
	MakeSerialPort(address, bitRate, configId)

	b := &ArduinoPWMOutput{
		address: address,
		question: ArduinoPWMPacket{
			Header1: ArduinoPWMHeader1{
				OpCode:     ARDUINOPWM_WRITEOP,
				PulseWidth: uint16(pulseWidth),
			},
			Header2: ArduinoPWMHeader2{
				ClearCount: uint8(clearCount),
				NumRepeat:  uint8(numRepeat),
			},
		},
	}

	return b
}

var ArduinoPWMOutputOk = AddConstructor("ArduinoPWMOutput", ArduinoPWMOutputConstructor)
<|endoftext|>


<commit_before>/*
 * Ferret
 * Copyright (c) 2016 Yieldbot, Inc.
 * For the full copyright and license information, please view the LICENSE.txt file.
 */

// Package search provides search interface and functionality
package search

import (
	"errors"
	"fmt"
	"log"
	"net/http"
	"os"
	"os/exec"
	"sort"
	"strconv"
	"time"

	"github.com/yieldbot/gocli"
	"golang.org/x/net/context"
)

var (
	goCommand     = "open"
	searchTimeout = "5000ms"
	providers     = make(map[string]Searcher)
)

func init() {
	if os.Getenv("FERRET_GOTO_CMD") != "" {
		goCommand = os.Getenv("FERRET_GOTO_CMD")
	}

	if os.Getenv("FERRET_SEARCH_TIMEOUT") != "" {
		searchTimeout = os.Getenv("FERRET_SEARCH_TIMEOUT")
	}
}

// Searcher is the interface that must be implemented by a search provider
type Searcher interface {
	// Search makes a search
	Search(ctx context.Context, keyword string, page int) (Results, error)
}

// Register registers a search provider
func Register(name string, provider Searcher) error {
	if _, ok := providers[name]; ok {
		return errors.New("provider " + name + " is already registered")
	}
	providers[name] = provider
	return nil
}

// Providers returns a sorted list of the names of the registered providers
func Providers() []string {
	var list = []string{}
	for name := range providers {
		list = append(list, name)
	}
	sort.Strings(list)
	return list
}

// ByKeyword make a search by the given provider and keyword
func ByKeyword(provider, keyword string, args map[string]string) {

	start := time.Now()

	// Check the args
	s, ok := providers[provider]
	if !ok {
		log.Fatalf("invalid provider. Possible providers are %s", Providers())
	}
	if keyword == "" {
		log.Fatal("missing keyword")
	}

	// Page
	page := 1
	p, ok := args["page"]
	if ok {
		i, err := strconv.Atoi(p)
		if err != nil || i <= 0 {
			log.Fatal("invalid page #. It should be greater than 0")
		}
		page = i
	}

	// Timeout
	var timeout = 5000 * time.Millisecond
	to, ok := args["timeout"]
	if ok {
		d, err := time.ParseDuration(to)
		if err != nil {
			log.Fatal("invalid timeout. It should be a duration (i.e. 5000ms)")
		}
		timeout = d
	} else {
		d, err := time.ParseDuration(searchTimeout)
		if err == nil {
			timeout = d
		}
	}

	// Search
	ctx, cancel := context.WithTimeout(context.Background(), timeout)
	defer cancel()
	results, err := s.Search(ctx, keyword, page)
	if err != nil {
		log.Fatalf("failed to search due to %s", err.Error())
	}
	elapsed := time.Since(start)

	// Goto
	if n, ok := args["goto"]; ok {
		i, err := strconv.Atoi(n)
		if err != nil || (i <= 0 || len(results) < i) {
			log.Fatalf("invalid result # to go. It should be between 1 and %d", len(results))
		}
		link := results[i-1].Link
		if _, err = exec.Command(goCommand, link).Output(); err != nil {
			log.Fatalf("failed to go to %s due to %s. Check FERRET_GOTO_CMD environment variable", link, err.Error())
		}
		return
	}

	// Prepare output
	var t = gocli.Table{}
	t.AddRow(1, "#", "DESCRIPTION")
	for i, v := range results {
		t.AddRow(i+2, fmt.Sprintf("%d", i+1), v.Description)
	}
	t.PrintData()
	fmt.Printf("\n%dms\n", int64(elapsed/time.Millisecond))
}

// DoRequest makes a HTTP request with contex
func DoRequest(ctx context.Context, req *http.Request, f func(*http.Response, error) error) error {
	tr := &http.Transport{}
	client := &http.Client{Transport: tr}
	c := make(chan error, 1)
	go func() {
		c <- f(client.Do(req))
	}()
	select {
	case <-ctx.Done():
		tr.CancelRequest(req)
		<-c
		return ctx.Err()
	case err := <-c:
		return err
	}
}
<commit_msg>Improve env handling<commit_after>/*
 * Ferret
 * Copyright (c) 2016 Yieldbot, Inc.
 * For the full copyright and license information, please view the LICENSE.txt file.
 */

// Package search provides search interface and functionality
package search

import (
	"errors"
	"fmt"
	"log"
	"net/http"
	"os"
	"os/exec"
	"sort"
	"strconv"
	"time"

	"github.com/yieldbot/gocli"
	"golang.org/x/net/context"
)

var (
	goCommand     = "open"
	searchTimeout = "5000ms"
	providers     = make(map[string]Searcher)
)

func init() {
	if e := os.Getenv("FERRET_GOTO_CMD"); e != "" {
		goCommand = e
	}

	if e := os.Getenv("FERRET_SEARCH_TIMEOUT"); e != "" {
		searchTimeout = e
	}
}

// Searcher is the interface that must be implemented by a search provider
type Searcher interface {
	// Search makes a search
	Search(ctx context.Context, keyword string, page int) (Results, error)
}

// Register registers a search provider
func Register(name string, provider Searcher) error {
	if _, ok := providers[name]; ok {
		return errors.New("provider " + name + " is already registered")
	}
	providers[name] = provider
	return nil
}

// Providers returns a sorted list of the names of the registered providers
func Providers() []string {
	var list = []string{}
	for name := range providers {
		list = append(list, name)
	}
	sort.Strings(list)
	return list
}

// ByKeyword make a search by the given provider and keyword
func ByKeyword(provider, keyword string, args map[string]string) {

	start := time.Now()

	// Check the args
	s, ok := providers[provider]
	if !ok {
		log.Fatalf("invalid provider. Possible providers are %s", Providers())
	}
	if keyword == "" {
		log.Fatal("missing keyword")
	}

	// Page
	page := 1
	p, ok := args["page"]
	if ok {
		i, err := strconv.Atoi(p)
		if err != nil || i <= 0 {
			log.Fatal("invalid page #. It should be greater than 0")
		}
		page = i
	}

	// Timeout
	var timeout = 5000 * time.Millisecond
	to, ok := args["timeout"]
	if ok {
		d, err := time.ParseDuration(to)
		if err != nil {
			log.Fatal("invalid timeout. It should be a duration (i.e. 5000ms)")
		}
		timeout = d
	} else {
		d, err := time.ParseDuration(searchTimeout)
		if err == nil {
			timeout = d
		}
	}

	// Search
	ctx, cancel := context.WithTimeout(context.Background(), timeout)
	defer cancel()
	results, err := s.Search(ctx, keyword, page)
	if err != nil {
		log.Fatalf("failed to search due to %s", err.Error())
	}
	elapsed := time.Since(start)

	// Goto
	if n, ok := args["goto"]; ok {
		i, err := strconv.Atoi(n)
		if err != nil || (i <= 0 || len(results) < i) {
			log.Fatalf("invalid result # to go. It should be between 1 and %d", len(results))
		}
		link := results[i-1].Link
		if _, err = exec.Command(goCommand, link).Output(); err != nil {
			log.Fatalf("failed to go to %s due to %s. Check FERRET_GOTO_CMD environment variable", link, err.Error())
		}
		return
	}

	// Prepare output
	var t = gocli.Table{}
	t.AddRow(1, "#", "DESCRIPTION")
	for i, v := range results {
		t.AddRow(i+2, fmt.Sprintf("%d", i+1), v.Description)
	}
	t.PrintData()
	fmt.Printf("\n%dms\n", int64(elapsed/time.Millisecond))
}

// DoRequest makes a HTTP request with contex
func DoRequest(ctx context.Context, req *http.Request, f func(*http.Response, error) error) error {
	tr := &http.Transport{}
	client := &http.Client{Transport: tr}
	c := make(chan error, 1)
	go func() {
		c <- f(client.Do(req))
	}()
	select {
	case <-ctx.Done():
		tr.CancelRequest(req)
		<-c
		return ctx.Err()
	case err := <-c:
		return err
	}
}
<|endoftext|>


<commit_before>package ca

import (
	"crypto"
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/hex"
	"fmt"
	"log"
	"math"
	"math/big"
	"net"
	"time"

	"github.com/letsencrypt/pebble/acme"
	"github.com/letsencrypt/pebble/core"
	"github.com/letsencrypt/pebble/db"
)

const (
	rootCAPrefix         = "Pebble Root CA "
	intermediateCAPrefix = "Pebble Intermediate CA "
)

type CAImpl struct {
	log              *log.Logger
	db               *db.MemoryStore
	ocspResponderURL string

	chains []*chain
}

type chain struct {
	root         *issuer
	intermediate *issuer
}

type issuer struct {
	key  crypto.Signer
	cert *core.Certificate
}

func makeSerial() *big.Int {
	serial, err := rand.Int(rand.Reader, big.NewInt(math.MaxInt64))
	if err != nil {
		panic(fmt.Sprintf("unable to create random serial number: %s", err.Error()))
	}
	return serial
}

// makeKey and makeRootCert are adapted from MiniCA:
// https://github.com/jsha/minica/blob/3a621c05b61fa1c24bcb42fbde4b261db504a74f/main.go

// makeKey creates a new 2048 bit RSA private key
func makeKey() (*rsa.PrivateKey, error) {
	key, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		return nil, err
	}
	return key, nil
}

func (ca *CAImpl) makeRootCert(
	subjectKey crypto.Signer,
	subject pkix.Name,
	signer *issuer) (*core.Certificate, error) {

	serial := makeSerial()
	template := &x509.Certificate{
		Subject:      subject,
		SerialNumber: serial,
		NotBefore:    time.Now(),
		NotAfter:     time.Now().AddDate(30, 0, 0),

		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
		BasicConstraintsValid: true,
		IsCA:                  true,
	}

	var signerKey crypto.Signer
	var parent *x509.Certificate
	if signer != nil && signer.key != nil && signer.cert != nil && signer.cert.Cert != nil {
		signerKey = signer.key
		parent = signer.cert.Cert
	} else {
		signerKey = subjectKey
		parent = template
	}

	der, err := x509.CreateCertificate(rand.Reader, template, parent, subjectKey.Public(), signerKey)
	if err != nil {
		return nil, err
	}

	cert, err := x509.ParseCertificate(der)
	if err != nil {
		return nil, err
	}

	hexSerial := hex.EncodeToString(cert.SerialNumber.Bytes())
	newCert := &core.Certificate{
		ID:   hexSerial,
		Cert: cert,
		DER:  der,
	}
	if signer != nil && signer.cert != nil {
		newCert.Issuers = make([]*core.Certificate, 1)
		newCert.Issuers[0] = signer.cert
	}
	_, err = ca.db.AddCertificate(newCert)
	if err != nil {
		return nil, err
	}
	return newCert, nil
}

func (ca *CAImpl) newRootIssuer() (*issuer, error) {
	// Make a root private key
	rk, err := makeKey()
	if err != nil {
		return nil, err
	}
	// Make a self-signed root certificate
	subject := pkix.Name{
		CommonName: rootCAPrefix + hex.EncodeToString(makeSerial().Bytes()[:3]),
	}
	rc, err := ca.makeRootCert(rk, subject, nil)
	if err != nil {
		return nil, err
	}

	ca.log.Printf("Generated new root issuer with serial %s\n", rc.ID)
	return &issuer{
		key:  rk,
		cert: rc,
	}, nil
}

func (ca *CAImpl) newIntermediateIssuer(root *issuer, intermediateKey crypto.Signer, subject pkix.Name) (*issuer, error) {
	if root == nil {
		return nil, fmt.Errorf("Internal error: root must not be nil")
	}
	// Make an intermediate certificate with the root issuer
	ic, err := ca.makeRootCert(intermediateKey, subject, root)
	if err != nil {
		return nil, err
	}
	ca.log.Printf("Generated new intermediate issuer with serial %s\n", ic.ID)
	return &issuer{
		key:  intermediateKey,
		cert: ic,
	}, nil
}

func (ca *CAImpl) newChain(intermediateKey crypto.Signer, intermediateSubject pkix.Name) *chain {
	root, err := ca.newRootIssuer()
	if err != nil {
		panic(fmt.Sprintf("Error creating new root issuer: %s", err.Error()))
	}
	intermediate, err := ca.newIntermediateIssuer(root, intermediateKey, intermediateSubject)
	if err != nil {
		panic(fmt.Sprintf("Error creating new intermediate issuer: %s", err.Error()))
	}
	return &chain{
		root:         root,
		intermediate: intermediate,
	}
}

func (ca *CAImpl) newCertificate(domains []string, ips []net.IP, key crypto.PublicKey, accountID string) (*core.Certificate, error) {
	var cn string
	if len(domains) > 0 {
		cn = domains[0]
	} else if len(ips) > 0 {
		cn = ips[0].String()
	} else {
		return nil, fmt.Errorf("must specify at least one domain name or IP address")
	}

	issuer := ca.chains[0].intermediate
	if issuer == nil || issuer.cert == nil {
		return nil, fmt.Errorf("cannot sign certificate - nil issuer")
	}

	serial := makeSerial()
	template := &x509.Certificate{
		DNSNames:    domains,
		IPAddresses: ips,
		Subject: pkix.Name{
			CommonName: cn,
		},
		SerialNumber: serial,
		NotBefore:    time.Now(),
		NotAfter:     time.Now().AddDate(5, 0, 0),

		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
		BasicConstraintsValid: true,
		IsCA:                  false,
	}

	if ca.ocspResponderURL != "" {
		template.OCSPServer = []string{ca.ocspResponderURL}
	}

	der, err := x509.CreateCertificate(rand.Reader, template, issuer.cert.Cert, key, issuer.key)
	if err != nil {
		return nil, err
	}
	cert, err := x509.ParseCertificate(der)
	if err != nil {
		return nil, err
	}

	issuers := make([]*core.Certificate, len(ca.chains))
	for i := 0; i < len(ca.chains); i++ {
		issuers[i] = ca.chains[i].intermediate.cert
	}

	hexSerial := hex.EncodeToString(cert.SerialNumber.Bytes())
	newCert := &core.Certificate{
		ID:        hexSerial,
		AccountID: accountID,
		Cert:      cert,
		DER:       der,
		Issuers:   issuers,
	}
	_, err = ca.db.AddCertificate(newCert)
	if err != nil {
		return nil, err
	}
	return newCert, nil
}

func New(log *log.Logger, db *db.MemoryStore, ocspResponderURL string, alternateRoots int) *CAImpl {
	ca := &CAImpl{
		log: log,
		db:  db,
	}

	if ocspResponderURL != "" {
		ca.ocspResponderURL = ocspResponderURL
		ca.log.Printf("Setting OCSP responder URL for issued certificates to %q", ca.ocspResponderURL)
	}

	intermediateSubject := pkix.Name{
		CommonName: intermediateCAPrefix + hex.EncodeToString(makeSerial().Bytes()[:3]),
	}
	intermediateKey, err := makeKey()
	if err != nil {
		panic(fmt.Sprintf("Error creating new intermediate private key: %s", err.Error()))
	}
	ca.chains = make([]*chain, 1+alternateRoots)
	for i := 0; i < len(ca.chains); i++ {
		ca.chains[i] = ca.newChain(intermediateKey, intermediateSubject)
	}
	return ca
}

func (ca *CAImpl) CompleteOrder(order *core.Order) {
	// Lock the order for reading
	order.RLock()
	// If the order isn't set as beganProcessing produce an error and immediately unlock
	if !order.BeganProcessing {
		ca.log.Printf("Error: Asked to complete order %s which had false beganProcessing.",
			order.ID)
		order.RUnlock()
		return
	}
	// Unlock the order again
	order.RUnlock()

	// Check the authorizations - this is done by the VA before calling
	// CompleteOrder but we do it again for robustness sake.
	for _, authz := range order.AuthorizationObjects {
		// Lock the authorization for reading
		authz.RLock()
		if authz.Status != acme.StatusValid {
			return
		}
		authz.RUnlock()
	}

	// issue a certificate for the csr
	csr := order.ParsedCSR
	cert, err := ca.newCertificate(csr.DNSNames, csr.IPAddresses, csr.PublicKey, order.AccountID)
	if err != nil {
		ca.log.Printf("Error: unable to issue order: %s", err.Error())
		return
	}
	ca.log.Printf("Issued certificate serial %s for order %s\n", cert.ID, order.ID)

	// Lock and update the order to store the issued certificate
	order.Lock()
	order.CertificateObject = cert
	order.Unlock()
}

func (ca *CAImpl) GetNumberOfRootCerts() int {
	return len(ca.chains)
}

func (ca *CAImpl) getChain(no int) *chain {
	if 0 <= no && no < len(ca.chains) {
		return ca.chains[no]
	}
	return nil
}

func (ca *CAImpl) GetRootCert(no int) *core.Certificate {
	chain := ca.getChain(no)
	if chain == nil {
		return nil
	}
	return chain.root.cert
}

func (ca *CAImpl) GetRootKey(no int) *rsa.PrivateKey {
	chain := ca.getChain(no)
	if chain == nil {
		return nil
	}

	switch key := chain.root.key.(type) {
	case *rsa.PrivateKey:
		return key
	}
	return nil
}

func (ca *CAImpl) GetIntermediateCert(no int) *core.Certificate {
	chain := ca.getChain(no)
	if chain == nil {
		return nil
	}
	return chain.intermediate.cert
}

func (ca *CAImpl) GetIntermediateKey(no int) *rsa.PrivateKey {
	chain := ca.getChain(no)
	if chain == nil {
		return nil
	}

	switch key := chain.intermediate.key.(type) {
	case *rsa.PrivateKey:
		return key
	}
	return nil
}
<commit_msg>CA: Add SubjectKeyId to certificates. (#269)<commit_after>package ca

import (
	"crypto"
	"crypto/rand"
	"crypto/rsa"
	"crypto/sha1"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/asn1"
	"encoding/hex"
	"fmt"
	"log"
	"math"
	"math/big"
	"net"
	"time"

	"github.com/letsencrypt/pebble/acme"
	"github.com/letsencrypt/pebble/core"
	"github.com/letsencrypt/pebble/db"
)

const (
	rootCAPrefix         = "Pebble Root CA "
	intermediateCAPrefix = "Pebble Intermediate CA "
)

type CAImpl struct {
	log              *log.Logger
	db               *db.MemoryStore
	ocspResponderURL string

	chains []*chain
}

type chain struct {
	root         *issuer
	intermediate *issuer
}

type issuer struct {
	key  crypto.Signer
	cert *core.Certificate
}

func makeSerial() *big.Int {
	serial, err := rand.Int(rand.Reader, big.NewInt(math.MaxInt64))
	if err != nil {
		panic(fmt.Sprintf("unable to create random serial number: %s", err.Error()))
	}
	return serial
}

// Taken from https://github.com/cloudflare/cfssl/blob/b94e044bb51ec8f5a7232c71b1ed05dbe4da96ce/signer/signer.go#L221-L244
func makeSubjectKeyID(key crypto.PublicKey) ([]byte, error) {
	// Marshal the public key as ASN.1
	pubAsDER, err := x509.MarshalPKIXPublicKey(key)
	if err != nil {
		return nil, err
	}

	// Unmarshal it again so we can extract the key bitstring bytes
	var pubInfo struct {
		Algorithm        pkix.AlgorithmIdentifier
		SubjectPublicKey asn1.BitString
	}
	_, err = asn1.Unmarshal(pubAsDER, &pubInfo)
	if err != nil {
		return nil, err
	}

	// Hash it according to https://tools.ietf.org/html/rfc5280#section-4.2.1.2 Method #1:
	ski := sha1.Sum(pubInfo.SubjectPublicKey.Bytes)
	return ski[:], nil
}

// makeKey and makeRootCert are adapted from MiniCA:
// https://github.com/jsha/minica/blob/3a621c05b61fa1c24bcb42fbde4b261db504a74f/main.go

// makeKey creates a new 2048 bit RSA private key and a Subject Key Identifier
func makeKey() (*rsa.PrivateKey, []byte, error) {
	key, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		return nil, nil, err
	}
	ski, err := makeSubjectKeyID(key.Public())
	if err != nil {
		return nil, nil, err
	}
	return key, ski, nil
}

func (ca *CAImpl) makeRootCert(
	subjectKey crypto.Signer,
	subject pkix.Name,
	subjectKeyID []byte,
	signer *issuer) (*core.Certificate, error) {

	serial := makeSerial()
	template := &x509.Certificate{
		Subject:      subject,
		SerialNumber: serial,
		NotBefore:    time.Now(),
		NotAfter:     time.Now().AddDate(30, 0, 0),

		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
		SubjectKeyId:          subjectKeyID,
		BasicConstraintsValid: true,
		IsCA:                  true,
	}

	var signerKey crypto.Signer
	var parent *x509.Certificate
	if signer != nil && signer.key != nil && signer.cert != nil && signer.cert.Cert != nil {
		signerKey = signer.key
		parent = signer.cert.Cert
	} else {
		signerKey = subjectKey
		parent = template
	}

	der, err := x509.CreateCertificate(rand.Reader, template, parent, subjectKey.Public(), signerKey)
	if err != nil {
		return nil, err
	}

	cert, err := x509.ParseCertificate(der)
	if err != nil {
		return nil, err
	}

	hexSerial := hex.EncodeToString(cert.SerialNumber.Bytes())
	newCert := &core.Certificate{
		ID:   hexSerial,
		Cert: cert,
		DER:  der,
	}
	if signer != nil && signer.cert != nil {
		newCert.Issuers = make([]*core.Certificate, 1)
		newCert.Issuers[0] = signer.cert
	}
	_, err = ca.db.AddCertificate(newCert)
	if err != nil {
		return nil, err
	}
	return newCert, nil
}

func (ca *CAImpl) newRootIssuer() (*issuer, error) {
	// Make a root private key
	rk, subjectKeyID, err := makeKey()
	if err != nil {
		return nil, err
	}
	// Make a self-signed root certificate
	subject := pkix.Name{
		CommonName: rootCAPrefix + hex.EncodeToString(makeSerial().Bytes()[:3]),
	}
	rc, err := ca.makeRootCert(rk, subject, subjectKeyID, nil)
	if err != nil {
		return nil, err
	}

	ca.log.Printf("Generated new root issuer with serial %s and SKI %x\n", rc.ID, subjectKeyID)
	return &issuer{
		key:  rk,
		cert: rc,
	}, nil
}

func (ca *CAImpl) newIntermediateIssuer(root *issuer, intermediateKey crypto.Signer, subject pkix.Name, subjectKeyID []byte) (*issuer, error) {
	if root == nil {
		return nil, fmt.Errorf("Internal error: root must not be nil")
	}
	// Make an intermediate certificate with the root issuer
	ic, err := ca.makeRootCert(intermediateKey, subject, subjectKeyID, root)
	if err != nil {
		return nil, err
	}
	ca.log.Printf("Generated new intermediate issuer with serial %s and SKI %x\n", ic.ID, subjectKeyID)
	return &issuer{
		key:  intermediateKey,
		cert: ic,
	}, nil
}

func (ca *CAImpl) newChain(intermediateKey crypto.Signer, intermediateSubject pkix.Name, subjectKeyID []byte) *chain {
	root, err := ca.newRootIssuer()
	if err != nil {
		panic(fmt.Sprintf("Error creating new root issuer: %s", err.Error()))
	}
	intermediate, err := ca.newIntermediateIssuer(root, intermediateKey, intermediateSubject, subjectKeyID)
	if err != nil {
		panic(fmt.Sprintf("Error creating new intermediate issuer: %s", err.Error()))
	}
	return &chain{
		root:         root,
		intermediate: intermediate,
	}
}

func (ca *CAImpl) newCertificate(domains []string, ips []net.IP, key crypto.PublicKey, accountID string) (*core.Certificate, error) {
	var cn string
	if len(domains) > 0 {
		cn = domains[0]
	} else if len(ips) > 0 {
		cn = ips[0].String()
	} else {
		return nil, fmt.Errorf("must specify at least one domain name or IP address")
	}

	issuer := ca.chains[0].intermediate
	if issuer == nil || issuer.cert == nil {
		return nil, fmt.Errorf("cannot sign certificate - nil issuer")
	}

	subjectKeyID, err := makeSubjectKeyID(key)
	if err != nil {
		return nil, fmt.Errorf("cannot create subject key ID: %s", err.Error())
	}

	serial := makeSerial()
	template := &x509.Certificate{
		DNSNames:    domains,
		IPAddresses: ips,
		Subject: pkix.Name{
			CommonName: cn,
		},
		SerialNumber: serial,
		NotBefore:    time.Now(),
		NotAfter:     time.Now().AddDate(5, 0, 0),

		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
		SubjectKeyId:          subjectKeyID,
		BasicConstraintsValid: true,
		IsCA:                  false,
	}

	if ca.ocspResponderURL != "" {
		template.OCSPServer = []string{ca.ocspResponderURL}
	}

	der, err := x509.CreateCertificate(rand.Reader, template, issuer.cert.Cert, key, issuer.key)
	if err != nil {
		return nil, err
	}
	cert, err := x509.ParseCertificate(der)
	if err != nil {
		return nil, err
	}

	issuers := make([]*core.Certificate, len(ca.chains))
	for i := 0; i < len(ca.chains); i++ {
		issuers[i] = ca.chains[i].intermediate.cert
	}

	hexSerial := hex.EncodeToString(cert.SerialNumber.Bytes())
	newCert := &core.Certificate{
		ID:        hexSerial,
		AccountID: accountID,
		Cert:      cert,
		DER:       der,
		Issuers:   issuers,
	}
	_, err = ca.db.AddCertificate(newCert)
	if err != nil {
		return nil, err
	}
	return newCert, nil
}

func New(log *log.Logger, db *db.MemoryStore, ocspResponderURL string, alternateRoots int) *CAImpl {
	ca := &CAImpl{
		log: log,
		db:  db,
	}

	if ocspResponderURL != "" {
		ca.ocspResponderURL = ocspResponderURL
		ca.log.Printf("Setting OCSP responder URL for issued certificates to %q", ca.ocspResponderURL)
	}

	intermediateSubject := pkix.Name{
		CommonName: intermediateCAPrefix + hex.EncodeToString(makeSerial().Bytes()[:3]),
	}
	intermediateKey, subjectKeyID, err := makeKey()
	if err != nil {
		panic(fmt.Sprintf("Error creating new intermediate private key: %s", err.Error()))
	}
	ca.chains = make([]*chain, 1+alternateRoots)
	for i := 0; i < len(ca.chains); i++ {
		ca.chains[i] = ca.newChain(intermediateKey, intermediateSubject, subjectKeyID)
	}
	return ca
}

func (ca *CAImpl) CompleteOrder(order *core.Order) {
	// Lock the order for reading
	order.RLock()
	// If the order isn't set as beganProcessing produce an error and immediately unlock
	if !order.BeganProcessing {
		ca.log.Printf("Error: Asked to complete order %s which had false beganProcessing.",
			order.ID)
		order.RUnlock()
		return
	}
	// Unlock the order again
	order.RUnlock()

	// Check the authorizations - this is done by the VA before calling
	// CompleteOrder but we do it again for robustness sake.
	for _, authz := range order.AuthorizationObjects {
		// Lock the authorization for reading
		authz.RLock()
		if authz.Status != acme.StatusValid {
			return
		}
		authz.RUnlock()
	}

	// issue a certificate for the csr
	csr := order.ParsedCSR
	cert, err := ca.newCertificate(csr.DNSNames, csr.IPAddresses, csr.PublicKey, order.AccountID)
	if err != nil {
		ca.log.Printf("Error: unable to issue order: %s", err.Error())
		return
	}
	ca.log.Printf("Issued certificate serial %s for order %s\n", cert.ID, order.ID)

	// Lock and update the order to store the issued certificate
	order.Lock()
	order.CertificateObject = cert
	order.Unlock()
}

func (ca *CAImpl) GetNumberOfRootCerts() int {
	return len(ca.chains)
}

func (ca *CAImpl) getChain(no int) *chain {
	if 0 <= no && no < len(ca.chains) {
		return ca.chains[no]
	}
	return nil
}

func (ca *CAImpl) GetRootCert(no int) *core.Certificate {
	chain := ca.getChain(no)
	if chain == nil {
		return nil
	}
	return chain.root.cert
}

func (ca *CAImpl) GetRootKey(no int) *rsa.PrivateKey {
	chain := ca.getChain(no)
	if chain == nil {
		return nil
	}

	switch key := chain.root.key.(type) {
	case *rsa.PrivateKey:
		return key
	}
	return nil
}

func (ca *CAImpl) GetIntermediateCert(no int) *core.Certificate {
	chain := ca.getChain(no)
	if chain == nil {
		return nil
	}
	return chain.intermediate.cert
}

func (ca *CAImpl) GetIntermediateKey(no int) *rsa.PrivateKey {
	chain := ca.getChain(no)
	if chain == nil {
		return nil
	}

	switch key := chain.intermediate.key.(type) {
	case *rsa.PrivateKey:
		return key
	}
	return nil
}
<|endoftext|>


<commit_before>package cache

import (
	"sort"
	"time"
)

// A Cache is a thread-safe store for fast item storage retrieval
type Cache struct {
	ops chan func(map[string]interface{})
}

// Returns an empty cache
func New() *Cache {
	c := &Cache{
		ops: make(chan func(map[string]interface{})),
	}

	go c.loop()
	return c
}

func (c *Cache) loop() {
	items := map[string]interface{}{}
	for op := range c.ops {
		op(items)
	}
}

// Adds an entry into the cache at the specified key.
// If an entry already exists at the specified key, it will be overwritten
func (c *Cache) Add(key string, val interface{}) {
	c.ops <- func(items map[string]interface{}) {
		items[key] = val
	}
}

// Removes all entries from the Cache
func (c *Cache) Clear() {
	c.ops <- func(items map[string]interface{}) {
		for key := range items {
			delete(items, key)
		}
	}
}

// todo: this doesn't clear entries after N time, clears eveyrtintg after N time
// should be clear on this
func (c *Cache) ClearEvery(d time.Duration) *time.Ticker {
        ticker := time.NewTicker(d)
        go func() {
                for range ticker.C {
                        c.Clear()
                }
        }()

        return ticker
}

// Delete an entry from the cache at the specified key.
// If no entry exists at the specified key, no action is taken
func (c *Cache) Delete(key string) {
	c.ops <- func(items map[string]interface{}) {
		if _, ok := items[key]; ok {
			delete(items, key)
		}
	}
}

// Retrieves an entry at the specified key
func (c *Cache) Get(key string) interface{} {
	result := make(chan interface{}, 1)
	c.ops <- func(items map[string]interface{}) {
		result <- items[key]
	}

	return <-result
}

// Retrieves an entry at the specified key.
// Returns bool specifying if the entry exists
func (c *Cache) Getf(key string) (interface{}, bool) {
	result := make(chan interface{}, 1)
	exists := make(chan bool, 1)
	c.ops <- func(items map[string]interface{}) {
		v, ok := items[key]
		result <- v
		exists <- ok
	}

	return <-result, <-exists
}

// Retrieves all entry in the cache
func (c *Cache) Items() map[string]interface{} {
	result := make(chan map[string]interface{}, 1)
	c.ops <- func(items map[string]interface{}) {
		result <- items
	}

	return <-result
}

// Retrieves a sorted list of all keys in the cache
func (c *Cache) Keys() []string {
	result := make(chan []string, 1)
	c.ops <- func(items map[string]interface{}) {
		keys := make([]string, 0, len(items))
		for k := range items {
			keys = append(keys, k)
		}

		sort.Strings(keys)
		result <- keys
	}

	return <-result
}
<commit_msg>fix comment<commit_after>package cache

import (
	"sort"
	"time"
)

// A Cache is a thread-safe store for fast item storage retrieval
type Cache struct {
	ops chan func(map[string]interface{})
}

// Returns an empty cache
func New() *Cache {
	c := &Cache{
		ops: make(chan func(map[string]interface{})),
	}

	go c.loop()
	return c
}

func (c *Cache) loop() {
	items := map[string]interface{}{}
	for op := range c.ops {
		op(items)
	}
}

// Adds an entry into the cache at the specified key.
// If an entry already exists at the specified key, it will be overwritten
func (c *Cache) Add(key string, val interface{}) {
	c.ops <- func(items map[string]interface{}) {
		items[key] = val
	}
}

// Removes all entries from the cache
func (c *Cache) Clear() {
	c.ops <- func(items map[string]interface{}) {
		for key := range items {
			delete(items, key)
		}
	}
}

// Clears the cache on a loop after the specified duration
func (c *Cache) ClearEvery(d time.Duration) *time.Ticker {
        ticker := time.NewTicker(d)
        go func() {
                for range ticker.C {
                        c.Clear()
                }
        }()

        return ticker
}

// Delete an entry from the cache at the specified key.
// If no entry exists at the specified key, no action is taken
func (c *Cache) Delete(key string) {
	c.ops <- func(items map[string]interface{}) {
		if _, ok := items[key]; ok {
			delete(items, key)
		}
	}
}

// Retrieves an entry at the specified key
func (c *Cache) Get(key string) interface{} {
	result := make(chan interface{}, 1)
	c.ops <- func(items map[string]interface{}) {
		result <- items[key]
	}

	return <-result
}

// Retrieves an entry at the specified key.
// Returns bool specifying if the entry exists
func (c *Cache) Getf(key string) (interface{}, bool) {
	result := make(chan interface{}, 1)
	exists := make(chan bool, 1)
	c.ops <- func(items map[string]interface{}) {
		v, ok := items[key]
		result <- v
		exists <- ok
	}

	return <-result, <-exists
}

// Retrieves all entry in the cache
func (c *Cache) Items() map[string]interface{} {
	result := make(chan map[string]interface{}, 1)
	c.ops <- func(items map[string]interface{}) {
		result <- items
	}

	return <-result
}

// Retrieves a sorted list of all keys in the cache
func (c *Cache) Keys() []string {
	result := make(chan []string, 1)
	c.ops <- func(items map[string]interface{}) {
		keys := make([]string, 0, len(items))
		for k := range items {
			keys = append(keys, k)
		}

		sort.Strings(keys)
		result <- keys
	}

	return <-result
}
<|endoftext|>