<commit_before>package core

import (
	"bytes"
	"crypto/sha1"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"time"
)

// Root of the repository
var RepoRoot string

// Initial working directory
var initialWorkingDir string

// Initial subdir of the working directory, ie. what package did we start in.
var initialPackage string

const DirPermissions = os.ModeDir | 0775

// FindRepoRoot returns the root directory of the current repo and sets the initial working dir.
// Dies on failure if 'die' is set.
func FindRepoRoot(die bool) {
	initialWorkingDir, _ = os.Getwd()
	RepoRoot, initialPackage = getRepoRoot(die)
}

// getRepoRoot returns the root directory of the current repo and the initial package.
func getRepoRoot(die bool) (string, string) {
	dir, err := os.Getwd()
	if err != nil {
		log.Fatalf("Couldn't determine working directory: %s", err)
	}
	// Walk up directories looking for a .plzconfig file, which we use to identify the root.
	initial := dir
	for dir != "" {
		if PathExists(path.Join(dir, ConfigFileName)) {
			return dir, strings.TrimLeft(initial[len(dir):], "/")
		}
		dir, _ = path.Split(dir)
		dir = strings.TrimRight(dir, "/")
	}
	if die {
		log.Fatalf("Couldn't locate the repo root. Are you sure you're inside a plz repo?")
	}
	return "", ""
}

// Returns true if the build was initiated from the repo root.
// Used to provide slightly nicer output in some places.
func StartedAtRepoRoot() bool {
	return RepoRoot == initialWorkingDir
}

func PathExists(filename string) bool {
	_, err := os.Stat(filename)
	return err == nil
}

func FileExists(filename string) bool {
	info, err := os.Stat(filename)
	return err == nil && !info.IsDir()
}

// CopyFile copies a file from 'from' to 'to', with an attempt to perform a copy & rename
// to avoid chaos if anything goes wrong partway.
func CopyFile(from string, to string, mode os.FileMode) error {
	fromFile, err := os.Open(from)
	if err != nil {
		return err
	}
	defer fromFile.Close()
	return WriteFile(fromFile, to, mode)
}

// WriteFile writes data from a reader to the file named 'to', with an attempt to perform
// a copy & rename to avoid chaos if anything goes wrong partway.
func WriteFile(fromFile io.Reader, to string, mode os.FileMode) error {
	if err := os.RemoveAll(to); err != nil {
		return err
	}
	dir, file := path.Split(to)
	if err := os.MkdirAll(dir, DirPermissions); err != nil {
		return err
	}
	tempFile, err := ioutil.TempFile(dir, file)
	if err != nil {
		return err
	}
	if _, err := io.Copy(tempFile, fromFile); err != nil {
		return err
	}
	if err := tempFile.Close(); err != nil {
		return err
	}
	// OK, now file is written; adjust permissions appropriately.
	if mode == 0 {
		mode = 0664
	}
	if err := os.Chmod(tempFile.Name(), mode); err != nil {
		return err
	}
	// And move it to its final destination.
	return os.Rename(tempFile.Name(), to)
}

// Copies either a single file or a directory.
// If 'link' is true then we'll attempt to hardlink files instead of copying them
// (on failure we still attempt a copy).
func RecursiveCopyFile(from string, to string, mode os.FileMode, link bool) error {
	if info, err := os.Stat(from); err == nil && info.IsDir() {
		return filepath.Walk(from, func(name string, info os.FileInfo, err error) error {
			dest := path.Join(to, name[len(from):])
			if err != nil {
				return err
			} else if info.IsDir() {
				return os.MkdirAll(dest, DirPermissions)
			} else if (info.Mode() & os.ModeSymlink) != 0 {
				fi, err := os.Stat(name)
				if err != nil {
					return err
				}
				if fi.IsDir() {
					return RecursiveCopyFile(name+"/", dest+"/", mode, link)
				} else {
					return copyOrLinkFile(name, dest, mode, link)
				}
			} else {
				return copyOrLinkFile(name, dest, mode, link)
			}
		})
	} else {
		return copyOrLinkFile(from, to, mode, link)
	}
}

// Either copies or hardlinks a file based on the link argument.
func copyOrLinkFile(from, to string, mode os.FileMode, link bool) error {
	if link {
		if err := os.Link(from, to); err == nil {
			return nil
		}
	}
	return CopyFile(from, to, mode)
}

// TimeoutError is the type of error that's issued when a command times out.
type TimeoutError int

// Error formats this error as a string.
func (i TimeoutError) Error() string {
	return fmt.Sprintf("Timeout (%d seconds) exceeded", i)
}

// ExecWithTimeout runs an external command with a timeout.
// If the command times out the returned error will be a TimeoutError.
func ExecWithTimeout(cmd *exec.Cmd, timeout int, defaultTimeout int) ([]byte, error) {
	var out bytes.Buffer
	if timeout == 0 {
		timeout = defaultTimeout
	}
	cmd.Stdout = &out
	cmd.Stderr = &out
	ch := make(chan error)
	go func() { ch <- cmd.Run() }()
	select {
	case <-time.After(time.Duration(timeout) * time.Second):
		if err := cmd.Process.Kill(); err != nil {
			log.Errorf("Process %d could not be killed after exceeding timeout of %d seconds", cmd.Process.Pid, timeout)
		}
		return out.Bytes(), TimeoutError(timeout)
	case err := <-ch:
		return out.Bytes(), err
	}
}

type sourcePair struct{ Src, Tmp string }

// IterSources returns all the sources for a function, allowing for sources that are other rules
// and rules that require transitive dependencies.
// Yielded values are pairs of the original source location and its temporary location for this rule.
func IterSources(graph *BuildGraph, target *BuildTarget) <-chan sourcePair {
	ch := make(chan sourcePair)
	done := map[BuildLabel]bool{}
	donePaths := map[string]bool{}
	tmpDir := target.TmpDir()
	var inner func(dependency *BuildTarget)
	inner = func(dependency *BuildTarget) {
		if target == dependency {
			// This is the current build rule, so link its sources.
			for source := range dependency.AllSources() {
				fullPaths := source.FullPaths(graph)
				for i, sourcePath := range source.Paths(graph) {
					tmpPath := path.Join(tmpDir, sourcePath)
					ch <- sourcePair{fullPaths[i], tmpPath}
					donePaths[tmpPath] = true
				}
				if label := source.Label(); label != nil {
					done[*label] = true
				}
			}
		} else {
			// This is a dependency of the rule, so link its outputs.
			for _, dep := range dependency.Outputs() {
				depPath := path.Join(dependency.OutDir(), dep)
				tmpPath := path.Join(tmpDir, dependency.Label.PackageName, dep)
				if !donePaths[tmpPath] {
					ch <- sourcePair{depPath, tmpPath}
					donePaths[tmpPath] = true
				}
			}
			// Mark any label-type outputs as done.
			for _, out := range dependency.DeclaredOutputs() {
				if LooksLikeABuildLabel(out) {
					label := ParseBuildLabel(out, target.Label.PackageName)
					done[label] = true
				}
			}
		}
		done[dependency.Label] = true
		if target == dependency || (target.NeedsTransitiveDependencies && !dependency.OutputIsComplete) {
			for _, dep := range dependency.Dependencies() {
				if !done[dep.Label] && !target.IsTool(dep.Label) {
					inner(dep)
				}
			}
		} else {
			for _, dep := range dependency.ExportedDependencies() {
				for _, dep2 := range recursivelyProvideFor(graph, target, dep) {
					if !done[dep2] {
						inner(graph.TargetOrDie(dep2))
					}
				}
			}
		}
	}
	go func() {
		inner(target)
		close(ch)
	}()
	return ch
}

// recursivelyProvideFor recursively applies ProvideFor to a target.
func recursivelyProvideFor(graph *BuildGraph, target *BuildTarget, dep BuildLabel) []BuildLabel {
	ret := graph.TargetOrDie(dep).ProvideFor(target)
	if len(ret) == 1 && ret[0] == dep {
		return ret // Providing itself, don't recurse
	}
	ret2 := []BuildLabel{}
	for _, r := range ret {
		ret2 = append(ret2, recursivelyProvideFor(graph, target, r)...)
	}
	return ret2
}

// Yields all the runtime files for a rule (outputs & data files), similar to above.
func IterRuntimeFiles(graph *BuildGraph, target *BuildTarget, absoluteOuts bool) <-chan sourcePair {
	done := map[string]bool{}
	ch := make(chan sourcePair)

	makeOut := func(out string) string {
		if absoluteOuts {
			return path.Join(RepoRoot, target.TestDir(), out)
		} else {
			return out
		}
	}

	pushOut := func(src, out string) {
		out = makeOut(out)
		if !done[out] {
			ch <- sourcePair{src, out}
			done[out] = true
		}
	}

	var inner func(*BuildTarget)
	inner = func(target *BuildTarget) {
		for _, out := range target.Outputs() {
			pushOut(path.Join(target.OutDir(), out), out)
		}
		for _, data := range target.Data {
			fullPaths := data.FullPaths(graph)
			for i, dataPath := range data.Paths(graph) {
				pushOut(fullPaths[i], dataPath)
			}
			if label := data.Label(); label != nil {
				for _, dep := range graph.TargetOrDie(*label).ExportedDependencies() {
					inner(graph.TargetOrDie(dep))
				}
			}
		}
		for _, dep := range target.ExportedDependencies() {
			inner(graph.TargetOrDie(dep))
		}
	}
	go func() {
		inner(target)
		close(ch)
	}()
	return ch
}

// Yields all the transitive input files for a rule (sources & data files), similar to above (again).
func IterInputPaths(graph *BuildGraph, target *BuildTarget) <-chan string {
	// Use a couple of maps to protect us from dep-graph loops and to stop parsing the same target
	// multiple times. We also only want to push files to the channel that it has not already seen.
	donePaths := map[string]bool{}
	doneTargets := map[*BuildTarget]bool{}
	ch := make(chan string)
	var inner func(*BuildTarget)
	inner = func(target *BuildTarget) {

		if !doneTargets[target] {
			// First yield all the sources of the target only ever pushing declared paths to
			// the channel to prevent us outputting any intermediate files.
			for source := range target.AllSources() {
				// If the label is nil add any input paths contained here.
				if label := source.Label(); label == nil {
					for _, sourcePath := range source.FullPaths(graph) {
						if !donePaths[sourcePath] {
							ch <- sourcePath
							donePaths[sourcePath] = true
						}
					}
					// Otherwise we should recurse for this build label (and gather its sources)
				} else {
					inner(graph.TargetOrDie(*label))
				}
			}

			// Now yield all the data deps of this rule.
			for _, data := range target.Data {
				// If the label is nil add any input paths contained here.
				if label := data.Label(); label == nil {
					for _, sourcePath := range data.FullPaths(graph) {
						if !donePaths[sourcePath] {
							ch <- sourcePath
							donePaths[sourcePath] = true
						}
					}
					// Otherwise we should recurse for this build label (and gather its sources)
				} else {
					inner(graph.TargetOrDie(*label))
				}
			}

			// Finally recurse for all the deps of this rule.
			for _, dep := range target.Dependencies() {
				inner(dep)
			}
			doneTargets[target] = true
		}
	}
	go func() {
		inner(target)
		close(ch)
	}()
	return ch
}

// Symlinks a single source file for a build rule.
func PrepareSource(sourcePath string, tmpPath string) error {
	dir := path.Dir(tmpPath)
	if !PathExists(dir) {
		if err := os.MkdirAll(dir, DirPermissions); err != nil {
			return err
		}
	}
	if !PathExists(sourcePath) {
		return fmt.Errorf("Source file %s doesn't exist", sourcePath)
	}
	return RecursiveCopyFile(sourcePath, tmpPath, 0, true)
}

func PrepareSourcePair(pair sourcePair) error {
	if path.IsAbs(pair.Src) {
		return PrepareSource(pair.Src, pair.Tmp)
	}
	return PrepareSource(path.Join(RepoRoot, pair.Src), pair.Tmp)
}

func PostBuildOutputFileName(target *BuildTarget) string {
	return ".build_output_" + target.Label.Name
}

// CollapseHash combines our usual four-part hash into one by XOR'ing them together.
// This helps keep things short in places where sometimes we get complaints about filenames being too long (?)
// and where we don't especially care about breaking out the individual parts of hashes, which
// is important for many parts of the system.
func CollapseHash(key []byte) []byte {
	short := [sha1.Size]byte{}
	// We store the rule hash twice, if it's repeated we must make sure not to xor it
	// against itself.
	if bytes.Equal(key[0:sha1.Size], key[sha1.Size:2*sha1.Size]) {
		for i := 0; i < sha1.Size; i++ {
			short[i] = key[i] ^ key[i+2*sha1.Size] ^ key[i+3*sha1.Size]
		}
	} else {
		for i := 0; i < sha1.Size; i++ {
			short[i] = key[i] ^ key[i+sha1.Size] ^ key[i+2*sha1.Size] ^ key[i+3*sha1.Size]
		}
	}
	return short[:]
}
<commit_msg>sources fix<commit_after>package core

import (
	"bytes"
	"crypto/sha1"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"time"
)

// Root of the repository
var RepoRoot string

// Initial working directory
var initialWorkingDir string

// Initial subdir of the working directory, ie. what package did we start in.
var initialPackage string

const DirPermissions = os.ModeDir | 0775

// FindRepoRoot returns the root directory of the current repo and sets the initial working dir.
// Dies on failure if 'die' is set.
func FindRepoRoot(die bool) {
	initialWorkingDir, _ = os.Getwd()
	RepoRoot, initialPackage = getRepoRoot(die)
}

// getRepoRoot returns the root directory of the current repo and the initial package.
func getRepoRoot(die bool) (string, string) {
	dir, err := os.Getwd()
	if err != nil {
		log.Fatalf("Couldn't determine working directory: %s", err)
	}
	// Walk up directories looking for a .plzconfig file, which we use to identify the root.
	initial := dir
	for dir != "" {
		if PathExists(path.Join(dir, ConfigFileName)) {
			return dir, strings.TrimLeft(initial[len(dir):], "/")
		}
		dir, _ = path.Split(dir)
		dir = strings.TrimRight(dir, "/")
	}
	if die {
		log.Fatalf("Couldn't locate the repo root. Are you sure you're inside a plz repo?")
	}
	return "", ""
}

// Returns true if the build was initiated from the repo root.
// Used to provide slightly nicer output in some places.
func StartedAtRepoRoot() bool {
	return RepoRoot == initialWorkingDir
}

func PathExists(filename string) bool {
	_, err := os.Stat(filename)
	return err == nil
}

func FileExists(filename string) bool {
	info, err := os.Stat(filename)
	return err == nil && !info.IsDir()
}

// CopyFile copies a file from 'from' to 'to', with an attempt to perform a copy & rename
// to avoid chaos if anything goes wrong partway.
func CopyFile(from string, to string, mode os.FileMode) error {
	fromFile, err := os.Open(from)
	if err != nil {
		return err
	}
	defer fromFile.Close()
	return WriteFile(fromFile, to, mode)
}

// WriteFile writes data from a reader to the file named 'to', with an attempt to perform
// a copy & rename to avoid chaos if anything goes wrong partway.
func WriteFile(fromFile io.Reader, to string, mode os.FileMode) error {
	if err := os.RemoveAll(to); err != nil {
		return err
	}
	dir, file := path.Split(to)
	if err := os.MkdirAll(dir, DirPermissions); err != nil {
		return err
	}
	tempFile, err := ioutil.TempFile(dir, file)
	if err != nil {
		return err
	}
	if _, err := io.Copy(tempFile, fromFile); err != nil {
		return err
	}
	if err := tempFile.Close(); err != nil {
		return err
	}
	// OK, now file is written; adjust permissions appropriately.
	if mode == 0 {
		mode = 0664
	}
	if err := os.Chmod(tempFile.Name(), mode); err != nil {
		return err
	}
	// And move it to its final destination.
	return os.Rename(tempFile.Name(), to)
}

// Copies either a single file or a directory.
// If 'link' is true then we'll attempt to hardlink files instead of copying them
// (on failure we still attempt a copy).
func RecursiveCopyFile(from string, to string, mode os.FileMode, link bool) error {
	if info, err := os.Stat(from); err == nil && info.IsDir() {
		return filepath.Walk(from, func(name string, info os.FileInfo, err error) error {
			dest := path.Join(to, name[len(from):])
			if err != nil {
				return err
			} else if info.IsDir() {
				return os.MkdirAll(dest, DirPermissions)
			} else if (info.Mode() & os.ModeSymlink) != 0 {
				fi, err := os.Stat(name)
				if err != nil {
					return err
				}
				if fi.IsDir() {
					return RecursiveCopyFile(name+"/", dest+"/", mode, link)
				} else {
					return copyOrLinkFile(name, dest, mode, link)
				}
			} else {
				return copyOrLinkFile(name, dest, mode, link)
			}
		})
	} else {
		return copyOrLinkFile(from, to, mode, link)
	}
}

// Either copies or hardlinks a file based on the link argument.
func copyOrLinkFile(from, to string, mode os.FileMode, link bool) error {
	if link {
		if err := os.Link(from, to); err == nil {
			return nil
		}
	}
	return CopyFile(from, to, mode)
}

// TimeoutError is the type of error that's issued when a command times out.
type TimeoutError int

// Error formats this error as a string.
func (i TimeoutError) Error() string {
	return fmt.Sprintf("Timeout (%d seconds) exceeded", i)
}

// ExecWithTimeout runs an external command with a timeout.
// If the command times out the returned error will be a TimeoutError.
func ExecWithTimeout(cmd *exec.Cmd, timeout int, defaultTimeout int) ([]byte, error) {
	var out bytes.Buffer
	if timeout == 0 {
		timeout = defaultTimeout
	}
	cmd.Stdout = &out
	cmd.Stderr = &out
	ch := make(chan error)
	go func() { ch <- cmd.Run() }()
	select {
	case <-time.After(time.Duration(timeout) * time.Second):
		if err := cmd.Process.Kill(); err != nil {
			log.Errorf("Process %d could not be killed after exceeding timeout of %d seconds", cmd.Process.Pid, timeout)
		}
		return out.Bytes(), TimeoutError(timeout)
	case err := <-ch:
		return out.Bytes(), err
	}
}

type sourcePair struct{ Src, Tmp string }

// IterSources returns all the sources for a function, allowing for sources that are other rules
// and rules that require transitive dependencies.
// Yielded values are pairs of the original source location and its temporary location for this rule.
func IterSources(graph *BuildGraph, target *BuildTarget) <-chan sourcePair {
	ch := make(chan sourcePair)
	done := map[BuildLabel]bool{}
	donePaths := map[string]bool{}
	tmpDir := target.TmpDir()
	var inner func(dependency *BuildTarget)
	inner = func(dependency *BuildTarget) {
		if target == dependency {
			// This is the current build rule, so link its sources.
			for source := range dependency.AllSources() {
				fullPaths := source.FullPaths(graph)
				for i, sourcePath := range source.Paths(graph) {
					tmpPath := path.Join(tmpDir, sourcePath)
					ch <- sourcePair{fullPaths[i], tmpPath}
					donePaths[tmpPath] = true
				}
			}
		} else {
			// This is a dependency of the rule, so link its outputs.
			for _, dep := range dependency.Outputs() {
				depPath := path.Join(dependency.OutDir(), dep)
				tmpPath := path.Join(tmpDir, dependency.Label.PackageName, dep)
				if !donePaths[tmpPath] {
					ch <- sourcePair{depPath, tmpPath}
					donePaths[tmpPath] = true
				}
			}
			// Mark any label-type outputs as done.
			for _, out := range dependency.DeclaredOutputs() {
				if LooksLikeABuildLabel(out) {
					label := ParseBuildLabel(out, target.Label.PackageName)
					done[label] = true
				}
			}
		}
		done[dependency.Label] = true
		if target == dependency || (target.NeedsTransitiveDependencies && !dependency.OutputIsComplete) {
			for _, dep := range dependency.Dependencies() {
				if !done[dep.Label] && !target.IsTool(dep.Label) {
					inner(dep)
				}
			}
		} else {
			for _, dep := range dependency.ExportedDependencies() {
				for _, dep2 := range recursivelyProvideFor(graph, target, dep) {
					if !done[dep2] {
						inner(graph.TargetOrDie(dep2))
					}
				}
			}
		}
	}
	go func() {
		inner(target)
		close(ch)
	}()
	return ch
}

// recursivelyProvideFor recursively applies ProvideFor to a target.
func recursivelyProvideFor(graph *BuildGraph, target *BuildTarget, dep BuildLabel) []BuildLabel {
	ret := graph.TargetOrDie(dep).ProvideFor(target)
	if len(ret) == 1 && ret[0] == dep {
		return ret // Providing itself, don't recurse
	}
	ret2 := []BuildLabel{}
	for _, r := range ret {
		ret2 = append(ret2, recursivelyProvideFor(graph, target, r)...)
	}
	return ret2
}

// Yields all the runtime files for a rule (outputs & data files), similar to above.
func IterRuntimeFiles(graph *BuildGraph, target *BuildTarget, absoluteOuts bool) <-chan sourcePair {
	done := map[string]bool{}
	ch := make(chan sourcePair)

	makeOut := func(out string) string {
		if absoluteOuts {
			return path.Join(RepoRoot, target.TestDir(), out)
		} else {
			return out
		}
	}

	pushOut := func(src, out string) {
		out = makeOut(out)
		if !done[out] {
			ch <- sourcePair{src, out}
			done[out] = true
		}
	}

	var inner func(*BuildTarget)
	inner = func(target *BuildTarget) {
		for _, out := range target.Outputs() {
			pushOut(path.Join(target.OutDir(), out), out)
		}
		for _, data := range target.Data {
			fullPaths := data.FullPaths(graph)
			for i, dataPath := range data.Paths(graph) {
				pushOut(fullPaths[i], dataPath)
			}
			if label := data.Label(); label != nil {
				for _, dep := range graph.TargetOrDie(*label).ExportedDependencies() {
					inner(graph.TargetOrDie(dep))
				}
			}
		}
		for _, dep := range target.ExportedDependencies() {
			inner(graph.TargetOrDie(dep))
		}
	}
	go func() {
		inner(target)
		close(ch)
	}()
	return ch
}

// Yields all the transitive input files for a rule (sources & data files), similar to above (again).
func IterInputPaths(graph *BuildGraph, target *BuildTarget) <-chan string {
	// Use a couple of maps to protect us from dep-graph loops and to stop parsing the same target
	// multiple times. We also only want to push files to the channel that it has not already seen.
	donePaths := map[string]bool{}
	doneTargets := map[*BuildTarget]bool{}
	ch := make(chan string)
	var inner func(*BuildTarget)
	inner = func(target *BuildTarget) {

		if !doneTargets[target] {
			// First yield all the sources of the target only ever pushing declared paths to
			// the channel to prevent us outputting any intermediate files.
			for source := range target.AllSources() {
				// If the label is nil add any input paths contained here.
				if label := source.Label(); label == nil {
					for _, sourcePath := range source.FullPaths(graph) {
						if !donePaths[sourcePath] {
							ch <- sourcePath
							donePaths[sourcePath] = true
						}
					}
					// Otherwise we should recurse for this build label (and gather its sources)
				} else {
					inner(graph.TargetOrDie(*label))
				}
			}

			// Now yield all the data deps of this rule.
			for _, data := range target.Data {
				// If the label is nil add any input paths contained here.
				if label := data.Label(); label == nil {
					for _, sourcePath := range data.FullPaths(graph) {
						if !donePaths[sourcePath] {
							ch <- sourcePath
							donePaths[sourcePath] = true
						}
					}
					// Otherwise we should recurse for this build label (and gather its sources)
				} else {
					inner(graph.TargetOrDie(*label))
				}
			}

			// Finally recurse for all the deps of this rule.
			for _, dep := range target.Dependencies() {
				inner(dep)
			}
			doneTargets[target] = true
		}
	}
	go func() {
		inner(target)
		close(ch)
	}()
	return ch
}

// Symlinks a single source file for a build rule.
func PrepareSource(sourcePath string, tmpPath string) error {
	dir := path.Dir(tmpPath)
	if !PathExists(dir) {
		if err := os.MkdirAll(dir, DirPermissions); err != nil {
			return err
		}
	}
	if !PathExists(sourcePath) {
		return fmt.Errorf("Source file %s doesn't exist", sourcePath)
	}
	return RecursiveCopyFile(sourcePath, tmpPath, 0, true)
}

func PrepareSourcePair(pair sourcePair) error {
	if path.IsAbs(pair.Src) {
		return PrepareSource(pair.Src, pair.Tmp)
	}
	return PrepareSource(path.Join(RepoRoot, pair.Src), pair.Tmp)
}

func PostBuildOutputFileName(target *BuildTarget) string {
	return ".build_output_" + target.Label.Name
}

// CollapseHash combines our usual four-part hash into one by XOR'ing them together.
// This helps keep things short in places where sometimes we get complaints about filenames being too long (?)
// and where we don't especially care about breaking out the individual parts of hashes, which
// is important for many parts of the system.
func CollapseHash(key []byte) []byte {
	short := [sha1.Size]byte{}
	// We store the rule hash twice, if it's repeated we must make sure not to xor it
	// against itself.
	if bytes.Equal(key[0:sha1.Size], key[sha1.Size:2*sha1.Size]) {
		for i := 0; i < sha1.Size; i++ {
			short[i] = key[i] ^ key[i+2*sha1.Size] ^ key[i+3*sha1.Size]
		}
	} else {
		for i := 0; i < sha1.Size; i++ {
			short[i] = key[i] ^ key[i+sha1.Size] ^ key[i+2*sha1.Size] ^ key[i+3*sha1.Size]
		}
	}
	return short[:]
}
<|endoftext|>


<commit_before>// Binary main defines an mud server.
package main

import (
	"fmt"
	"mudlib"
	"os"
)

const (
	playerDir   = "players/"
	roomDir     = "rooms/"
	configFile  = "config"
	startRoomId = "start"
)

func main() {
	if err := mudlib.LoadPlayerDb(playerDir); err != nil {
		fmt.Println("Failed to load player db: %+v", err)
		os.Exit(1)
	}

	if err := mudlib.LoadRoomDb(roomDir, startRoomId); err != nil {
		fmt.Printf("Failed to load room db: %+v\n", err)
		os.Exit(1)
	}

	if err := mudlib.Run(configFile); err != nil {
		fmt.Printf("Failed to run: %+v\n", err)
		os.Exit(1)
	}
}
<commit_msg>main logging cleanup<commit_after>// Binary main defines an mud server.
package main

import (
	"fmt"
	"mudlib"
	"os"
)

const (
	playerDir   = "players/"
	roomDir     = "rooms/"
	configFile  = "config"
	startRoomId = "start"
)

func main() {
	if err := mudlib.LoadPlayerDb(playerDir); err != nil {
		fmt.Println("Failed to load player db: ", err)
		os.Exit(1)
	}

	if err := mudlib.LoadRoomDb(roomDir, startRoomId); err != nil {
		fmt.Printf("Failed to load room db: ", err)
		os.Exit(1)
	}

	if err := mudlib.Run(configFile); err != nil {
		fmt.Printf("Failed to run: %+v\n", err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>// BaruwaAPI Golang bindings for Baruwa REST API
// Copyright (C) 2019 Andrew Colin Kissa <andrew@topdog.za.net>

// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.

// Package api Golang bindings for Baruwa REST API
package api

// DomainDeliveryServer holds domain delivery servers
type DomainDeliveryServer struct {
	ID               int    `json:"id,omitempty"`
	Address          string `json:"address"`
	Protocol         int    `json:"protocol"`
	Port             int    `json:"port"`
	RequireTLS       bool   `json:"require_tls"`
	VerificationOnly bool   `json:"verification_only"`
	Enabled          bool   `json:"enabled"`
}

// GetDomainDeliveryServer returns a domain delivery server
// https://www.baruwa.com/docs/api/#retrieve-a-delivery-server
func (c *Client) GetDomainDeliveryServer(id int) (server *DomainDeliveryServer, err error) {
	return
}

// CreateDomainDeliveryServer creates a domain delivery server
// https://www.baruwa.com/docs/api/#create-a-delivery-server
func (c *Client) CreateDomainDeliveryServer(server *DomainDeliveryServer) (err error) {
	return
}

// UpdateDomainDeliveryServer updates a domain delivery server
// https://www.baruwa.com/docs/api/#update-a-delivery-server
func (c *Client) UpdateDomainDeliveryServer(server *DomainDeliveryServer) (err error) {
	return
}

// DeleteDomainDeliveryServer deletes a domain delivery server
// https://www.baruwa.com/docs/api/#delete-a-delivery-server
func (c *Client) DeleteDomainDeliveryServer(id int) (err error) {
	return
}
<commit_msg>FET: Domain Delivery servers implementation<commit_after>// BaruwaAPI Golang bindings for Baruwa REST API
// Copyright (C) 2019 Andrew Colin Kissa <andrew@topdog.za.net>

// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.

// Package api Golang bindings for Baruwa REST API
package api

import (
	"fmt"
	"net/url"

	"github.com/google/go-querystring/query"
)

// DomainDeliveryServer holds domain delivery servers
type DomainDeliveryServer struct {
	ID               int    `json:"id,omitempty"`
	Address          string `json:"address"`
	Protocol         int    `json:"protocol"`
	Port             int    `json:"port"`
	RequireTLS       bool   `json:"require_tls"`
	VerificationOnly bool   `json:"verification_only"`
	Enabled          bool   `json:"enabled"`
}

// GetDomainDeliveryServer returns a domain delivery server
// https://www.baruwa.com/docs/api/#retrieve-a-delivery-server
func (c *Client) GetDomainDeliveryServer(domainid, serverid int) (server *DomainDeliveryServer, err error) {
	if domainid <= 0 {
		err = fmt.Errorf("The domainid param should be > 0")
		return
	}

	if serverid <= 0 {
		err = fmt.Errorf("The serverid param should be > 0")
		return
	}

	err = c.get(fmt.Sprintf("deliveryservers/%d/%d", domainid, serverid), server)

	return
}

// CreateDomainDeliveryServer creates a domain delivery server
// https://www.baruwa.com/docs/api/#create-a-delivery-server
func (c *Client) CreateDomainDeliveryServer(domainid int, server *DomainDeliveryServer) (err error) {
	var v url.Values

	if domainid <= 0 {
		err = fmt.Errorf("The domainid param should be > 0")
		return
	}

	if server == nil {
		err = fmt.Errorf("The server param cannot be nil")
		return
	}

	if v, err = query.Values(server); err != nil {
		return
	}

	err = c.post(fmt.Sprintf("deliveryservers/%d", domainid), v, server)

	return
}

// UpdateDomainDeliveryServer updates a domain delivery server
// https://www.baruwa.com/docs/api/#update-a-delivery-server
func (c *Client) UpdateDomainDeliveryServer(domainid int, server *DomainDeliveryServer) (err error) {
	var v url.Values

	if domainid <= 0 {
		err = fmt.Errorf("The domainid param should be > 0")
		return
	}

	if server == nil {
		err = fmt.Errorf("The server param cannot be nil")
		return
	}

	if server.ID <= 0 {
		err = fmt.Errorf("The server.ID param should be > 0")
		return
	}

	if v, err = query.Values(server); err != nil {
		return
	}

	err = c.post(fmt.Sprintf("deliveryservers/%d/%d", domainid, server.ID), v, server)

	return
}

// DeleteDomainDeliveryServer deletes a domain delivery server
// https://www.baruwa.com/docs/api/#delete-a-delivery-server
func (c *Client) DeleteDomainDeliveryServer(domainid int, server *DomainDeliveryServer) (err error) {
	var v url.Values

	if domainid <= 0 {
		err = fmt.Errorf("The domainid param should be > 0")
		return
	}

	if server == nil {
		err = fmt.Errorf("The server param cannot be nil")
		return
	}

	if server.ID <= 0 {
		err = fmt.Errorf("The server.ID param should be > 0")
		return
	}

	if v, err = query.Values(server); err != nil {
		return
	}

	err = c.delete(fmt.Sprintf("deliveryservers/%d/%d", domainid, server.ID), v)

	return
}
<|endoftext|>


<commit_before>package bot

import (
	"bytes"
	"fmt"
	"github.com/ChimeraCoder/anaconda"
	"github.com/dkua/go-sudoku"
	"regexp"
	"strconv"
	"strings"
)

func parseTimeline(timeline []anaconda.Tweet, tweets [][]string) [][]string {
	for _, tweet := range timeline {
		tweets = append(tweets, []string{strconv.FormatInt(tweet.Id, 10), parseTweet(tweet)})
	}
	return tweets
}

func parseTweet(tweet anaconda.Tweet) string {
	text, err := parseText(tweet.Text)
	if err != nil {
		return fmt.Sprintf("%v@%v", err, tweet.User.ScreenName)
	} else {
		return fmt.Sprintf("%v@%v", text, tweet.User.ScreenName)
	}
}

func parseText(tweet_text string) (string, error) {

	// Build puzzle from tweet, puzzle is either lines of 9 units separated by newlines
	// or a single 81 char string
	var buffer bytes.Buffer
	text_array := strings.Fields(tweet_text)
	for _, text := range text_array {
		if len(text) == 9 || len(text) == 81 {
			buffer.WriteString(text)
		}
	}

	// Check validity of the puzzle and return, else return error
	regex := regexp.MustCompile(`[0-9]*\.*`)
	puzzle := strings.Join(regex.FindAllString(buffer.String(), -1), "")
	if len(puzzle) == 81 {
		solution, err := sudoku.Solve(puzzle)
		if err != nil {
			return "", fmt.Errorf("Sorry couldn't solve this puzzle.")
		}
		return sudoku.Display(solution, err), nil
	}
	return "", fmt.Errorf("Sorry couldn't parse this tweet")
}
<commit_msg>Added missing space<commit_after>package bot

import (
	"bytes"
	"fmt"
	"github.com/ChimeraCoder/anaconda"
	"github.com/dkua/go-sudoku"
	"regexp"
	"strconv"
	"strings"
)

func parseTimeline(timeline []anaconda.Tweet, tweets [][]string) [][]string {
	for _, tweet := range timeline {
		tweets = append(tweets, []string{strconv.FormatInt(tweet.Id, 10), parseTweet(tweet)})
	}
	return tweets
}

func parseTweet(tweet anaconda.Tweet) string {
	text, err := parseText(tweet.Text)
	if err != nil {
		return fmt.Sprintf("%v @%v", err, tweet.User.ScreenName)
	} else {
		return fmt.Sprintf("%v@%v", text, tweet.User.ScreenName)
	}
}

func parseText(tweet_text string) (string, error) {

	// Build puzzle from tweet, puzzle is either lines of 9 units separated by newlines
	// or a single 81 char string
	var buffer bytes.Buffer
	text_array := strings.Fields(tweet_text)
	for _, text := range text_array {
		if len(text) == 9 || len(text) == 81 {
			buffer.WriteString(text)
		}
	}

	// Check validity of the puzzle and return, else return error
	regex := regexp.MustCompile(`[0-9]*\.*`)
	puzzle := strings.Join(regex.FindAllString(buffer.String(), -1), "")
	if len(puzzle) == 81 {
		solution, err := sudoku.Solve(puzzle)
		if err != nil {
			return "", fmt.Errorf("Sorry couldn't solve this puzzle.")
		}
		return sudoku.Display(solution, err), nil
	}
	return "", fmt.Errorf("Sorry couldn't parse this tweet")
}
<|endoftext|>


<commit_before>package search

import (
	"github.com/ewhal/nyaa/model"
	"github.com/ewhal/nyaa/service/torrent"
	"github.com/ewhal/nyaa/util/log"
	"html"
	"net/http"
	"strconv"
	"strings"
	"unicode"
	"unicode/utf8"
)

type SearchParam struct {
	Category string
	Order    string
	Query    string
	Max      int
	Status   string
	Sort     string
}

func SearchByQuery(r *http.Request, pagenum int) (SearchParam, []model.Torrents, int) {
	maxPerPage, errConv := strconv.Atoi(r.URL.Query().Get("max"))
	if errConv != nil {
		maxPerPage = 50 // default Value maxPerPage
	}

	if maxPerPage > 300 {
		maxPerPage = 300
	}
	search_param := SearchParam{}
	search_param.Max = maxPerPage
	search_param.Query = r.URL.Query().Get("q")
	search_param.Category = r.URL.Query().Get("c")
	search_param.Status = r.URL.Query().Get("s")
	search_param.Sort = r.URL.Query().Get("sort")
	search_param.Order = r.URL.Query().Get("order")

	catsSplit := strings.Split(search_param.Category, "_")
	// need this to prevent out of index panics
	var searchCatId, searchSubCatId string
	if len(catsSplit) == 2 {

		searchCatId = html.EscapeString(catsSplit[0])
		searchSubCatId = html.EscapeString(catsSplit[1])
	}

	switch search_param.Sort {
	case "torrent_name":
		search_param.Sort = "torrent_name"
		break
	case "date":
		search_param.Sort = "date"
		break
	case "downloads":
		search_param.Sort = "downloads"
		break
	case "filesize":
		search_param.Sort = "filesize"
	case "torrent_id":
	default:
		search_param.Sort = "torrent_id"
	}

	switch search_param.Order {
	case "asc":
		search_param.Order = "asc"
		break
	case "desc":
	default:
		search_param.Order = "desc"
	}

	order_by := search_param.Sort + " " + search_param.Order

	parameters := torrentService.WhereParams{}
	conditions := []string{}
	if searchCatId != "" {
		conditions = append(conditions, "category_id = ?")
		parameters.Params = append(parameters.Params, searchCatId)
	}
	if searchSubCatId != "" {
		conditions = append(conditions, "sub_category_id = ?")
		parameters.Params = append(parameters.Params, searchSubCatId)
	}
	if search_param.Status != "" {
		if search_param.Status == "2" {
			conditions = append(conditions, "status_id != ?")
		} else {
			conditions = append(conditions, "status_id = ?")
		}
		parameters.Params = append(parameters.Params, search_param.Status)
	}
	searchQuerySplit := strings.Fields(search_param.Query)
	for i, word := range searchQuerySplit {
		firstRune, _ := utf8.DecodeRuneInString(word)
		if len(word) == 1 && unicode.IsPunct(firstRune) {
			// some queries have a single punctuation character
			// which causes a full scan instead of using the index
			// and yields no meaningful results.
			// due to len() == 1 we're just looking at 1-byte/ascii
			// punctuation characters.
			continue
		}
		// TODO: make this faster ?
		conditions = append(conditions, "torrent_name ILIKE ?")
		parameters.Params = append(parameters.Params, "%"+searchQuerySplit[i]+"%")
	}

	parameters.Conditions = strings.Join(conditions[:], " AND ")
	log.Infof("SQL query is :: %s\n", parameters.Conditions)
	torrents, n := torrentService.GetTorrentsOrderBy(&parameters, order_by, maxPerPage, maxPerPage*(pagenum-1))
	return search_param, torrents, n
}
<commit_msg>category and status search not workign<commit_after>package search

import (
	"github.com/ewhal/nyaa/model"
	"github.com/ewhal/nyaa/service/torrent"
	"github.com/ewhal/nyaa/util/log"
	"html"
	"net/http"
	"strconv"
	"strings"
	"unicode"
	"unicode/utf8"
)

type SearchParam struct {
	Category string
	Order    string
	Query    string
	Max      int
	Status   string
	Sort     string
}

func SearchByQuery(r *http.Request, pagenum int) (SearchParam, []model.Torrents, int) {
	maxPerPage, errConv := strconv.Atoi(r.URL.Query().Get("max"))
	if errConv != nil {
		maxPerPage = 50 // default Value maxPerPage
	}

	if maxPerPage > 300 {
		maxPerPage = 300
	}
	search_param := SearchParam{}
	search_param.Max = maxPerPage
	search_param.Query = r.URL.Query().Get("q")
	search_param.Category = r.URL.Query().Get("c")
	search_param.Status = r.URL.Query().Get("s")
	search_param.Sort = r.URL.Query().Get("sort")
	search_param.Order = r.URL.Query().Get("order")

	catsSplit := strings.Split(search_param.Category, "_")
	// need this to prevent out of index panics
	var searchCatId, searchSubCatId string
	if len(catsSplit) == 2 {

		searchCatId = html.EscapeString(catsSplit[0])
		searchSubCatId = html.EscapeString(catsSplit[1])
	}

	switch search_param.Sort {
	case "torrent_name":
		search_param.Sort = "torrent_name"
		break
	case "date":
		search_param.Sort = "date"
		break
	case "downloads":
		search_param.Sort = "downloads"
		break
	case "filesize":
		search_param.Sort = "filesize"
	case "torrent_id":
	default:
		search_param.Sort = "torrent_id"
	}

	switch search_param.Order {
	case "asc":
		search_param.Order = "asc"
		break
	case "desc":
	default:
		search_param.Order = "desc"
	}

	order_by := search_param.Sort + " " + search_param.Order

	parameters := torrentService.WhereParams{}
	conditions := []string{}
	if searchCatId != "" {
		conditions = append(conditions, "category = ?")
		parameters.Params = append(parameters.Params, searchCatId)
	}
	if searchSubCatId != "" {
		conditions = append(conditions, "sub_category = ?")
		parameters.Params = append(parameters.Params, searchSubCatId)
	}
	if search_param.Status != "" {
		if search_param.Status == "2" {
			conditions = append(conditions, "status != ?")
		} else {
			conditions = append(conditions, "status = ?")
		}
		parameters.Params = append(parameters.Params, search_param.Status)
	}
	searchQuerySplit := strings.Fields(search_param.Query)
	for i, word := range searchQuerySplit {
		firstRune, _ := utf8.DecodeRuneInString(word)
		if len(word) == 1 && unicode.IsPunct(firstRune) {
			// some queries have a single punctuation character
			// which causes a full scan instead of using the index
			// and yields no meaningful results.
			// due to len() == 1 we're just looking at 1-byte/ascii
			// punctuation characters.
			continue
		}
		// TODO: make this faster ?
		conditions = append(conditions, "torrent_name ILIKE ?")
		parameters.Params = append(parameters.Params, "%"+searchQuerySplit[i]+"%")
	}

	parameters.Conditions = strings.Join(conditions[:], " AND ")
	log.Infof("SQL query is :: %s\n", parameters.Conditions)
	torrents, n := torrentService.GetTorrentsOrderBy(&parameters, order_by, maxPerPage, maxPerPage*(pagenum-1))
	return search_param, torrents, n
}
<|endoftext|>


<commit_before>package index

import (
	"bytes"
	"database/sql"
	"encoding/xml"
	"html/template"
	"log"
	"net/http"
	"strconv"

	_ "github.com/mattn/go-sqlite3" // use sqlite
	"github.com/xoriath/alexandria-go/types"

	"errors"
	"os"
)

// Store is the store for keywords
type Store struct {
	FileName string
	handle   *sql.DB

	prefix string
	ext    string

	indexWriteChan     chan *types.Indexes
	f1FragmentTemplate *template.Template
}

// NewStore creates a index store
func NewStore(prefix, ext, f1FragmentPattern string) *Store {
	store := &Store{
		prefix:             prefix,
		ext:                ext,
		f1FragmentTemplate: template.Must(template.New("").Parse(f1FragmentPattern))}

	store.prepareDb()

	store.indexWriteChan = store.insertIndexes()

	return store
}

// OldStore open a already existing index store
func OldStore(file, f1FragmentPattern string) *Store {
	store := &Store{
		FileName:           file,
		f1FragmentTemplate: template.Must(template.New("").Parse(f1FragmentPattern))}

	store.prepareDb()

	return store
}

func (i *Store) getDbFile(prefix, ext string) (string, error) {

	if i.FileName == "" {

		for j := 0; j < 10000; j++ {
			filename := "./" + prefix + "-" + strconv.Itoa(j) + "." + ext
			if _, err := os.Stat(filename); os.IsNotExist(err) {
				i.FileName = filename
				return filename, nil
			}
		}

		return "", errors.New("Failed to find free file for db")
	}

	return i.FileName, nil
}

func (i *Store) getDb() *sql.DB {

	if i.handle == nil {

		if i.prefix != "" && i.ext != "" {
			fileName, err := i.getDbFile(i.prefix, i.ext)
			if err != nil {
				panic(err)
			}

			i.FileName = fileName
		}

		handle, err := sql.Open("sqlite3", i.FileName)
		if err != nil {
			panic(err)
		}

		if err = handle.Ping(); err != nil {
			panic(err)
		}

		i.handle = handle
	}

	return i.handle
}

func (i *Store) prepareDb() {
	db := i.getDb()

	createTableStmt := `
		CREATE TABLE IF NOT EXISTS files (
			file		INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
			book 		VARCHAR(49) NOT NULL,
			filename	VARCHAR(46) NOT NULL,

			CONSTRAINT file_constraint UNIQUE (book, filename)
		);

		CREATE TABLE IF NOT EXISTS keywords (
			keyword 	TEXT NOT NULL COLLATE NOCASE,
			file		INTEGER NOT NULL,

			FOREIGN KEY(file) REFERENCES files(file)

			-- CONSTRAINT keywords_constraint UNIQUE(keyword, file)
		);

		CREATE INDEX IF NOT EXISTS keywords_index ON keywords(keyword);
		`

	_, err := db.Exec(createTableStmt)

	if err != nil {
		panic(err)
	}
}

func (i *Store) findID(tx *sql.Tx, bookID, file string) (id int64) {

	stmt, _ := tx.Prepare("SELECT file FROM files WHERE book = '?' AND filename = '?'")

	rows, err := tx.Stmt(stmt).Query(bookID, file)

	if err != nil {
		panic(err)
	}

	defer rows.Close()

	for rows.Next() {
		err := rows.Scan(&id)

		if err != nil {
			panic(err)
		}

		break
	}

	log.Println("Fetched existing id", id)

	return
}

func (i *Store) insertIndexes() chan *types.Indexes {

	ch := make(chan *types.Indexes)

	go func() {

		db := i.getDb()

		bookStmt, err := db.Prepare("INSERT INTO files(book, filename) values(?, ?)")
		if err != nil {
			panic(err)
		}

		keywordStmt, err := db.Prepare("INSERT INTO keywords(keyword, file) values(?, ?)")
		if err != nil {
			panic(err)
		}

		idMap := make(map[string]int64)

		for {
			indexes := <-ch

			tx, err := db.Begin()
			if err != nil {
				panic(err)
			}

			for _, index := range indexes.Keywords {

				// Check if we have this id already
				id := idMap[indexes.BookID+index.File]
				if id == 0 {
					// Try to insert, will fail if exists due to unique constraint
					res, err := tx.Stmt(bookStmt).Exec(indexes.BookID, index.File)
					if err != nil {
						// unique failed, time to search
						id = i.findID(tx, indexes.BookID, index.File)
					} else {
						// fetch id that we just inserted
						id, err = res.LastInsertId()
						if err != nil {
							tx.Rollback()
							panic(err)
						}
					}
				}

				idMap[indexes.BookID+index.File] = id

				_, err := tx.Stmt(keywordStmt).Exec(index.Keyword, id)
				if err != nil {
					//tx.Rollback()
					log.Fatalln("Panic for", indexes.BookID, id)
					log.Fatalf("%+v\n", err)
					//panic(err)
				}
			}

			tx.Commit()
		}
	}()

	return ch
}

// FetchIndex downloads the index file, decodes it and requests it to be added to the store
func (i *Store) FetchIndex(book *types.Book) {

	parts := map[string]string{"Id": book.ID, "Language": book.Language, "Version": book.Version}
	var url bytes.Buffer
	i.f1FragmentTemplate.Execute(&url, parts)

	resp, err := http.Get(url.String())
	if err != nil {
		panic(err)
	}

	defer resp.Body.Close()

	decoder := xml.NewDecoder(resp.Body)
	indexes := new(types.Indexes)
	indexes.Etag = resp.Header.Get("Etag")

	err = decoder.Decode(indexes)
	if err != nil {
		log.Panic("Failed to parse", url)
		panic(err)
	}

	if i.indexWriteChan == nil {
		i.indexWriteChan = i.insertIndexes()
	}

	i.indexWriteChan <- indexes
}

// KeywordResult is a single result, pointing to a BookID and a Filename inside this book which has the keyword
type KeywordResult struct {
	BookID   string
	Filename string
}

// LookupKeyword looks up the keyword in the store, and returns a slice of KeywordResult
func (i *Store) LookupKeyword(keyword string) []KeywordResult {
	stmt, err := i.handle.Prepare(`
		SELECT files.book, files.filename
		FROM keywords
		INNER JOIN files
		ON keywords.file = files.file
		WHERE keywords.keyword = ? COLLATE NOCASE`)

	if err != nil {
		panic(err)
	}

	rows, err := stmt.Query(keyword)
	if err != nil {
		panic(err)
	}

	defer rows.Close()

	var results []KeywordResult

	for rows.Next() {
		result := KeywordResult{}
		rows.Scan(&result.BookID, &result.Filename)

		results = append(results, result)
	}

	return results
}

// DatabaseStatistics contains some small statistic numbers from the amount of content in the system
type DatabaseStatistics struct {
	NumberOfFiles int `json:"number_of_files"`
	KeywordCount  int `json:"number_of_keywords"`
}

// GetStatistics reads the statistics from the db storage and returns them
func (i *Store) GetStatistics() DatabaseStatistics {
	fileCount := i.getFileCount()
	keywordCount := i.getKeywordCount()

	return DatabaseStatistics{NumberOfFiles: fileCount, KeywordCount: keywordCount}
}

func (i *Store) getFileCount() (count int) {
	rows, err := i.handle.Query(`
		SELECT COUNT()
		as count
		FROM files`)
	if err != nil {
		panic(err)
	}

	defer rows.Close()

	for rows.Next() {
		err := rows.Scan(&count)
		if err != nil {
			panic(err)
		} else {
			return
		}
	}

	return
}

func (i *Store) getKeywordCount() (count int) {
	rows, err := i.handle.Query(`
		SELECT COUNT()
		as count
		FROM keywords`)
	if err != nil {
		panic(err)
	}

	defer rows.Close()

	for rows.Next() {
		err := rows.Scan(&count)
		if err != nil {
			panic(err)
		} else {
			return
		}
	}

	return
}

func (i *Store) getKeywords() (keywords []string) {
	rows, err := i.handle.Query(`
		SELECT keyword
		FROM keywords
		ORDER BY keyword`)

	if err != nil {
		panic(err)
	}

	defer rows.Close()

	for rows.Next() {
		var keyword string

		err := rows.Scan(&keyword)
		if err != nil {
			panic(err)
		}

		keywords = append(keywords, keyword)
	}

	return
}
<commit_msg>Fix error when preparing keyword statement<commit_after>package index

import (
	"bytes"
	"database/sql"
	"encoding/xml"
	"html/template"
	"log"
	"net/http"
	"strconv"

	_ "github.com/mattn/go-sqlite3" // use sqlite
	"github.com/xoriath/alexandria-go/types"

	"errors"
	"os"
)

// Store is the store for keywords
type Store struct {
	FileName string
	handle   *sql.DB

	prefix string
	ext    string

	indexWriteChan     chan *types.Indexes
	f1FragmentTemplate *template.Template
}

// NewStore creates a index store
func NewStore(prefix, ext, f1FragmentPattern string) *Store {
	store := &Store{
		prefix:             prefix,
		ext:                ext,
		f1FragmentTemplate: template.Must(template.New("").Parse(f1FragmentPattern))}

	store.prepareDb()

	store.indexWriteChan = store.insertIndexes()

	return store
}

// OldStore open a already existing index store
func OldStore(file, f1FragmentPattern string) *Store {
	store := &Store{
		FileName:           file,
		f1FragmentTemplate: template.Must(template.New("").Parse(f1FragmentPattern))}

	store.prepareDb()

	return store
}

func (i *Store) getDbFile(prefix, ext string) (string, error) {

	if i.FileName == "" {

		for j := 0; j < 10000; j++ {
			filename := "./" + prefix + "-" + strconv.Itoa(j) + "." + ext
			if _, err := os.Stat(filename); os.IsNotExist(err) {
				i.FileName = filename
				return filename, nil
			}
		}

		return "", errors.New("Failed to find free file for db")
	}

	return i.FileName, nil
}

func (i *Store) getDb() *sql.DB {

	if i.handle == nil {

		if i.prefix != "" && i.ext != "" {
			fileName, err := i.getDbFile(i.prefix, i.ext)
			if err != nil {
				panic(err)
			}

			i.FileName = fileName
		}

		handle, err := sql.Open("sqlite3", i.FileName)
		if err != nil {
			panic(err)
		}

		if err = handle.Ping(); err != nil {
			panic(err)
		}

		i.handle = handle
	}

	return i.handle
}

func (i *Store) prepareDb() {
	db := i.getDb()

	createTableStmt := `
		CREATE TABLE IF NOT EXISTS files (
			file		INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
			book 		VARCHAR(49) NOT NULL,
			filename	VARCHAR(46) NOT NULL,

			CONSTRAINT file_constraint UNIQUE (book, filename)
		);

		CREATE TABLE IF NOT EXISTS keywords (
			keyword 	TEXT NOT NULL COLLATE NOCASE,
			file		INTEGER NOT NULL,

			FOREIGN KEY(file) REFERENCES files(file)

			-- CONSTRAINT keywords_constraint UNIQUE(keyword, file)
		);

		CREATE INDEX IF NOT EXISTS keywords_index ON keywords(keyword);
		`

	_, err := db.Exec(createTableStmt)

	if err != nil {
		panic(err)
	}
}

func (i *Store) findID(tx *sql.Tx, bookID, file string) (id int64) {

	stmt, _ := tx.Prepare("SELECT file FROM files WHERE book = '?' AND filename = '?'")

	rows, err := tx.Stmt(stmt).Query(bookID, file)

	if err != nil {
		panic(err)
	}

	defer rows.Close()

	for rows.Next() {
		err := rows.Scan(&id)

		if err != nil {
			panic(err)
		}

		break
	}

	log.Println("Fetched existing id", id)

	return
}

func (i *Store) insertIndexes() chan *types.Indexes {

	ch := make(chan *types.Indexes)

	go func() {

		db := i.getDb()

		bookStmt, err := db.Prepare("INSERT INTO files(book, filename) values(?, ?)")
		if err != nil {
			panic(err)
		}

		keywordStmt, err := db.Prepare("INSERT INTO keywords(keyword, file) values(?, ?)")
		if err != nil {
			panic(err)
		}

		idMap := make(map[string]int64)

		for {
			indexes := <-ch

			tx, err := db.Begin()
			if err != nil {
				panic(err)
			}

			for _, index := range indexes.Keywords {

				// Check if we have this id already
				id := idMap[indexes.BookID+index.File]
				if id == 0 {
					// Try to insert, will fail if exists due to unique constraint
					res, err := tx.Stmt(bookStmt).Exec(indexes.BookID, index.File)
					if err != nil {
						// unique failed, time to search
						id = i.findID(tx, indexes.BookID, index.File)
					} else {
						// fetch id that we just inserted
						id, err = res.LastInsertId()
						if err != nil {
							tx.Rollback()
							panic(err)
						}
					}
				}

				idMap[indexes.BookID+index.File] = id

				_, err := tx.Stmt(keywordStmt).Exec(index.Keyword, id)
				if err != nil {
					//tx.Rollback()
					log.Fatalln("Panic for", indexes.BookID, id)
					log.Fatalf("%+v\n", err)
					//panic(err)
				}
			}

			tx.Commit()
		}
	}()

	return ch
}

// FetchIndex downloads the index file, decodes it and requests it to be added to the store
func (i *Store) FetchIndex(book *types.Book) {

	parts := map[string]string{"Id": book.ID, "Language": book.Language, "Version": book.Version}
	var url bytes.Buffer
	i.f1FragmentTemplate.Execute(&url, parts)

	resp, err := http.Get(url.String())
	if err != nil {
		panic(err)
	}

	defer resp.Body.Close()

	decoder := xml.NewDecoder(resp.Body)
	indexes := new(types.Indexes)
	indexes.Etag = resp.Header.Get("Etag")

	err = decoder.Decode(indexes)
	if err != nil {
		log.Panic("Failed to parse", url)
		panic(err)
	}

	if i.indexWriteChan == nil {
		i.indexWriteChan = i.insertIndexes()
	}

	i.indexWriteChan <- indexes
}

// KeywordResult is a single result, pointing to a BookID and a Filename inside this book which has the keyword
type KeywordResult struct {
	BookID   string
	Filename string
}

// LookupKeyword looks up the keyword in the store, and returns a slice of KeywordResult
func (i *Store) LookupKeyword(keyword string) []KeywordResult {
	stmt, err := i.handle.Prepare(`
		SELECT files.book, files.filename
		FROM keywords
		INNER JOIN files
		ON keywords.file = files.file
		WHERE keywords.keyword = '?' COLLATE NOCASE`)

	if err != nil {
		panic(err)
	}

	rows, err := stmt.Query(keyword)
	if err != nil {
		panic(err)
	}

	defer rows.Close()

	var results []KeywordResult

	for rows.Next() {
		result := KeywordResult{}
		rows.Scan(&result.BookID, &result.Filename)

		results = append(results, result)
	}

	return results
}

// DatabaseStatistics contains some small statistic numbers from the amount of content in the system
type DatabaseStatistics struct {
	NumberOfFiles int `json:"number_of_files"`
	KeywordCount  int `json:"number_of_keywords"`
}

// GetStatistics reads the statistics from the db storage and returns them
func (i *Store) GetStatistics() DatabaseStatistics {
	fileCount := i.getFileCount()
	keywordCount := i.getKeywordCount()

	return DatabaseStatistics{NumberOfFiles: fileCount, KeywordCount: keywordCount}
}

func (i *Store) getFileCount() (count int) {
	rows, err := i.handle.Query(`
		SELECT COUNT()
		as count
		FROM files`)
	if err != nil {
		panic(err)
	}

	defer rows.Close()

	for rows.Next() {
		err := rows.Scan(&count)
		if err != nil {
			panic(err)
		} else {
			return
		}
	}

	return
}

func (i *Store) getKeywordCount() (count int) {
	rows, err := i.handle.Query(`
		SELECT COUNT()
		as count
		FROM keywords`)
	if err != nil {
		panic(err)
	}

	defer rows.Close()

	for rows.Next() {
		err := rows.Scan(&count)
		if err != nil {
			panic(err)
		} else {
			return
		}
	}

	return
}

func (i *Store) getKeywords() (keywords []string) {
	rows, err := i.handle.Query(`
		SELECT keyword
		FROM keywords
		ORDER BY keyword`)

	if err != nil {
		panic(err)
	}

	defer rows.Close()

	for rows.Next() {
		var keyword string

		err := rows.Scan(&keyword)
		if err != nil {
			panic(err)
		}

		keywords = append(keywords, keyword)
	}

	return
}
<|endoftext|>


<commit_before>package assets

import (
	"fmt"
	"path/filepath"
	"regexp"
)

type CdnMap map[*regexp.Regexp]string

var cdnMap = CdnMap{
	regexp.MustCompile("angular-([\\d\\.]+\\d)"):            "//ajax.googleapis.com/ajax/libs/angularjs/$1/angular.min.js",
	regexp.MustCompile("CFInstall-([\\d\\.]+\\d)"):          "//ajax.googleapis.com/ajax/libs/chrome-frame/$1/CFInstall.min.js",
	regexp.MustCompile("dojo-([\\d\\.]+\\d)"):               "//ajax.googleapis.com/ajax/libs/dojo/$1/dojo/dojo.js",
	regexp.MustCompile("ext-core-([\\d\\.]+\\d)"):           "//ajax.googleapis.com/ajax/libs/ext-core/$1/ext-core.js",
	regexp.MustCompile("jquery-([\\d\\.]+\\d)"):             "//ajax.googleapis.com/ajax/libs/jquery/$1/jquery.min.js",
	regexp.MustCompile("jquery-ui-([\\d\\.]+\\d)"):          "//ajax.googleapis.com/ajax/libs/jquery/$1/jquery-ui.min.js",
	regexp.MustCompile("mootools(:?-core)?-([\\d\\.]+\\d)"): "//ajax.googleapis.com/ajax/libs/mootools/$1/mootools-yui-compressed.js",
	regexp.MustCompile("prototype-([\\d\\.]+\\d)"):          "//ajax.googleapis.com/ajax/libs/prototype/$1/prototype.js",
	regexp.MustCompile("scriptaculous-([\\d\\.]+\\d)"):      "//ajax.googleapis.com/ajax/libs/scriptaculous/$1/scriptaculous.js",
	regexp.MustCompile("swfobject-([\\d\\.]+\\d)"):          "//ajax.googleapis.com/ajax/libs/swfobject/$1/swfobject.js",
	regexp.MustCompile("webfont-([\\d\\.]+\\d)"):            "//ajax.googleapis.com/ajax/libs/webfont/$1/webfont.js",
}

func Cdn(name string) (string, error) {
	base := filepath.Base(name)
	for k, v := range cdnMap {
		m := k.FindStringSubmatchIndex(base)
		if m != nil {
			var dst []byte
			return string(k.ExpandString(dst, v, base, m)), nil
		}
	}
	return "", fmt.Errorf("Could not find CDN URL for %q", name)
}
<commit_msg>Cosmetic change in error message<commit_after>package assets

import (
	"fmt"
	"path/filepath"
	"regexp"
)

type CdnMap map[*regexp.Regexp]string

var cdnMap = CdnMap{
	regexp.MustCompile("angular-([\\d\\.]+\\d)"):            "//ajax.googleapis.com/ajax/libs/angularjs/$1/angular.min.js",
	regexp.MustCompile("CFInstall-([\\d\\.]+\\d)"):          "//ajax.googleapis.com/ajax/libs/chrome-frame/$1/CFInstall.min.js",
	regexp.MustCompile("dojo-([\\d\\.]+\\d)"):               "//ajax.googleapis.com/ajax/libs/dojo/$1/dojo/dojo.js",
	regexp.MustCompile("ext-core-([\\d\\.]+\\d)"):           "//ajax.googleapis.com/ajax/libs/ext-core/$1/ext-core.js",
	regexp.MustCompile("jquery-([\\d\\.]+\\d)"):             "//ajax.googleapis.com/ajax/libs/jquery/$1/jquery.min.js",
	regexp.MustCompile("jquery-ui-([\\d\\.]+\\d)"):          "//ajax.googleapis.com/ajax/libs/jquery/$1/jquery-ui.min.js",
	regexp.MustCompile("mootools(:?-core)?-([\\d\\.]+\\d)"): "//ajax.googleapis.com/ajax/libs/mootools/$1/mootools-yui-compressed.js",
	regexp.MustCompile("prototype-([\\d\\.]+\\d)"):          "//ajax.googleapis.com/ajax/libs/prototype/$1/prototype.js",
	regexp.MustCompile("scriptaculous-([\\d\\.]+\\d)"):      "//ajax.googleapis.com/ajax/libs/scriptaculous/$1/scriptaculous.js",
	regexp.MustCompile("swfobject-([\\d\\.]+\\d)"):          "//ajax.googleapis.com/ajax/libs/swfobject/$1/swfobject.js",
	regexp.MustCompile("webfont-([\\d\\.]+\\d)"):            "//ajax.googleapis.com/ajax/libs/webfont/$1/webfont.js",
}

func Cdn(name string) (string, error) {
	base := filepath.Base(name)
	for k, v := range cdnMap {
		m := k.FindStringSubmatchIndex(base)
		if m != nil {
			var dst []byte
			return string(k.ExpandString(dst, v, base, m)), nil
		}
	}
	return "", fmt.Errorf("could not find CDN URL for %q", name)
}
<|endoftext|>


<commit_before>package assets

type CssAsset struct {
	*CommonAsset
	attributes Attributes
}

func (c *CssAsset) Tag() string {
	return "link"
}

func (c *CssAsset) Closed() bool {
	return false
}

func (c *CssAsset) Position() Position {
	return Top
}

func (c *CssAsset) Attributes() Attributes {
	return c.attributes
}

func (c *CssAsset) HTML() string {
	return ""
}

func (c *CssAsset) CodeType() int {
	return CodeTypeCss
}

func CssParser(m Manager, names []string, options Options) ([]Asset, error) {
	common, err := ParseCommonAssets(m, names, options)
	if err != nil {
		return nil, err
	}
	attrs := Attributes{"rel": "stylesheet"}
	for k, v := range options {
		if k == "media" {
			attrs[k] = v
		}
	}
	assets := make([]Asset, len(common))
	for ii, v := range common {
		attributes := make(Attributes, len(attrs)+1)
		for ak, av := range attrs {
			attributes[ak] = av
		}
		attributes["href"] = m.URL(v.Name())
		assets[ii] = &CssAsset{
			CommonAsset: v,
			attributes:  attributes,
		}
	}
	return assets, nil
}

func init() {
	Register("css", CssParser)
	Register("style", CssParser)
	Register("styles", CssParser)
}
<commit_msg>Add type="text/css" to css assets<commit_after>package assets

type CssAsset struct {
	*CommonAsset
	attributes Attributes
}

func (c *CssAsset) Tag() string {
	return "link"
}

func (c *CssAsset) Closed() bool {
	return false
}

func (c *CssAsset) Position() Position {
	return Top
}

func (c *CssAsset) Attributes() Attributes {
	return c.attributes
}

func (c *CssAsset) HTML() string {
	return ""
}

func (c *CssAsset) CodeType() int {
	return CodeTypeCss
}

func CssParser(m Manager, names []string, options Options) ([]Asset, error) {
	common, err := ParseCommonAssets(m, names, options)
	if err != nil {
		return nil, err
	}
	attrs := Attributes{"rel": "stylesheet", "type": "text/css"}
	for k, v := range options {
		if k == "media" {
			attrs[k] = v
		}
	}
	assets := make([]Asset, len(common))
	for ii, v := range common {
		attributes := make(Attributes, len(attrs)+1)
		for ak, av := range attrs {
			attributes[ak] = av
		}
		attributes["href"] = m.URL(v.Name())
		assets[ii] = &CssAsset{
			CommonAsset: v,
			attributes:  attributes,
		}
	}
	return assets, nil
}

func init() {
	Register("css", CssParser)
	Register("style", CssParser)
	Register("styles", CssParser)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"html/template"
	"io/ioutil"
	"os"
)

// Directory type for recursive/template purposes
type Directory struct {
	Name string
	Path string
	List []os.FileInfo
}

func main() {

	funcMap := template.FuncMap{
		"formatSize": FormatBytes,
		"type":       DetectType,
	}

	dir := Directory{Name: "", Path: "."}
	list, err := ioutil.ReadDir(dir.Path)
	if err != nil {
		panic(err)
	}
	dir.List = list

	f, err := os.Create(fmt.Sprintf("%s/index.html", dir.Path))
	if err != nil {
		panic(err)
	}
	defer f.Close()

	t := template.Must(template.New("index").Funcs(funcMap).Parse(tmpl))
	err = t.Execute(f, dir)
	if err != nil {
		panic(err)
	}

}
<commit_msg>Constructor and error handling<commit_after>package main

import (
	"fmt"
	"html/template"
	"io/ioutil"
	"os"
)

// Directory type for recursive/template purposes
type Directory struct {
	Path string
	List []os.FileInfo
}

// NewDirectory returns a new isntance of a directory type
func NewDirectory(path string) *Directory {
	return &Directory{Path: path}
}

func main() {

	funcMap := template.FuncMap{
		"formatSize": FormatBytes,
		"type":       DetectType,
		"title":      NameFromPath,
	}

	path, err := os.Getwd()
	if err != nil {
		panic(err)
	}

	dir := NewDirectory(path)
	dir.List, err = ioutil.ReadDir(dir.Path)
	if err != nil {
		panic(err)
	}

	f, err := os.Create(fmt.Sprintf("%s/index.html", dir.Path))
	if err != nil {
		panic(err)
	}
	defer f.Close()

	t := template.Must(template.New("index").Funcs(funcMap).Parse(tmpl))
	err = t.Execute(f, dir)
	if err != nil {
		panic(err)
	}

}
<|endoftext|>


<commit_before>package aero

import (
	"encoding/json"
	"fmt"
	"strconv"
	"time"
	"unsafe"

	"github.com/OneOfOne/xxhash"
	cache "github.com/patrickmn/go-cache"
	"github.com/valyala/fasthttp"
)

// This should be close to the MTU size of a TCP packet.
// Regarding performance it makes no sense to compress smaller files.
// Bandwidth can be saved however the savings are minimal for small files
// and the overhead of compressing can lead up to a 75% reduction
// in server speed under high load. Therefore in this case
// we're trying to optimize for performance, not bandwidth.
const gzipThreshold = 1450

var (
	serverHeader               = []byte("Server")
	server                     = []byte("Aero")
	cacheControlHeader         = []byte("Cache-Control")
	cacheControlAlwaysValidate = []byte("no-cache")
	contentTypeOptionsHeader   = []byte("X-Content-Type-Options")
	contentTypeOptions         = []byte("nosniff")
	xssProtectionHeader        = []byte("X-XSS-Protection")
	xssProtection              = []byte("1; mode=block")
	etagHeader                 = []byte("ETag")
	contentTypeHeader          = []byte("Content-Type")
	contentTypeHTML            = []byte("text/html; charset=utf-8")
	contentTypeJSON            = []byte("application/json")
	contentTypePlainText       = []byte("text/plain; charset=utf-8")
	contentEncodingHeader      = []byte("Content-Encoding")
	contentEncodingGzip        = []byte("gzip")
	responseTimeHeader         = []byte("X-Response-Time")
	ifNoneMatchHeader          = []byte("If-None-Match")
)

var ifNoneMatchHeaderBytes []byte

func init() {
	ifNoneMatchHeaderBytes = []byte(ifNoneMatchHeader)
}

// Context ...
type Context struct {
	// Keep this as the first parameter for quick pointer acquisition.
	requestCtx *fasthttp.RequestCtx

	// A pointer to the application this request occured on.
	App *Application

	// Start time
	start time.Time
}

// Handle ...
type Handle func(*Context) string

// JSON encodes the object to a JSON strings and responds.
func (ctx *Context) JSON(value interface{}) string {
	bytes, _ := json.Marshal(value)

	ctx.requestCtx.Response.Header.SetBytesKV(contentTypeHeader, contentTypeJSON)
	return string(bytes)
}

// HTML sends a HTML string.
func (ctx *Context) HTML(html string) string {
	ctx.requestCtx.Response.Header.SetBytesKV(contentTypeHeader, contentTypeHTML)
	return html
}

// Text sends a plain text string.
func (ctx *Context) Text(text string) string {
	ctx.requestCtx.Response.Header.SetBytesKV(contentTypeHeader, contentTypePlainText)
	return text
}

// SetHeader sets header to value.
func (ctx *Context) SetHeader(header string, value string) {
	ctx.requestCtx.Response.Header.Set(header, value)
}

// Get retrieves an URL parameter.
func (ctx *Context) Get(param string) string {
	return fmt.Sprint(ctx.requestCtx.UserValue(param))
}

// GetInt retrieves an URL parameter as an integer.
func (ctx *Context) GetInt(param string) (int, error) {
	return strconv.Atoi(ctx.Get(param))
}

// Respond responds either with raw code or gzipped if the
// code length is greater than the gzip threshold.
func (ctx *Context) Respond(code string) {
	ctx.RespondBytes(StringToBytesUnsafe(code))
}

// RespondBytes responds either with raw code or gzipped if the
// code length is greater than the gzip threshold. Requires a byte slice.
func (ctx *Context) RespondBytes(b []byte) {
	http := ctx.requestCtx

	// Headers
	http.Response.Header.SetBytesKV(cacheControlHeader, cacheControlAlwaysValidate)
	http.Response.Header.SetBytesKV(serverHeader, server)
	http.Response.Header.SetBytesKV(contentTypeOptionsHeader, contentTypeOptions)
	http.Response.Header.SetBytesKV(xssProtectionHeader, xssProtection)
	// http.Response.Header.Set(responseTimeHeader, strconv.FormatInt(time.Since(ctx.start).Nanoseconds()/1000, 10)+" us")

	if ctx.App.Security.Certificate != nil {
		http.Response.Header.Set("Content-Security-Policy", "default-src https:; script-src 'self'; style-src 'sha256-"+ctx.App.cssHash+"'; connect-src https: wss:")
	}

	// Body
	if ctx.App.Config.GZip && len(b) >= gzipThreshold {
		http.Response.Header.SetBytesKV(contentEncodingHeader, contentEncodingGzip)

		// ETag generation
		h := xxhash.NewS64(0)
		h.Write(b)
		etag := strconv.FormatUint(h.Sum64(), 16)

		// If client cache is up to date, send 304 with no response body.
		clientETag := http.Request.Header.PeekBytes(ifNoneMatchHeader)

		if etag == *(*string)(unsafe.Pointer(&clientETag)) {
			http.SetStatusCode(304)
			return
		}

		// Set ETag
		http.Response.Header.SetBytesK(etagHeader, etag)

		if ctx.App.Config.GZipCache {
			cachedResponse, found := ctx.App.gzipCache.Get(etag)

			if found {
				http.Write(cachedResponse.([]byte))
				return
			}
		}

		fasthttp.WriteGzipLevel(http.Response.BodyWriter(), b, 1)

		if ctx.App.Config.GZipCache {
			body := http.Response.Body()
			gzipped := make([]byte, len(body))
			copy(gzipped, body)
			ctx.App.gzipCache.Set(etag, gzipped, cache.DefaultExpiration)
		}
	} else {
		http.Write(b)
	}
}
<commit_msg>SetStatusCode and Error methods<commit_after>package aero

import (
	"encoding/json"
	"fmt"
	"strconv"
	"time"
	"unsafe"

	"github.com/OneOfOne/xxhash"
	cache "github.com/patrickmn/go-cache"
	"github.com/valyala/fasthttp"
)

// This should be close to the MTU size of a TCP packet.
// Regarding performance it makes no sense to compress smaller files.
// Bandwidth can be saved however the savings are minimal for small files
// and the overhead of compressing can lead up to a 75% reduction
// in server speed under high load. Therefore in this case
// we're trying to optimize for performance, not bandwidth.
const gzipThreshold = 1450

var (
	serverHeader               = []byte("Server")
	server                     = []byte("Aero")
	cacheControlHeader         = []byte("Cache-Control")
	cacheControlAlwaysValidate = []byte("no-cache")
	contentTypeOptionsHeader   = []byte("X-Content-Type-Options")
	contentTypeOptions         = []byte("nosniff")
	xssProtectionHeader        = []byte("X-XSS-Protection")
	xssProtection              = []byte("1; mode=block")
	etagHeader                 = []byte("ETag")
	contentTypeHeader          = []byte("Content-Type")
	contentTypeHTML            = []byte("text/html; charset=utf-8")
	contentTypeJSON            = []byte("application/json")
	contentTypePlainText       = []byte("text/plain; charset=utf-8")
	contentEncodingHeader      = []byte("Content-Encoding")
	contentEncodingGzip        = []byte("gzip")
	responseTimeHeader         = []byte("X-Response-Time")
	ifNoneMatchHeader          = []byte("If-None-Match")
)

var ifNoneMatchHeaderBytes []byte

func init() {
	ifNoneMatchHeaderBytes = []byte(ifNoneMatchHeader)
}

// Context ...
type Context struct {
	// Keep this as the first parameter for quick pointer acquisition.
	requestCtx *fasthttp.RequestCtx

	// A pointer to the application this request occured on.
	App *Application

	// Start time
	start time.Time
}

// Handle ...
type Handle func(*Context) string

// JSON encodes the object to a JSON strings and responds.
func (ctx *Context) JSON(value interface{}) string {
	bytes, _ := json.Marshal(value)

	ctx.requestCtx.Response.Header.SetBytesKV(contentTypeHeader, contentTypeJSON)
	return string(bytes)
}

// HTML sends a HTML string.
func (ctx *Context) HTML(html string) string {
	ctx.requestCtx.Response.Header.SetBytesKV(contentTypeHeader, contentTypeHTML)
	return html
}

// Text sends a plain text string.
func (ctx *Context) Text(text string) string {
	ctx.requestCtx.Response.Header.SetBytesKV(contentTypeHeader, contentTypePlainText)
	return text
}

// Error should be used for sending HTML error messages to the user.
func (ctx *Context) Error(statusCode int, html string) string {
	ctx.SetStatusCode(statusCode)
	ctx.requestCtx.Response.Header.SetBytesKV(contentTypeHeader, contentTypeHTML)
	return html
}

// SetStatusCode sets the status code of the request.
func (ctx *Context) SetStatusCode(status int) {
	ctx.requestCtx.SetStatusCode(status)
}

// SetHeader sets header to value.
func (ctx *Context) SetHeader(header string, value string) {
	ctx.requestCtx.Response.Header.Set(header, value)
}

// Get retrieves an URL parameter.
func (ctx *Context) Get(param string) string {
	return fmt.Sprint(ctx.requestCtx.UserValue(param))
}

// GetInt retrieves an URL parameter as an integer.
func (ctx *Context) GetInt(param string) (int, error) {
	return strconv.Atoi(ctx.Get(param))
}

// Respond responds either with raw code or gzipped if the
// code length is greater than the gzip threshold.
func (ctx *Context) Respond(code string) {
	ctx.RespondBytes(StringToBytesUnsafe(code))
}

// RespondBytes responds either with raw code or gzipped if the
// code length is greater than the gzip threshold. Requires a byte slice.
func (ctx *Context) RespondBytes(b []byte) {
	http := ctx.requestCtx

	// Headers
	http.Response.Header.SetBytesKV(cacheControlHeader, cacheControlAlwaysValidate)
	http.Response.Header.SetBytesKV(serverHeader, server)
	http.Response.Header.SetBytesKV(contentTypeOptionsHeader, contentTypeOptions)
	http.Response.Header.SetBytesKV(xssProtectionHeader, xssProtection)
	// http.Response.Header.Set(responseTimeHeader, strconv.FormatInt(time.Since(ctx.start).Nanoseconds()/1000, 10)+" us")

	if ctx.App.Security.Certificate != nil {
		http.Response.Header.Set("Content-Security-Policy", "default-src https:; script-src 'self'; style-src 'sha256-"+ctx.App.cssHash+"'; connect-src https: wss:")
	}

	// Body
	if ctx.App.Config.GZip && len(b) >= gzipThreshold {
		http.Response.Header.SetBytesKV(contentEncodingHeader, contentEncodingGzip)

		// ETag generation
		h := xxhash.NewS64(0)
		h.Write(b)
		etag := strconv.FormatUint(h.Sum64(), 16)

		// If client cache is up to date, send 304 with no response body.
		clientETag := http.Request.Header.PeekBytes(ifNoneMatchHeader)

		if etag == *(*string)(unsafe.Pointer(&clientETag)) {
			http.SetStatusCode(304)
			return
		}

		// Set ETag
		http.Response.Header.SetBytesK(etagHeader, etag)

		if ctx.App.Config.GZipCache {
			cachedResponse, found := ctx.App.gzipCache.Get(etag)

			if found {
				http.Write(cachedResponse.([]byte))
				return
			}
		}

		fasthttp.WriteGzipLevel(http.Response.BodyWriter(), b, 1)

		if ctx.App.Config.GZipCache {
			body := http.Response.Body()
			gzipped := make([]byte, len(body))
			copy(gzipped, body)
			ctx.App.gzipCache.Set(etag, gzipped, cache.DefaultExpiration)
		}
	} else {
		http.Write(b)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"archive/zip"
	"bufio"
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"

	"github.com/docopt/docopt-go"
)

// Configuration
type Configuration struct {
	Url      string `json:"url"`
	Username string `json:"username"`
	Password string `json:"password"`
}

func main() {

	usage := `SOA Software Community Manager Helper Tool.

Usage:
  atmosphere zip --prefix <prefix> --config <config> [--dir <dir>]
  atmosphere upload less <file> --config <config>
  atmosphere upload file --path <path> --config <config> <files>...
  atmosphere upload all --config <config> [--dir <dir>]
  atmosphere -h | --help
  atmosphere --version

Options:
  -h --help  Show help message and exit.
  --version  Show version and exit.
  --dir=<dir>  Directory. [default: .]
  --path=<cms_path>  CM CMS path.
`
	arguments, _ := docopt.Parse(usage, nil, true, "1.0 cirrus", false)

	// Debug for command-line args
	/*
		var keys []string
		for k := range arguments {
			keys = append(keys, k)
		}
		sort.Strings(keys)
		// print the argument keys and values
		for _, k := range keys {
			fmt.Printf("%9s %v\n", k, arguments[k])
		}
	*/

	configLocation, _ := arguments["<config>"].(string)
	configBytes, err := ioutil.ReadFile(configLocation)
	if err != nil {
		fmt.Printf("Error opening config file %s\n", err)
		flag.Usage()
		os.Exit(1)
	}
	var config Configuration
	err = json.Unmarshal(configBytes, &config)
	if err != nil {
		fmt.Printf("Unable to parse configuration file. %s\n", err)
		os.Exit(1)
	}

	if len(config.Password) < 1 {
		fmt.Printf("Missing or blank password.")
		os.Exit(1)
	}

	if arguments["upload"] == true {
		if arguments["less"] == true {
			lessFilePath := arguments["<file>"].(string)
			uploadLessFile(lessFilePath, config)
		} else if arguments["all"] == true {
			dir, _ := arguments["--dir"].(string)
			uploadAllHelper(dir, config)
		} else if arguments["file"] == true {
			var files []string
			for _, v := range arguments["<files>"].([]string) {
				files = append(files, v)
			}
			path, _ := arguments["--path"].(string)
			upload(files, config, path)
		}
	} else if arguments["zip"] == true {
		prefix, _ := arguments["<prefix>"].(string)
		dir, _ := arguments["--dir"].(string)
		zipPredefinedPath(prefix, dir)
	}
}

func zipPredefinedPath(prefix string, dir string) {
	//fmt.Printf("Zipping '%s' with prefix '%s'\n", dir, prefix)

	// test to see if expected paths exist
	resources_dir := dir + "/resources/theme/default"
	content_dir := dir + "/landing"

	resources_src, err := os.Stat(resources_dir)
	if err != nil {
		fmt.Printf("Error with resource dir ", err)
		os.Exit(1)
	}
	if !resources_src.IsDir() {
		fmt.Printf("%s is not a directory.", resources_dir)
	}

	content_src, err := os.Stat(content_dir)
	if err != nil {
		fmt.Printf("Error with content dir ", err)
		os.Exit(1)
	}
	if !content_src.IsDir() {
		fmt.Printf("%s is not a directory.", content_dir)
		os.Exit(1)
	}

	exclusions := []string{".DS_Store", ".zip"}

	// Get the file lists
	resourcesFileList := listFilesInDir(resources_dir, "", exclusions, false)
	contentFileList := listFilesInDir(content_dir, "", exclusions, false)

	// Create zip files
	writeZipTo(createZipBuffer(resourcesFileList), prefix+"_resourcesThemeDefault.zip")
	zipTheseFiles(contentFileList, prefix+"_contentHomeLanding.zip")
}

// writes a zip buffer to a filename
func writeZipTo(zipbuffer *bytes.Buffer, filename string) {

	fout, err := os.Create(filename)
	if err != nil {
		panic(err)
	}
	defer func() {
		if err := fout.Close(); err != nil {
			panic(err)
		}
	}()

	fw := bufio.NewWriter(fout)
	fw.Write(zipbuffer.Bytes())
	fw.Flush()

	fileinfo, _ := os.Stat(filename)
	fmt.Printf("Created %s (%v)\n", filename, fileinfo.Size())

}

// creates a zip byte buffer and returns
func createZipBuffer(filesList []string) *bytes.Buffer {

	buf := new(bytes.Buffer)
	w := zip.NewWriter(buf)
	for _, file := range filesList {
		f, err := w.Create(file)
		if err != nil {
			log.Fatal(err)
		}
		byts, _ := ioutil.ReadFile(file)
		_, err = f.Write(byts)
		if err != nil {
			log.Fatal(err)
		}
	}

	// Make sure to check the error on Close.
	err := w.Close()
	if err != nil {
		log.Fatal(err)
	}

	return buf

}

// convenience method
func zipTheseFiles(filesList []string, filename string) {

	writeZipTo(createZipBuffer(filesList), filename)

}

func listFilesInDir(path string, relpath string, exclusions []string, debug bool) []string {

	if debug {
		fmt.Printf("%s\n", path)
	}

	var fileList []string

	files, _ := ioutil.ReadDir(path)
	for _, f := range files {

		exclude := false
		for _, e := range exclusions {
			if exclude = strings.Contains(f.Name(), e); exclude {
				break
			}
		}

		if debug && exclude {
			fmt.Printf("  - %s\n", f.Name())
		}

		if !exclude {
			if debug {
				fmt.Printf("   %s\n", f.Name())
			}
			if f.IsDir() {
				dirFileList := listFilesInDir(path+"/"+f.Name(), f.Name()+"/", exclusions, debug)
				fileList = append(fileList, dirFileList...)
			}
			fileList = append(fileList, relpath+f.Name())
		}

	}

	return fileList
}

func uploadLessFile(lessFilePath string, config Configuration) {
	fmt.Printf("Uploading Less file %s to %s\n", lessFilePath, config.Url)
}

func uploadAllHelper(dir string, config Configuration) {
	fmt.Printf("Uploading all in %s to %s\n", dir, config.Url)
}

func upload(files []string, config Configuration, path string) {
	fmt.Printf("Uploading to %s cms location %s these: %s\n", config.Url, path, files)
}
<commit_msg>some comments<commit_after>package main

import (
	"archive/zip"
	"bufio"
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"

	"github.com/docopt/docopt-go"
)

// Configuration
type Configuration struct {
	Url      string `json:"url"`
	Username string `json:"username"`
	Password string `json:"password"`
}

func main() {

	usage := `SOA Software Community Manager Helper Tool.

Usage:
  atmosphere zip --prefix <prefix> --config <config> [--dir <dir>]
  atmosphere upload less <file> --config <config>
  atmosphere upload file --path <path> --config <config> <files>...
  atmosphere upload all --config <config> [--dir <dir>]
  atmosphere -h | --help
  atmosphere --version

Options:
  -h --help  Show help message and exit.
  --version  Show version and exit.
  --dir=<dir>  Directory. [default: .]
  --path=<cms_path>  CM CMS path.
`
	arguments, _ := docopt.Parse(usage, nil, true, "1.0 cirrus", false)

	// Debug for command-line args
	/*
		var keys []string
		for k := range arguments {
			keys = append(keys, k)
		}
		sort.Strings(keys)
		// print the argument keys and values
		for _, k := range keys {
			fmt.Printf("%9s %v\n", k, arguments[k])
		}
	*/

	configLocation, _ := arguments["<config>"].(string)
	configBytes, err := ioutil.ReadFile(configLocation)
	if err != nil {
		fmt.Printf("Error opening config file %s\n", err)
		flag.Usage()
		os.Exit(1)
	}
	var config Configuration
	err = json.Unmarshal(configBytes, &config)
	if err != nil {
		fmt.Printf("Unable to parse configuration file. %s\n", err)
		os.Exit(1)
	}

	if len(config.Password) < 1 {
		fmt.Printf("Missing or blank password.")
		os.Exit(1)
	}

	if arguments["upload"] == true {
		if arguments["less"] == true {
			lessFilePath := arguments["<file>"].(string)
			uploadLessFile(lessFilePath, config)
		} else if arguments["all"] == true {
			dir, _ := arguments["--dir"].(string)
			uploadAllHelper(dir, config)
		} else if arguments["file"] == true {
			var files []string
			for _, v := range arguments["<files>"].([]string) {
				files = append(files, v)
			}
			path, _ := arguments["--path"].(string)
			upload(files, config, path)
		}
	} else if arguments["zip"] == true {
		prefix, _ := arguments["<prefix>"].(string)
		dir, _ := arguments["--dir"].(string)
		zipPredefinedPath(prefix, dir)
	}
}

func zipPredefinedPath(prefix string, dir string) {
	//fmt.Printf("Zipping '%s' with prefix '%s'\n", dir, prefix)

	// test to see if expected paths exist
	resources_dir := dir + "/resources/theme/default"
	content_dir := dir + "/landing"

	resources_src, err := os.Stat(resources_dir)
	if err != nil {
		fmt.Printf("Error with resource dir ", err)
		os.Exit(1)
	}
	if !resources_src.IsDir() {
		fmt.Printf("%s is not a directory.", resources_dir)
	}

	content_src, err := os.Stat(content_dir)
	if err != nil {
		fmt.Printf("Error with content dir ", err)
		os.Exit(1)
	}
	if !content_src.IsDir() {
		fmt.Printf("%s is not a directory.", content_dir)
		os.Exit(1)
	}

	exclusions := []string{".DS_Store", ".zip"}

	// Get the file lists
	resourcesFileList := listFilesInDir(resources_dir, "", exclusions, false)
	contentFileList := listFilesInDir(content_dir, "", exclusions, false)

	// Create zip files
	writeZipTo(createZipBuffer(resourcesFileList), prefix+"_resourcesThemeDefault.zip")
	zipTheseFiles(contentFileList, prefix+"_contentHomeLanding.zip")
}

// writes a zip buffer to a filename
func writeZipTo(zipbuffer *bytes.Buffer, filename string) {

	fout, err := os.Create(filename)
	if err != nil {
		panic(err)
	}
	defer func() {
		if err := fout.Close(); err != nil {
			panic(err)
		}
	}()

	fw := bufio.NewWriter(fout)
	fw.Write(zipbuffer.Bytes())
	fw.Flush()

	fileinfo, _ := os.Stat(filename)
	fmt.Printf("Created %s (%v)\n", filename, fileinfo.Size())

}

// creates a zip byte buffer and returns
func createZipBuffer(filesList []string) *bytes.Buffer {

	buf := new(bytes.Buffer)
	w := zip.NewWriter(buf)
	for _, file := range filesList {
		f, err := w.Create(file)
		if err != nil {
			log.Fatal(err)
		}
		byts, _ := ioutil.ReadFile(file)
		_, err = f.Write(byts)
		if err != nil {
			log.Fatal(err)
		}
	}

	// Make sure to check the error on Close.
	err := w.Close()
	if err != nil {
		log.Fatal(err)
	}

	return buf

}

// convenience method
func zipTheseFiles(filesList []string, filename string) {

	writeZipTo(createZipBuffer(filesList), filename)

}

// creates a list of file paths suitable for zipping
func listFilesInDir(path string, relpath string, exclusions []string, debug bool) []string {

	if debug {
		fmt.Printf("%s\n", path)
	}

	var fileList []string

	files, _ := ioutil.ReadDir(path)
	for _, f := range files {

		exclude := false
		for _, e := range exclusions {
			if exclude = strings.Contains(f.Name(), e); exclude {
				break
			}
		}

		if debug && exclude {
			fmt.Printf("  - %s\n", f.Name())
		}

		if !exclude {
			if debug {
				fmt.Printf("   %s\n", f.Name())
			}
			if f.IsDir() {
				dirFileList := listFilesInDir(path+"/"+f.Name(), f.Name()+"/", exclusions, debug)
				fileList = append(fileList, dirFileList...)
			}
			fileList = append(fileList, relpath+f.Name())
		}

	}

	return fileList
}

// Convenience method
func uploadLessFile(lessFilePath string, config Configuration) {
	fmt.Printf("Uploading Less file %s to %s\n", lessFilePath, config.Url)
	// upload to CMS path ??
	// call rebuild styles API

}

// Convenience method
func uploadAllHelper(dir string, config Configuration) {
	fmt.Printf("Uploading all in %s to %s\n", dir, config.Url)
	// upload PREFIX_resourcesThemeDefault.zip to CMS /resources/theme/default
	// upload PREFIX_contentHomeLanding.zip to CMS /content/home/landing
	// upload less file to CMS ??
	// call rebuild styles API

}

// basic upload to CMS
func upload(files []string, config Configuration, path string) {
	fmt.Printf("Uploading to %s cms location %s these: %s\n", config.Url, path, files)
	// upload FILE to CMS path PATH
	// iterate through []FILE
}
<|endoftext|>


<commit_before>package service

import (
	commonModel "github.com/Cepave/open-falcon-backend/common/model"
	commonQueue "github.com/Cepave/open-falcon-backend/common/queue"
	"github.com/Cepave/open-falcon-backend/modules/nqm-mng/model"
	"github.com/icrowley/fake"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

type fakeHeartbeat struct {
	rowsAffectedCnt int
	alwaysFail      bool
}

func (a *fakeHeartbeat) calling(agents []*model.AgentHeartbeat) (int64, int64) {
	a.rowsAffectedCnt += len(agents)

	if a.alwaysFail {
		return int64(len(agents)), 0
	} else {
		return 0, int64(len(agents))
	}
}

func generateRandomHeartbeat() *commonModel.AgentReportRequest {
	return &commonModel.AgentReportRequest{
		Hostname:     fake.DomainName(),
		IP:           fake.IPv4(),
		AgentVersion: fake.Digits(),
	}
}

var _ = Describe("Test Put() of AgentHeartbeat service", func() {
	var (
		agentHeartbeatService *AgentHeartbeatService
		heartbeatImpl         *fakeHeartbeat
	)

	BeforeEach(func() {
		agentHeartbeatService = NewAgentHeartbeatService(
			&commonQueue.Config{Num: 16},
		)

		heartbeatImpl = &fakeHeartbeat{alwaysFail: false}
		agentHeartbeatService.heartbeatCall = heartbeatImpl.calling
	})

	AfterEach(func() {
		agentHeartbeatService.Stop()
	})

	Context("when service is not running", func() {
		It("should not add data", func() {
			agentHeartbeatService.Put(generateRandomHeartbeat())
			Expect(agentHeartbeatService.CumulativeAgentsPut()).To(Equal(int64(0)))
			Expect(agentHeartbeatService.CurrentSize()).To(Equal(0))
		})
	})

	Context("when service is running", func() {
		It("should add data", func() {
			agentHeartbeatService.Start()
			agentHeartbeatService.Put(generateRandomHeartbeat())

			Expect(agentHeartbeatService.CumulativeAgentsPut()).To(Equal(int64(1)))
		})
	})
})

var _ = Describe("Test Start()/Stop() of AgentHeartbeat service", func() {
	var (
		agentHeartbeatService *AgentHeartbeatService
	)

	BeforeEach(func() {
		agentHeartbeatService = NewAgentHeartbeatService(
			&commonQueue.Config{},
		)
	})

	AfterEach(func() {
		agentHeartbeatService.Stop()
	})

	Context("when service is stopped", func() {
		It("Stop() should not change the running status", func() {
			agentHeartbeatService.Stop()
			Expect(agentHeartbeatService.running).To(Equal(false))
		})
	})

	Context("when service is started", func() {
		It("Start() should not change the running status", func() {
			agentHeartbeatService.running = true
			agentHeartbeatService.Start()
			Expect(agentHeartbeatService.running).To(Equal(true))
		})
	})
})
<commit_msg>[OWL-1695] Add test of consumeHeartbeatQueue(). Rename a non-logical variable name.<commit_after>package service

import (
	commonModel "github.com/Cepave/open-falcon-backend/common/model"
	commonQueue "github.com/Cepave/open-falcon-backend/common/queue"
	"github.com/Cepave/open-falcon-backend/modules/nqm-mng/model"
	"github.com/icrowley/fake"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

type fakeHeartbeat struct {
	rowsAffectedCnt int
	alwaysSuccess   bool
}

func (a *fakeHeartbeat) calling(agents []*model.AgentHeartbeat) (int64, int64) {
	count := len(agents)
	a.rowsAffectedCnt += count

	if a.alwaysSuccess {
		return int64(count), 0
	} else {
		return 0, int64(count)
	}
}

func generateRandomHeartbeat() *commonModel.AgentReportRequest {
	return &commonModel.AgentReportRequest{
		Hostname:     fake.DomainName(),
		IP:           fake.IPv4(),
		AgentVersion: fake.Digits(),
	}
}

var _ = Describe("Test Put() of AgentHeartbeat service", func() {
	var (
		agentHeartbeatService *AgentHeartbeatService
		heartbeatImpl         *fakeHeartbeat
	)

	BeforeEach(func() {
		agentHeartbeatService = NewAgentHeartbeatService(
			&commonQueue.Config{Num: 16},
		)

		heartbeatImpl = &fakeHeartbeat{alwaysSuccess: true}
		agentHeartbeatService.heartbeatCall = heartbeatImpl.calling
	})

	AfterEach(func() {
		agentHeartbeatService.Stop()
	})

	Context("when service is not running", func() {
		It("should not add data", func() {
			agentHeartbeatService.Put(generateRandomHeartbeat())
			Expect(agentHeartbeatService.CumulativeAgentsPut()).To(Equal(int64(0)))
			Expect(agentHeartbeatService.CurrentSize()).To(Equal(0))
		})
	})

	Context("when service is running", func() {
		It("should add data", func() {
			agentHeartbeatService.Start()
			agentHeartbeatService.Put(generateRandomHeartbeat())

			Expect(agentHeartbeatService.CumulativeAgentsPut()).To(Equal(int64(1)))
		})
	})
})

var _ = Describe("Test Start()/Stop() of AgentHeartbeat service", func() {
	var (
		agentHeartbeatService *AgentHeartbeatService
	)

	BeforeEach(func() {
		agentHeartbeatService = NewAgentHeartbeatService(
			&commonQueue.Config{},
		)
	})

	AfterEach(func() {
		agentHeartbeatService.Stop()
	})

	Context("when service is stopped", func() {
		It("Stop() should not change the running status", func() {
			agentHeartbeatService.Stop()
			Expect(agentHeartbeatService.running).To(Equal(false))
		})
	})

	Context("when service is started", func() {
		It("Start() should not change the running status", func() {
			agentHeartbeatService.running = true
			agentHeartbeatService.Start()
			Expect(agentHeartbeatService.running).To(Equal(true))
		})
	})
})

var _ = Describe("Test consumeHeartbeatQueue() of AgentHeartbeat service", func() {
	var (
		agentHeartbeatService *AgentHeartbeatService
		heartbeatImpl         *fakeHeartbeat
	)

	BeforeEach(func() {
		heartbeatImpl = &fakeHeartbeat{alwaysSuccess: true}
	})

	JustBeforeEach(func() {
		agentHeartbeatService = NewAgentHeartbeatService(
			&commonQueue.Config{Num: 16},
		)
		agentHeartbeatService.heartbeatCall = heartbeatImpl.calling
		agentHeartbeatService.running = true
	})

	Context("when success", func() {
		It("rowsAffectedCnt should be incremented normally", func() {
			agentHeartbeatService.Put(generateRandomHeartbeat())
			agentHeartbeatService.consumeHeartbeatQueue(false)

			Expect(heartbeatImpl.rowsAffectedCnt).To(Equal(1))
			Expect(agentHeartbeatService.CumulativeRowsAffected()).To(Equal(int64(1)))
			Expect(agentHeartbeatService.CumulativeAgentsDropped()).To(Equal(int64(0)))
		})
	})

	Context("when failure", func() {
		BeforeEach(func() {
			heartbeatImpl = &fakeHeartbeat{alwaysSuccess: false}
		})

		It("agentsDroppedCnt should be incremented normally", func() {
			agentHeartbeatService.Put(generateRandomHeartbeat())
			agentHeartbeatService.consumeHeartbeatQueue(false)

			Expect(heartbeatImpl.rowsAffectedCnt).To(Equal(1))
			Expect(agentHeartbeatService.CumulativeRowsAffected()).To(Equal(int64(0)))
			Expect(agentHeartbeatService.CumulativeAgentsDropped()).To(Equal(int64(1)))
		})
	})

	Context("when non-flushing mode", func() {
		It("should consume an amount of data = batch size", func() {
			batchSize := agentHeartbeatService.qConfig.Num
			dataNum := batchSize*2 - 1
			for i := 0; i < dataNum; i++ {
				agentHeartbeatService.Put(generateRandomHeartbeat())
			}
			agentHeartbeatService.consumeHeartbeatQueue(false)

			Expect(heartbeatImpl.rowsAffectedCnt).To(Equal(batchSize))
			Expect(agentHeartbeatService.CurrentSize()).To(Equal(batchSize - 1))
		})
	})

	Context("when flushing mode", func() {
		It("should flush data to 0", func() {
			dataNum := agentHeartbeatService.qConfig.Num*2 - 1
			for i := 0; i < dataNum; i++ {
				agentHeartbeatService.Put(generateRandomHeartbeat())
			}
			agentHeartbeatService.consumeHeartbeatQueue(true)

			Expect(heartbeatImpl.rowsAffectedCnt).To(Equal(dataNum))
			Expect(agentHeartbeatService.CurrentSize()).To(Equal(0))
		})
	})
})
<|endoftext|>


<commit_before>// Copyright 2015 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package auth

import (
	"errors"
	"strings"

	"github.com/tsuru/tsuru/permission"
)

type Token interface {
	GetValue() string
	GetAppName() string
	GetUserName() string
	IsAppToken() bool
	User() (*User, error)
	Permissions() ([]permission.Permission, error)
}

var ErrInvalidToken = errors.New("Invalid token")

// ParseToken extracts token from a header:
// 'type token' or 'token'
func ParseToken(header string) (string, error) {
	s := strings.Split(header, " ")
	var value string
	if len(s) < 3 {
		value = s[len(s)-1]
	}
	if value != "" {
		return value, nil
	}
	return value, ErrInvalidToken
}

func BaseTokenPermission(t Token) ([]permission.Permission, error) {
	if t.IsAppToken() {
		// TODO(cezarsa): Improve handling of app tokens. These permissions
		// listed here are the ones required by deploy-agent and legacy tsuru-
		// unit-agent.
		return []permission.Permission{
			{
				Scheme:  permission.PermAppUpdateUnitRegister,
				Context: permission.Context(permission.CtxApp, t.GetAppName()),
			},
			{
				Scheme:  permission.PermAppUpdateLog,
				Context: permission.Context(permission.CtxApp, t.GetAppName()),
			},
			{
				Scheme:  permission.PermAppUpdateUnitStatus,
				Context: permission.Context(permission.CtxApp, t.GetAppName()),
			},
		}, nil
	}
	user, err := t.User()
	if err != nil {
		return nil, err
	}
	return user.Permissions()
}
<commit_msg>auth: add permission to Deploy-agent Related to #1315<commit_after>// Copyright 2015 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package auth

import (
	"errors"
	"strings"

	"github.com/tsuru/tsuru/permission"
)

type Token interface {
	GetValue() string
	GetAppName() string
	GetUserName() string
	IsAppToken() bool
	User() (*User, error)
	Permissions() ([]permission.Permission, error)
}

var ErrInvalidToken = errors.New("Invalid token")

// ParseToken extracts token from a header:
// 'type token' or 'token'
func ParseToken(header string) (string, error) {
	s := strings.Split(header, " ")
	var value string
	if len(s) < 3 {
		value = s[len(s)-1]
	}
	if value != "" {
		return value, nil
	}
	return value, ErrInvalidToken
}

func BaseTokenPermission(t Token) ([]permission.Permission, error) {
	if t.IsAppToken() {
		// TODO(cezarsa): Improve handling of app tokens. These permissions
		// listed here are the ones required by deploy-agent and legacy tsuru-
		// unit-agent.
		return []permission.Permission{
			{
				Scheme:  permission.PermAppUpdateUnitRegister,
				Context: permission.Context(permission.CtxApp, t.GetAppName()),
			},
			{
				Scheme:  permission.PermAppUpdateLog,
				Context: permission.Context(permission.CtxApp, t.GetAppName()),
			},
			{
				Scheme:  permission.PermAppUpdateUnitStatus,
				Context: permission.Context(permission.CtxApp, t.GetAppName()),
			},
			{
				Scheme:  permission.PermAppReadDeploy,
				Context: permission.Context(permission.CtxApp, t.GetAppName()),
			},
		}, nil
	}
	user, err := t.User()
	if err != nil {
		return nil, err
	}
	return user.Permissions()
}
<|endoftext|>


<commit_before>package http

import (
	"github.com/Cepave/open-falcon-backend/modules/transfer/proc"
	"github.com/Cepave/open-falcon-backend/modules/transfer/sender"
	cutils "github.com/open-falcon/common/utils"
	"net/http"
	"strconv"
	"strings"
)

func configProcHttpRoutes() {
	// counter
	http.HandleFunc("/counter/all", func(w http.ResponseWriter, r *http.Request) {
		RenderDataJson(w, proc.GetAll())
	})

	// TO BE DISCARDed
	http.HandleFunc("/statistics/all", func(w http.ResponseWriter, r *http.Request) {
		RenderDataJson(w, proc.GetAll())
	})

	// step
	http.HandleFunc("/proc/step", func(w http.ResponseWriter, r *http.Request) {
		RenderDataJson(w, map[string]interface{}{"min_step": sender.MinStep})
	})

	// trace
	http.HandleFunc("/trace/", func(w http.ResponseWriter, r *http.Request) {
		urlParam := r.URL.Path[len("/trace/"):]
		args := strings.Split(urlParam, "/")

		argsLen := len(args)
		endpoint := args[0]
		metric := args[1]
		tags := make(map[string]string)
		if argsLen > 2 {
			tagVals := strings.Split(args[2], ",")
			for _, tag := range tagVals {
				tagPairs := strings.Split(tag, "=")
				if len(tagPairs) == 2 {
					tags[tagPairs[0]] = tagPairs[1]
				}
			}
		}
		proc.RecvDataTrace.SetPK(cutils.PK(endpoint, metric, tags))
		RenderDataJson(w, proc.RecvDataTrace.GetAllTraced())
	})

	// filter
	http.HandleFunc("/filter/", func(w http.ResponseWriter, r *http.Request) {
		urlParam := r.URL.Path[len("/filter/"):]
		args := strings.Split(urlParam, "/")

		argsLen := len(args)
		endpoint := args[0]
		metric := args[1]
		opt := args[2]

		threadholdStr := args[3]
		threadhold, err := strconv.ParseFloat(threadholdStr, 64)
		if err != nil {
			RenderDataJson(w, "bad threadhold")
			return
		}

		tags := make(map[string]string)
		if argsLen > 4 {
			tagVals := strings.Split(args[4], ",")
			for _, tag := range tagVals {
				tagPairs := strings.Split(tag, "=")
				if len(tagPairs) == 2 {
					tags[tagPairs[0]] = tagPairs[1]
				}
			}
		}

		err = proc.RecvDataFilter.SetFilter(cutils.PK(endpoint, metric, tags), opt, threadhold)
		if err != nil {
			RenderDataJson(w, err.Error())
			return
		}

		RenderDataJson(w, proc.RecvDataFilter.GetAllFiltered())
	})
}
<commit_msg>[OWL-989] transfer send out proper message when API trace was misused.<commit_after>package http

import (
	"net/http"
	"strconv"
	"strings"

	"github.com/Cepave/open-falcon-backend/modules/transfer/proc"
	"github.com/Cepave/open-falcon-backend/modules/transfer/sender"
	cutils "github.com/open-falcon/common/utils"
)

func configProcHttpRoutes() {
	// counter
	http.HandleFunc("/counter/all", func(w http.ResponseWriter, r *http.Request) {
		RenderDataJson(w, proc.GetAll())
	})

	// TO BE DISCARDed
	http.HandleFunc("/statistics/all", func(w http.ResponseWriter, r *http.Request) {
		RenderDataJson(w, proc.GetAll())
	})

	// step
	http.HandleFunc("/proc/step", func(w http.ResponseWriter, r *http.Request) {
		RenderDataJson(w, map[string]interface{}{"min_step": sender.MinStep})
	})

	// trace
	http.HandleFunc("/trace/", func(w http.ResponseWriter, r *http.Request) {
		urlParam := r.URL.Path[len("/trace/"):]
		args := strings.Split(urlParam, "/")

		argsLen := len(args)
		if argsLen >= 2 {
			endpoint := args[0]
			metric := args[1]
			tags := make(map[string]string)
			if argsLen > 2 {
				tagVals := strings.Split(args[2], ",")
				for _, tag := range tagVals {
					tagPairs := strings.Split(tag, "=")
					if len(tagPairs) == 2 {
						tags[tagPairs[0]] = tagPairs[1]
					}
				}
			}
			proc.RecvDataTrace.SetPK(cutils.PK(endpoint, metric, tags))
			RenderDataJson(w, proc.RecvDataTrace.GetAllTraced())
		} else {
			msg := "API trace needs to have endpoint, metric, parameter"
			RenderDataJson(w, msg)
		}
	})

	// filter
	http.HandleFunc("/filter/", func(w http.ResponseWriter, r *http.Request) {
		urlParam := r.URL.Path[len("/filter/"):]
		args := strings.Split(urlParam, "/")

		argsLen := len(args)
		endpoint := args[0]
		metric := args[1]
		opt := args[2]

		threadholdStr := args[3]
		threadhold, err := strconv.ParseFloat(threadholdStr, 64)
		if err != nil {
			RenderDataJson(w, "bad threadhold")
			return
		}

		tags := make(map[string]string)
		if argsLen > 4 {
			tagVals := strings.Split(args[4], ",")
			for _, tag := range tagVals {
				tagPairs := strings.Split(tag, "=")
				if len(tagPairs) == 2 {
					tags[tagPairs[0]] = tagPairs[1]
				}
			}
		}

		err = proc.RecvDataFilter.SetFilter(cutils.PK(endpoint, metric, tags), opt, threadhold)
		if err != nil {
			RenderDataJson(w, err.Error())
			return
		}

		RenderDataJson(w, proc.RecvDataFilter.GetAllFiltered())
	})
}
<|endoftext|>