<commit_before>/*
	Go Language Raspberry Pi Interface
	(c) Copyright David Thorpe 2016-2017
	All Rights Reserved

    Documentation http://djthorpe.github.io/gopi/
	For Licensing and Usage information, please see LICENSE.md
*/

package rpi /* import "github.com/djthorpe/gopi/device/rpi" */

import (
	"fmt"
)

////////////////////////////////////////////////////////////////////////////////
// TYPES

type (
	Model       uint32
	Processor   uint32
	PCBRevision uint32
)

////////////////////////////////////////////////////////////////////////////////
// CONSTANTS

const (
	RPI_REVISION_WARRANTY_MASK     uint32 = 0x03000000
	RPI_REVISION_ENCODING_MASK     uint32 = 0x00800000
	RPI_REVISION_PCB_MASK          uint32 = 0x0000000F
	RPI_REVISION_PRODUCT_MASK      uint32 = 0x00000FF0
	RPI_REVISION_PROCESSOR_MASK    uint32 = 0x0000F000
	RPI_REVISION_MANUFACTURER_MASK uint32 = 0x000F0000
	RPI_REVISION_MEMORY_MASK       uint32 = 0x00700000
)

const (
	RPI_MODEL_A              Model = (0 << 4)
	RPI_MODEL_B              Model = (1 << 4)
	RPI_MODEL_A_PLUS         Model = (2 << 4)
	RPI_MODEL_B_PLUS         Model = (3 << 4)
	RPI_MODEL_B_PI_2         Model = (4 << 4)
	RPI_MODEL_ALPHA          Model = (5 << 4)
	RPI_MODEL_COMPUTE_MODULE Model = (6 << 4)
	RPI_MODEL_UNKNOWN        Model = (7 << 4)
	RPI_MODEL_B_PI_3         Model = (8 << 4)
	RPI_MODEL_ZERO           Model = (9 << 4)
)

const (
	RPI_PROCESSOR_UNKNOWN Processor = 0xFFFFFFFF
	RPI_PROCESSOR_BCM2835 Processor = (0 << 12)
	RPI_PROCESSOR_BCM2836 Processor = (1 << 12)
	RPI_PROCESSOR_BCM2837 Processor = (2 << 12)
)

const (
	RPI_PCB_UNKNOWN PCBRevision = 0
)

////////////////////////////////////////////////////////////////////////////////
// GLOBAL VARIABLES

var (
	productmap1 = map[uint32]Model{
		0x02: RPI_MODEL_B,
		0x03: RPI_MODEL_B,
		0x04: RPI_MODEL_B,
		0x05: RPI_MODEL_B,
		0x06: RPI_MODEL_B,
		0x07: RPI_MODEL_A,
		0x08: RPI_MODEL_A,
		0x09: RPI_MODEL_A,
		0x0D: RPI_MODEL_B,
		0x0E: RPI_MODEL_B,
		0x0F: RPI_MODEL_B,
		0x10: RPI_MODEL_B_PLUS,
		0x11: RPI_MODEL_COMPUTE_MODULE,
		0x12: RPI_MODEL_A_PLUS,
		0x13: RPI_MODEL_B_PLUS,
		0x14: RPI_MODEL_COMPUTE_MODULE,
		0x15: RPI_MODEL_A_PLUS,
	}
	pcbmap1 = map[uint32]PCBRevision{
		0x02: PCBRevision(1),
		0x03: PCBRevision(1),
		0x04: PCBRevision(2),
		0x05: PCBRevision(2),
		0x06: PCBRevision(2),
		0x07: PCBRevision(2),
		0x08: PCBRevision(2),
		0x09: PCBRevision(2),
		0x0D: PCBRevision(2),
		0x0E: PCBRevision(2),
		0x0F: PCBRevision(2),
		0x10: PCBRevision(1),
		0x11: PCBRevision(1),
		0x12: PCBRevision(1),
		0x13: PCBRevision(1),
		0x14: PCBRevision(1),
		0x15: PCBRevision(1),
	}
)

////////////////////////////////////////////////////////////////////////////////
// PUBLIC METHODS

// Returns the value of the warranty bit
func (this *Device) GetWarrantyBit() (bool, error) {
	revision, err := this.GetRevision()
	if err != nil {
		return false, err
	}

	// get warranty bit
	w := uint32(RPI_REVISION_WARRANTY_MASK)
	return ((revision & w) != 0), nil
}

// Return product model and PCB revision information
func (this *Device) GetModel() (Model, PCBRevision, error) {
	revision, err := this.GetRevision()
	if err != nil {
		return RPI_MODEL_UNKNOWN, RPI_PCB_UNKNOWN, err
	}

	// Decode differently depending on the format
	var model Model
	var pcb PCBRevision
	var ok bool

	revision = revision & ^RPI_REVISION_WARRANTY_MASK
	if (revision & RPI_REVISION_ENCODING_MASK) != 0 {
		// Raspberry Pi 2 style revision coding
		model = Model(revision & RPI_REVISION_PRODUCT_MASK)
		pcb = PCBRevision(revision & RPI_REVISION_PCB_MASK)
	} else {
		// Raspberry Pi 1 style revision coding
		model, ok = productmap1[revision]
		if ok == false {
			return RPI_MODEL_UNKNOWN, RPI_PCB_UNKNOWN, nil
		}
		pcb, ok = pcbmap1[revision]
		if ok == false {
			return RPI_MODEL_UNKNOWN, RPI_PCB_UNKNOWN, nil
		}
	}

	return model, pcb, nil
}

func (this *Device) GetProcessor() (Processor, error) {
	revision, err := this.GetRevision()
	if err != nil {
		return RPI_PROCESSOR_UNKNOWN, err
	}

	// Decode differently depending on the format
	revision = revision & ^RPI_REVISION_WARRANTY_MASK
	if (revision & RPI_REVISION_ENCODING_MASK) != 0 {
		// Raspberry Pi 2 style revision coding
		return Processor(revision & RPI_REVISION_PROCESSOR_MASK), nil
	} else {
		// Raspberry Pi 1 style revision coding
		return RPI_PROCESSOR_BCM2835, nil
	}
}

func (m Model) String() string {
	switch m {
	case RPI_MODEL_A:
		return "RPI_MODEL_A"
	case RPI_MODEL_B:
		return "RPI_MODEL_B"
	case RPI_MODEL_A_PLUS:
		return "RPI_MODEL_A_PLUS"
	case RPI_MODEL_B_PLUS:
		return "RPI_MODEL_B_PLUS"
	case RPI_MODEL_B_PI_2:
		return "RPI_MODEL_B_PI_2"
	case RPI_MODEL_B_PI_3:
		return "RPI_MODEL_B_PI_3"
	case RPI_MODEL_ALPHA:
		return "RPI_MODEL_ALPHA"
	case RPI_MODEL_COMPUTE_MODULE:
		return "RPI_MODEL_COMPUTE_MODULE"
	case RPI_MODEL_ZERO:
		return "RPI_MODEL_ZERO"
	default:
		return "[?? Unknown Model value]"
	}
}

func (p Processor) String() string {
	switch p {
	case RPI_PROCESSOR_BCM2835:
		return "RPI_PROCESSOR_BCM2835"
	case RPI_PROCESSOR_BCM2836:
		return "RPI_PROCESSOR_BCM2836"
	case RPI_PROCESSOR_BCM2837:
		return "RPI_PROCESSOR_BCM2837"
	default:
		return "[?? Unknown Processor value]"
	}
}

func (p PCBRevision) String() string {
	if p == RPI_PCB_UNKNOWN {
		return "[?? Unknwon PCBRevision value]"
	}
	return fmt.Sprintf("RPI_PCB_V%v", uint(p))
}
<commit_msg>Changes to model file<commit_after>/*
	Go Language Raspberry Pi Interface
	(c) Copyright David Thorpe 2016-2017
	All Rights Reserved

    Documentation http://djthorpe.github.io/gopi/
	For Licensing and Usage information, please see LICENSE.md
*/

package rpi /* import "github.com/djthorpe/gopi/device/rpi" */

////////////////////////////////////////////////////////////////////////////////
// TYPES

type (
	Model       uint32
	Processor   uint32
	PCBRevision uint32
)

////////////////////////////////////////////////////////////////////////////////
// CONSTANTS

const (
	RPI_REVISION_WARRANTY_MASK     uint32 = 0x03000000
	RPI_REVISION_ENCODING_MASK     uint32 = 0x00800000
	RPI_REVISION_PCB_MASK          uint32 = 0x0000000F
	RPI_REVISION_PRODUCT_MASK      uint32 = 0x00000FF0
	RPI_REVISION_PROCESSOR_MASK    uint32 = 0x0000F000
	RPI_REVISION_MANUFACTURER_MASK uint32 = 0x000F0000
	RPI_REVISION_MEMORY_MASK       uint32 = 0x00700000
)

const (
	RPI_MODEL_A              Model = (0 << 4)
	RPI_MODEL_B              Model = (1 << 4)
	RPI_MODEL_A_PLUS         Model = (2 << 4)
	RPI_MODEL_B_PLUS         Model = (3 << 4)
	RPI_MODEL_B_PI_2         Model = (4 << 4)
	RPI_MODEL_ALPHA          Model = (5 << 4)
	RPI_MODEL_COMPUTE_MODULE Model = (6 << 4)
	RPI_MODEL_UNKNOWN        Model = (7 << 4)
	RPI_MODEL_B_PI_3         Model = (8 << 4)
	RPI_MODEL_ZERO           Model = (9 << 4)
)

const (
	RPI_PROCESSOR_UNKNOWN Processor = 0xFFFFFFFF
	RPI_PROCESSOR_BCM2835 Processor = (0 << 12)
	RPI_PROCESSOR_BCM2836 Processor = (1 << 12)
	RPI_PROCESSOR_BCM2837 Processor = (2 << 12)
)

const (
	RPI_PCB_UNKNOWN PCBRevision = 0
)

////////////////////////////////////////////////////////////////////////////////
// GLOBAL VARIABLES

var (
	productmap1 = map[uint32]Model{
		0x02: RPI_MODEL_B,
		0x03: RPI_MODEL_B,
		0x04: RPI_MODEL_B,
		0x05: RPI_MODEL_B,
		0x06: RPI_MODEL_B,
		0x07: RPI_MODEL_A,
		0x08: RPI_MODEL_A,
		0x09: RPI_MODEL_A,
		0x0D: RPI_MODEL_B,
		0x0E: RPI_MODEL_B,
		0x0F: RPI_MODEL_B,
		0x10: RPI_MODEL_B_PLUS,
		0x11: RPI_MODEL_COMPUTE_MODULE,
		0x12: RPI_MODEL_A_PLUS,
		0x13: RPI_MODEL_B_PLUS,
		0x14: RPI_MODEL_COMPUTE_MODULE,
		0x15: RPI_MODEL_A_PLUS,
	}
	pcbmap1 = map[uint32]PCBRevision{
		0x02: PCBRevision(1),
		0x03: PCBRevision(1),
		0x04: PCBRevision(2),
		0x05: PCBRevision(2),
		0x06: PCBRevision(2),
		0x07: PCBRevision(2),
		0x08: PCBRevision(2),
		0x09: PCBRevision(2),
		0x0D: PCBRevision(2),
		0x0E: PCBRevision(2),
		0x0F: PCBRevision(2),
		0x10: PCBRevision(1),
		0x11: PCBRevision(1),
		0x12: PCBRevision(1),
		0x13: PCBRevision(1),
		0x14: PCBRevision(1),
		0x15: PCBRevision(1),
	}
)

////////////////////////////////////////////////////////////////////////////////
// PUBLIC METHODS

// Returns the value of the warranty bit
func (this *Device) GetWarrantyBit() (bool, error) {
	revision, err := this.GetRevision()
	if err != nil {
		return false, err
	}

	// get warranty bit
	w := uint32(RPI_REVISION_WARRANTY_MASK)
	return ((revision & w) != 0), nil
}

// Return product model and PCB revision information
func (this *Device) GetModel() (Model, PCBRevision, error) {
	revision, err := this.GetRevision()
	if err != nil {
		return RPI_MODEL_UNKNOWN, RPI_PCB_UNKNOWN, err
	}

	// Decode differently depending on the format
	var model Model
	var pcb PCBRevision
	var ok bool

	revision = revision & ^RPI_REVISION_WARRANTY_MASK
	if (revision & RPI_REVISION_ENCODING_MASK) != 0 {
		// Raspberry Pi 2 style revision coding
		model = Model(revision & RPI_REVISION_PRODUCT_MASK)
		pcb = PCBRevision(revision & RPI_REVISION_PCB_MASK)
	} else {
		// Raspberry Pi 1 style revision coding
		model, ok = productmap1[revision]
		if ok == false {
			return RPI_MODEL_UNKNOWN, RPI_PCB_UNKNOWN, nil
		}
		pcb, ok = pcbmap1[revision]
		if ok == false {
			return RPI_MODEL_UNKNOWN, RPI_PCB_UNKNOWN, nil
		}
	}

	return model, pcb, nil
}

func (this *Device) GetProcessor() (Processor, error) {
	revision, err := this.GetRevision()
	if err != nil {
		return RPI_PROCESSOR_UNKNOWN, err
	}

	// Decode differently depending on the format
	revision = revision & ^RPI_REVISION_WARRANTY_MASK
	if (revision & RPI_REVISION_ENCODING_MASK) != 0 {
		// Raspberry Pi 2 style revision coding
		return Processor(revision & RPI_REVISION_PROCESSOR_MASK), nil
	} else {
		// Raspberry Pi 1 style revision coding
		return RPI_PROCESSOR_BCM2835, nil
	}
}
<|endoftext|>


<commit_before>// Copyright (C) 2017, 2018 Damon Revoe. All rights reserved.
// Use of this source code is governed by the MIT
// license, which can be found in the LICENSE file.

package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/spf13/cobra"
)

func configurePackage(privateDir string, wp *workspaceParams,
	pd *packageDefinition, env []string) error {
	fmt.Println("[configure] " + pd.PackageName)

	conftab, err := readConftab(privateDir + "/" + conftabFilename)

	pkgRootDir := getGeneratedPkgRootDir(privateDir)
	pkgDir := pkgRootDir + "/" + pd.PackageName

	buildDir := getBuildDir(privateDir, wp)
	pkgBuildDir := buildDir + "/" + pd.PackageName

	relPkgSrcDir, err := filepath.Rel(pkgBuildDir, pkgDir)
	if err != nil {
		relPkgSrcDir, err = filepath.Abs(pkgDir)
		if err != nil {
			return err
		}
	}

	err = os.MkdirAll(pkgBuildDir, os.FileMode(0775))
	if err != nil {
		return nil
	}

	configurePathname := relPkgSrcDir + "/configure"

	configureArgs := conftab.getConfigureArgs(pd.PackageName)
	configureArgs = append(configureArgs, "--quiet")

	configureCmd := exec.Command(configurePathname, configureArgs...)
	configureCmd.Dir = pkgBuildDir
	configureCmd.Stdout = os.Stdout
	configureCmd.Stderr = os.Stderr
	configureCmd.Env = env
	if err := configureCmd.Run(); err != nil {
		return errors.New(configurePathname + ": " + err.Error())
	}

	return nil
}

func updateEnviron(buildDir string) ([]string, error) {
	env := os.Environ()

	absBuildDir, err := filepath.Abs(buildDir)
	if err != nil {
		return nil, err
	}

	configuredPackageDirs, err := ioutil.ReadDir(absBuildDir)
	if err != nil {
		return nil, err
	}

	if len(configuredPackageDirs) == 0 {
		return env, nil
	}

	var configuredPackagePathames []string

	for _, d := range configuredPackageDirs {
		configuredPackagePathames = append(
			configuredPackagePathames, absBuildDir+"/"+d.Name())
	}

	uninstalledConfigPath := strings.Join(configuredPackagePathames, ":")

	const pkgConfigPathVarName = "PKG_CONFIG_PATH"

	for i, v := range env {
		if strings.HasPrefix(v, pkgConfigPathVarName+"=") {
			split := strings.SplitAfterN(v, "=", 2)
			newValue := split[0] + uninstalledConfigPath
			if len(split[1]) > 0 {
				newValue += ":"
				newValue += split[1]
			}
			env[i] = newValue
			return env, nil
		}
	}

	return append(env, pkgConfigPathVarName+"="+uninstalledConfigPath), nil
}

func configurePackages(pkgNames []string) error {
	workspaceDir := getWorkspaceDir()

	wp, err := readWorkspaceParams(workspaceDir)
	if err != nil {
		return err
	}

	pi, err := readPackageDefinitions(workspaceDir, wp)
	if err != nil {
		return err
	}

	privateDir := getPrivateDir(workspaceDir)

	buildDir := getBuildDir(privateDir, wp)

	env, err := updateEnviron(buildDir)
	if err != nil {
		return err
	}

	if len(pkgNames) > 0 {
		pd, err := pi.getPackageByName(pkgNames[0])
		if err != nil {
			return err
		}
		return configurePackage(privateDir, wp, pd, env)
	}

	selection, err := readPackageSelection(pi, privateDir)
	if err != nil {
		return err
	}

	for _, pd := range selection {
		err = configurePackage(privateDir, wp, pd, env)
		if err != nil {
			return err
		}
	}

	return nil
}

// configureCmd represents the configure command
var configureCmd = &cobra.Command{
	Use:   "configure",
	Short: "Configure all selected packages or the specified package",
	Args:  cobra.MaximumNArgs(1),
	Run: func(_ *cobra.Command, args []string) {
		err := configurePackages(args)
		if err != nil {
			log.Fatal(err)
		}
	},
}

func init() {
	RootCmd.AddCommand(configureCmd)

	configureCmd.Flags().SortFlags = false
	addQuietFlag(configureCmd)
	addWorkspaceDirFlag(configureCmd)
}
<commit_msg>Prepare to generate per-package PKG_CONFIG_PATH<commit_after>// Copyright (C) 2017, 2018 Damon Revoe. All rights reserved.
// Use of this source code is governed by the MIT
// license, which can be found in the LICENSE file.

package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/spf13/cobra"
)

// configureEnv is a cache of environment variables that is
// reused when configuring multiple packages.
type configureEnv struct {
	envSansPkgConfigPath []string
	origPkgConfigPath    string // original value of PKG_CONFIG_PATH
}

const pkgConfigPathVarName = "PKG_CONFIG_PATH"

func dropEnvVar(env []string, i int) []string {
	if i < len(env)-1 {
		env[i] = env[len(env)-1]
	}
	return env[:len(env)-1]
}

func prepareConfigureEnv() *configureEnv {
	ce := &configureEnv{envSansPkgConfigPath: os.Environ()}

	for i, v := range ce.envSansPkgConfigPath {
		if strings.HasPrefix(v, pkgConfigPathVarName+"=") {
			ce.envSansPkgConfigPath = dropEnvVar(
				ce.envSansPkgConfigPath, i)
			ce.origPkgConfigPath = strings.SplitN(v, "=", 2)[1]
			break
		}
	}

	return ce
}

func (ce *configureEnv) makeEnv(buildDir string) ([]string, error) {
	absBuildDir, err := filepath.Abs(buildDir)
	if err != nil {
		return nil, err
	}

	configuredPackageDirs, err := ioutil.ReadDir(absBuildDir)
	if err != nil {
		return nil, err
	}

	var configuredPackagePathames []string

	for _, d := range configuredPackageDirs {
		configuredPackagePathames = append(
			configuredPackagePathames, absBuildDir+"/"+d.Name())
	}

	pkgConfigPath := strings.Join(configuredPackagePathames, ":")

	if len(pkgConfigPath) == 0 {
		if len(ce.origPkgConfigPath) == 0 {
			return ce.envSansPkgConfigPath, nil
		}
		pkgConfigPath = ce.origPkgConfigPath
	} else if len(ce.origPkgConfigPath) > 0 {
		pkgConfigPath += ":"
		pkgConfigPath += ce.origPkgConfigPath
	}

	return append(ce.envSansPkgConfigPath,
		pkgConfigPathVarName+"="+pkgConfigPath), nil
}

func configurePackage(privateDir string, wp *workspaceParams,
	pd *packageDefinition, cfgEnv *configureEnv) error {
	fmt.Println("[configure] " + pd.PackageName)

	conftab, err := readConftab(privateDir + "/" + conftabFilename)
	if err != nil {
		return err
	}

	pkgRootDir := getGeneratedPkgRootDir(privateDir)
	pkgDir := pkgRootDir + "/" + pd.PackageName

	buildDir := getBuildDir(privateDir, wp)

	env, err := cfgEnv.makeEnv(buildDir)
	if err != nil {
		return err
	}

	pkgBuildDir := buildDir + "/" + pd.PackageName

	relPkgSrcDir, err := filepath.Rel(pkgBuildDir, pkgDir)
	if err != nil {
		relPkgSrcDir, err = filepath.Abs(pkgDir)
		if err != nil {
			return err
		}
	}

	err = os.MkdirAll(pkgBuildDir, os.FileMode(0775))
	if err != nil {
		return nil
	}

	configurePathname := relPkgSrcDir + "/configure"

	configureArgs := conftab.getConfigureArgs(pd.PackageName)
	configureArgs = append(configureArgs, "--quiet")

	configureCmd := exec.Command(configurePathname, configureArgs...)
	configureCmd.Dir = pkgBuildDir
	configureCmd.Stdout = os.Stdout
	configureCmd.Stderr = os.Stderr
	configureCmd.Env = env
	if err := configureCmd.Run(); err != nil {
		return errors.New(configurePathname + ": " + err.Error())
	}

	return nil
}

func configurePackages(pkgNames []string) error {
	workspaceDir := getWorkspaceDir()

	wp, err := readWorkspaceParams(workspaceDir)
	if err != nil {
		return err
	}

	pi, err := readPackageDefinitions(workspaceDir, wp)
	if err != nil {
		return err
	}

	privateDir := getPrivateDir(workspaceDir)

	cfgEnv := prepareConfigureEnv()

	if len(pkgNames) > 0 {
		pd, err := pi.getPackageByName(pkgNames[0])
		if err != nil {
			return err
		}
		return configurePackage(privateDir, wp, pd, cfgEnv)
	}

	selection, err := readPackageSelection(pi, privateDir)
	if err != nil {
		return err
	}

	for _, pd := range selection {
		err = configurePackage(privateDir, wp, pd, cfgEnv)
		if err != nil {
			return err
		}
	}

	return nil
}

// configureCmd represents the configure command
var configureCmd = &cobra.Command{
	Use:   "configure",
	Short: "Configure all selected packages or the specified package",
	Args:  cobra.MaximumNArgs(1),
	Run: func(_ *cobra.Command, args []string) {
		err := configurePackages(args)
		if err != nil {
			log.Fatal(err)
		}
	},
}

func init() {
	RootCmd.AddCommand(configureCmd)

	configureCmd.Flags().SortFlags = false
	addQuietFlag(configureCmd)
	addWorkspaceDirFlag(configureCmd)
}
<|endoftext|>


<commit_before>/*
Copyright 2018 Google, Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package differs

import (
	"bytes"
	"context"
	"fmt"
	"math/rand"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/google/go-containerregistry/pkg/name"
	"github.com/google/go-containerregistry/pkg/v1"
	"github.com/google/go-containerregistry/pkg/v1/daemon"

	pkgutil "github.com/GoogleContainerTools/container-diff/pkg/util"
	"github.com/GoogleContainerTools/container-diff/util"
	godocker "github.com/fsouza/go-dockerclient"

	"github.com/nightlyone/lockfile"
	"github.com/sirupsen/logrus"
)

var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")

// daemonMutex is required to protect against other go-routines, as
// nightlyone/lockfile implements a recursive lock, which doesn't protect
// against other go-routines that have the same PID.  Note that the mutex
// *must* always be locked prior to the lockfile, and unlocked after.
var daemonMutex sync.Mutex

type RPMAnalyzer struct {
}

// Name returns the name of the analyzer.
func (a RPMAnalyzer) Name() string {
	return "RPMAnalyzer"
}

// Diff compares the installed rpm packages of image1 and image2.
func (a RPMAnalyzer) Diff(image1, image2 pkgutil.Image) (util.Result, error) {
	diff, err := singleVersionDiff(image1, image2, a)
	return diff, err
}

// Analyze collects information of the installed rpm packages on image.
func (a RPMAnalyzer) Analyze(image pkgutil.Image) (util.Result, error) {
	analysis, err := singleVersionAnalysis(image, a)
	return analysis, err
}

// getPackages returns a map of installed rpm package on image.
func (a RPMAnalyzer) getPackages(image pkgutil.Image) (map[string]util.PackageInfo, error) {
	path := image.FSPath
	packages := make(map[string]util.PackageInfo)
	if _, err := os.Stat(path); err != nil {
		// invalid image directory path
		return packages, err
	}

	// try to find the rpm binary in bin/ or usr/bin/
	rpmBinary := filepath.Join(path, "bin/rpm")
	if _, err := os.Stat(rpmBinary); err != nil {
		rpmBinary = filepath.Join(path, "usr/bin/rpm")
		if _, err = os.Stat(rpmBinary); err != nil {
			logrus.Errorf("Could not detect RPM binary in unpacked image %s", image.Source)
			return packages, nil
		}
	}

	return rpmDataFromContainer(image)
}

// rpmDataFromContainer runs image in a container, queries the data of
// installed rpm packages and returns a map of packages.
func rpmDataFromContainer(image pkgutil.Image) (map[string]util.PackageInfo, error) {
	packages := make(map[string]util.PackageInfo)

	client, err := godocker.NewClientFromEnv()
	if err != nil {
		return packages, err
	}
	if err := lock(); err != nil {
		return packages, err
	}

	imageName, err := loadImageToDaemon(image.Image)

	if err != nil {
		return packages, fmt.Errorf("Error loading image: %s", err)
	}
	unlock()

	defer client.RemoveImage(imageName)
	defer logrus.Infof("Removing image %s", imageName)

	contConf := godocker.Config{
		Entrypoint: []string{"rpm", "--nodigest", "--nosignature", "-qa", "--qf", "%{NAME}\t%{VERSION}\t%{SIZE}\n"},
		Image:      imageName,
	}

	hostConf := godocker.HostConfig{
		AutoRemove: true,
	}

	contOpts := godocker.CreateContainerOptions{Config: &contConf}
	container, err := client.CreateContainer(contOpts)
	if err != nil {
		return packages, err
	}
	logrus.Infof("Created container %s", container.ID)

	removeOpts := godocker.RemoveContainerOptions{
		ID: container.ID,
	}
	defer client.RemoveContainer(removeOpts)

	if err := client.StartContainer(container.ID, &hostConf); err != nil {
		return packages, err
	}

	exitCode, err := client.WaitContainer(container.ID)
	if err != nil {
		return packages, err
	}

	outBuf := new(bytes.Buffer)
	errBuf := new(bytes.Buffer)
	logOpts := godocker.LogsOptions{
		Context:      context.Background(),
		Container:    container.ID,
		Stdout:       true,
		Stderr:       true,
		OutputStream: outBuf,
		ErrorStream:  errBuf,
	}

	if err := client.Logs(logOpts); err != nil {
		return packages, err
	}

	if exitCode != 0 {
		return packages, fmt.Errorf("non-zero exit code %d: %s", exitCode, errBuf.String())
	}

	output := strings.Split(outBuf.String(), "\n")
	return parsePackageData(output)
}

// parsePackageData parses the package data of each line in rpmOutput and
// returns a map of packages.
func parsePackageData(rpmOutput []string) (map[string]util.PackageInfo, error) {
	packages := make(map[string]util.PackageInfo)

	for _, output := range rpmOutput {
		spl := strings.Split(output, "\t")
		if len(spl) != 3 {
			// ignore the empty (last) line
			if output != "" {
				logrus.Errorf("unexpected rpm-query output: '%s'", output)
			}
			continue
		}
		pkg := util.PackageInfo{}

		var err error
		pkg.Size, err = strconv.ParseInt(spl[2], 10, 64)
		if err != nil {
			return packages, fmt.Errorf("error converting package size: %s", spl[2])
		}

		pkg.Version = spl[1]
		packages[spl[0]] = pkg
	}

	return packages, nil
}

// loadImageToDaemon loads the image specified to the docker daemon.
func loadImageToDaemon(img v1.Image) (string, error) {
	tag := generateValidImageTag()
	resp, err := daemon.Write(tag, img, daemon.WriteOptions{})
	if err != nil {
		return "", err
	}
	logrus.Infof("daemon response: %s", resp)
	return tag.Name(), nil
}

// generate random image name until we find one that isn't in use
func generateValidImageTag() name.Tag {
	var tag name.Tag
	var err error
	var i int
	b := make([]rune, 12)
	for {
		for i = 0; i < len(b); i++ {
			b[i] = letters[rand.Intn(len(letters))]
		}
		tag, err = name.NewTag("rpm_test_image:"+string(b), name.WeakValidation)
		if err != nil {
			logrus.Warn(err.Error())
			continue
		}
		img, _ := daemon.Image(tag, &daemon.ReadOptions{})
		if img == nil {
			break
		}
	}
	return tag
}

// unlock returns the containerdiff file-system lock.  It is placed in the
// system's temporary directory to make sure it's accessible for all users in
// the system; no root required.
func getLockfile() (lockfile.Lockfile, error) {
	lockPath := filepath.Join(os.TempDir(), ".containerdiff.lock")
	lock, err := lockfile.New(lockPath)
	if err != nil {
		return lock, err
	}
	return lock, nil
}

// lock acquires the containerdiff file-system lock.
func lock() error {
	var err error
	var lock lockfile.Lockfile

	daemonMutex.Lock()
	lock, err = getLockfile()
	if err != nil {
		daemonMutex.Unlock()
		return fmt.Errorf("[lock] cannot init lockfile: %v", err)
	}

	// Try to acquire the lock and in case of a temporary error, sleep for
	// two seconds until the next retry (at most 10 times).  Return fatal
	// errors immediately, as we can't recover.
	for i := 0; i < 10; i++ {
		if err = lock.TryLock(); err != nil {
			switch err.(type) {
			case lockfile.TemporaryError:
				logrus.Debugf("[lock] busy: next retry in two seconds")
				time.Sleep(2 * time.Second)
			default:
				daemonMutex.Unlock()
				return fmt.Errorf("[lock] error acquiring lock: %s", err)
			}
		}
	}
	if err != nil {
		daemonMutex.Unlock()
		return fmt.Errorf("[lock] error acquiring lock: too many tries")
	}

	logrus.Debugf("[lock] lock acquired")
	return nil
}

// unlock releases the containerdiff file-system lock.  Note that errors can be
// ignored as there's no meaningful way to recover.
func unlock() error {
	lock, err := getLockfile()
	if err != nil {
		return fmt.Errorf("[unlock] cannot init lockfile: %v", err)
	}
	err = lock.Unlock()
	if err != nil {
		return fmt.Errorf("[unlock] error releasing lock: %s", err)
	}
	logrus.Debugf("[unlock] lock released")
	daemonMutex.Unlock()
	return nil
}
<commit_msg>Try to use local rpm binary to query rpmdb<commit_after>/*
Copyright 2018 Google, Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package differs

import (
	"bufio"
	"bytes"
	"context"
	"errors"
	"fmt"
	"math/rand"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/google/go-containerregistry/pkg/name"
	"github.com/google/go-containerregistry/pkg/v1"
	"github.com/google/go-containerregistry/pkg/v1/daemon"

	pkgutil "github.com/GoogleContainerTools/container-diff/pkg/util"
	"github.com/GoogleContainerTools/container-diff/util"
	godocker "github.com/fsouza/go-dockerclient"

	"github.com/nightlyone/lockfile"
	"github.com/sirupsen/logrus"
)

//RPM command to extract packages from the rpm database
var rpmCmd = []string{
	"rpm", "--nodigest", "--nosignature",
	"-qa", "--qf", "%{NAME}\t%{VERSION}\t%{SIZE}\n",
}
var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")

// daemonMutex is required to protect against other go-routines, as
// nightlyone/lockfile implements a recursive lock, which doesn't protect
// against other go-routines that have the same PID.  Note that the mutex
// *must* always be locked prior to the lockfile, and unlocked after.
var daemonMutex sync.Mutex

type RPMAnalyzer struct {
}

// Name returns the name of the analyzer.
func (a RPMAnalyzer) Name() string {
	return "RPMAnalyzer"
}

// Diff compares the installed rpm packages of image1 and image2.
func (a RPMAnalyzer) Diff(image1, image2 pkgutil.Image) (util.Result, error) {
	diff, err := singleVersionDiff(image1, image2, a)
	return diff, err
}

// Analyze collects information of the installed rpm packages on image.
func (a RPMAnalyzer) Analyze(image pkgutil.Image) (util.Result, error) {
	analysis, err := singleVersionAnalysis(image, a)
	return analysis, err
}

// getPackages returns a map of installed rpm package on image.
func (a RPMAnalyzer) getPackages(image pkgutil.Image) (map[string]util.PackageInfo, error) {
	path := image.FSPath
	packages := make(map[string]util.PackageInfo)
	if _, err := os.Stat(path); err != nil {
		// invalid image directory path
		return packages, err
	}

	// try to find the rpm binary in bin/ or usr/bin/
	rpmBinary := filepath.Join(path, "bin/rpm")
	if _, err := os.Stat(rpmBinary); err != nil {
		rpmBinary = filepath.Join(path, "usr/bin/rpm")
		if _, err = os.Stat(rpmBinary); err != nil {
			logrus.Errorf("Could not detect RPM binary in unpacked image %s", image.Source)
			return packages, nil
		}
	}

	packages, err := rpmDataFromImageFS(image)
	if err != nil {
		logrus.Warn("Trying to run the RPM binary of the image in a container")
		return rpmDataFromContainer(image)
	}
	return packages, err
}

// rpmDataFromImageFS runs a local rpm binary, if any, to query the image
// rpmdb and returns a map of installed packages.
func rpmDataFromImageFS(image pkgutil.Image) (map[string]util.PackageInfo, error) {
	packages := make(map[string]util.PackageInfo)
	// Check there is an executable rpm tool in host
	if err := exec.Command("rpm", "--version").Run(); err != nil {
		logrus.Warn("No RPM binary in host")
		return packages, err
	}
	dbPath, err := rpmDBPath(image.FSPath)
	if err != nil {
		logrus.Warnf("Couldn't find RPM database: %s", err.Error())
		return packages, err
	}
	cmdArgs := append([]string{"--root", image.FSPath, "--dbpath", dbPath}, rpmCmd[1:]...)
	out, err := exec.Command(rpmCmd[0], cmdArgs...).Output()
	if err != nil {
		logrus.Warnf("RPM call failed: %s", err.Error())
		return packages, err
	}
	output := strings.Split(string(out), "\n")
	return parsePackageData(output)
}

// rpmDBPath tries to get the RPM database path from the /usr/lib/rpm/macros
// file in the image rootfs.
func rpmDBPath(rootFSPath string) (string, error) {
	rpmMacros, err := os.Open(filepath.Join(rootFSPath, "usr/lib/rpm/macros"))
	if err != nil {
		return "", err
	}
	defer rpmMacros.Close()

	scanner := bufio.NewScanner(rpmMacros)
	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		if strings.HasPrefix(line, "%_dbpath") {
			fields := strings.Fields(line)
			if len(fields) < 2 {
				break
			}
			out, err := exec.Command("rpm", "-E", fields[1]).Output()
			if err != nil {
				return "", err
			}
			dbPath := strings.TrimRight(string(out), "\r\n")
			_, err = os.Stat(filepath.Join(rootFSPath, dbPath))
			return dbPath, err
		}
	}
	return "", errors.New("Failed parsing macros file")
}

// rpmDataFromContainer runs image in a container, queries the data of
// installed rpm packages and returns a map of packages.
func rpmDataFromContainer(image pkgutil.Image) (map[string]util.PackageInfo, error) {
	packages := make(map[string]util.PackageInfo)

	client, err := godocker.NewClientFromEnv()
	if err != nil {
		return packages, err
	}
	if err := lock(); err != nil {
		return packages, err
	}

	imageName, err := loadImageToDaemon(image.Image)

	if err != nil {
		return packages, fmt.Errorf("Error loading image: %s", err)
	}
	unlock()

	defer client.RemoveImage(imageName)
	defer logrus.Infof("Removing image %s", imageName)

	contConf := godocker.Config{
		Entrypoint: rpmCmd,
		Image:      imageName,
	}

	hostConf := godocker.HostConfig{
		AutoRemove: true,
	}

	contOpts := godocker.CreateContainerOptions{Config: &contConf}
	container, err := client.CreateContainer(contOpts)
	if err != nil {
		return packages, err
	}
	logrus.Infof("Created container %s", container.ID)

	removeOpts := godocker.RemoveContainerOptions{
		ID: container.ID,
	}
	defer client.RemoveContainer(removeOpts)

	if err := client.StartContainer(container.ID, &hostConf); err != nil {
		return packages, err
	}

	exitCode, err := client.WaitContainer(container.ID)
	if err != nil {
		return packages, err
	}

	outBuf := new(bytes.Buffer)
	errBuf := new(bytes.Buffer)
	logOpts := godocker.LogsOptions{
		Context:      context.Background(),
		Container:    container.ID,
		Stdout:       true,
		Stderr:       true,
		OutputStream: outBuf,
		ErrorStream:  errBuf,
	}

	if err := client.Logs(logOpts); err != nil {
		return packages, err
	}

	if exitCode != 0 {
		return packages, fmt.Errorf("non-zero exit code %d: %s", exitCode, errBuf.String())
	}

	output := strings.Split(outBuf.String(), "\n")
	return parsePackageData(output)
}

// parsePackageData parses the package data of each line in rpmOutput and
// returns a map of packages.
func parsePackageData(rpmOutput []string) (map[string]util.PackageInfo, error) {
	packages := make(map[string]util.PackageInfo)

	for _, output := range rpmOutput {
		spl := strings.Split(output, "\t")
		if len(spl) != 3 {
			// ignore the empty (last) line
			if output != "" {
				logrus.Errorf("unexpected rpm-query output: '%s'", output)
			}
			continue
		}
		pkg := util.PackageInfo{}

		var err error
		pkg.Size, err = strconv.ParseInt(spl[2], 10, 64)
		if err != nil {
			return packages, fmt.Errorf("error converting package size: %s", spl[2])
		}

		pkg.Version = spl[1]
		packages[spl[0]] = pkg
	}

	return packages, nil
}

// loadImageToDaemon loads the image specified to the docker daemon.
func loadImageToDaemon(img v1.Image) (string, error) {
	tag := generateValidImageTag()
	resp, err := daemon.Write(tag, img, daemon.WriteOptions{})
	if err != nil {
		return "", err
	}
	logrus.Infof("daemon response: %s", resp)
	return tag.Name(), nil
}

// generate random image name until we find one that isn't in use
func generateValidImageTag() name.Tag {
	var tag name.Tag
	var err error
	var i int
	b := make([]rune, 12)
	for {
		for i = 0; i < len(b); i++ {
			b[i] = letters[rand.Intn(len(letters))]
		}
		tag, err = name.NewTag("rpm_test_image:"+string(b), name.WeakValidation)
		if err != nil {
			logrus.Warn(err.Error())
			continue
		}
		img, _ := daemon.Image(tag, &daemon.ReadOptions{})
		if img == nil {
			break
		}
	}
	return tag
}

// unlock returns the containerdiff file-system lock.  It is placed in the
// system's temporary directory to make sure it's accessible for all users in
// the system; no root required.
func getLockfile() (lockfile.Lockfile, error) {
	lockPath := filepath.Join(os.TempDir(), ".containerdiff.lock")
	lock, err := lockfile.New(lockPath)
	if err != nil {
		return lock, err
	}
	return lock, nil
}

// lock acquires the containerdiff file-system lock.
func lock() error {
	var err error
	var lock lockfile.Lockfile

	daemonMutex.Lock()
	lock, err = getLockfile()
	if err != nil {
		daemonMutex.Unlock()
		return fmt.Errorf("[lock] cannot init lockfile: %v", err)
	}

	// Try to acquire the lock and in case of a temporary error, sleep for
	// two seconds until the next retry (at most 10 times).  Return fatal
	// errors immediately, as we can't recover.
	for i := 0; i < 10; i++ {
		if err = lock.TryLock(); err != nil {
			switch err.(type) {
			case lockfile.TemporaryError:
				logrus.Debugf("[lock] busy: next retry in two seconds")
				time.Sleep(2 * time.Second)
			default:
				daemonMutex.Unlock()
				return fmt.Errorf("[lock] error acquiring lock: %s", err)
			}
		}
	}
	if err != nil {
		daemonMutex.Unlock()
		return fmt.Errorf("[lock] error acquiring lock: too many tries")
	}

	logrus.Debugf("[lock] lock acquired")
	return nil
}

// unlock releases the containerdiff file-system lock.  Note that errors can be
// ignored as there's no meaningful way to recover.
func unlock() error {
	lock, err := getLockfile()
	if err != nil {
		return fmt.Errorf("[unlock] cannot init lockfile: %v", err)
	}
	err = lock.Unlock()
	if err != nil {
		return fmt.Errorf("[unlock] error releasing lock: %s", err)
	}
	logrus.Debugf("[unlock] lock released")
	daemonMutex.Unlock()
	return nil
}
<|endoftext|>


<commit_before>package eval

import (
	"sync"
	"time"

	"github.com/gansoi/gansoi/checks"
	"github.com/gansoi/gansoi/database"
)

type (
	// Evaluation describes the current state of a check.
	Evaluation struct {
		ID          int64                `json:"id" storm:"id,increment"`
		CheckHostID string               `json:"check_host_id" storm:"index"`
		CheckID     string               `json:"check_id" storm:"index"`
		HostID      string               `json:"host_id"`
		History     States               `json:"history"`
		State       State                `json:"state"`
		Start       time.Time            `json:"start"`
		End         time.Time            `json:"end"`
		Hosts       map[string]State     `json:"hosts"`
		Results     []checks.CheckResult `json:"-"`
	}
)

var (
	cacheLock sync.RWMutex
	cache     = make(map[string]*Evaluation)
)

func init() {
	database.RegisterType(Evaluation{})
}

// NewEvaluation returns a new evaluation.
func NewEvaluation(clock time.Time, result *checks.CheckResult) *Evaluation {
	return &Evaluation{
		ID:          time.Now().UnixNano(),
		CheckHostID: result.CheckHostID,
		CheckID:     result.CheckID,
		HostID:      result.HostID,
		Start:       clock,
		End:         clock,
		Hosts:       make(map[string]State),
	}
}

func (e *Evaluation) Save(db database.Writer) error {
	err := db.Save(e)

	if err == nil {
		cacheLock.Lock()
		cache[e.CheckHostID] = e
		cacheLock.Unlock()
	}

	return err
}

// LatestEvaluation retrieves the latest evaluation if any.
func LatestEvaluation(db database.Reader, result *checks.CheckResult) (*Evaluation, error) {
	if result.CheckHostID == "" {
		return nil, database.ErrNotFound
	}

	cacheLock.RLock()
	eval, found := cache[result.CheckHostID]
	cacheLock.RUnlock()

	if found {
		return eval, nil
	}

	var results []Evaluation
	db.Find("CheckHostID", result.CheckHostID, &results, 1, 0, true)
	if len(results) != 1 {
		return nil, database.ErrNotFound
	}

	return &results[0], nil
}
<commit_msg>Added a Final() method that can determine if an evaluation is for a check or a host.<commit_after>package eval

import (
	"sync"
	"time"

	"github.com/gansoi/gansoi/checks"
	"github.com/gansoi/gansoi/database"
)

type (
	// Evaluation describes the current state of a check.
	Evaluation struct {
		ID          int64                `json:"id" storm:"id,increment"`
		CheckHostID string               `json:"check_host_id" storm:"index"`
		CheckID     string               `json:"check_id" storm:"index"`
		HostID      string               `json:"host_id"`
		History     States               `json:"history"`
		State       State                `json:"state"`
		Start       time.Time            `json:"start"`
		End         time.Time            `json:"end"`
		Hosts       map[string]State     `json:"hosts"`
		Results     []checks.CheckResult `json:"-"`
	}
)

var (
	cacheLock sync.RWMutex
	cache     = make(map[string]*Evaluation)
)

func init() {
	database.RegisterType(Evaluation{})
}

// NewEvaluation returns a new evaluation.
func NewEvaluation(clock time.Time, result *checks.CheckResult) *Evaluation {
	return &Evaluation{
		ID:          time.Now().UnixNano(),
		CheckHostID: result.CheckHostID,
		CheckID:     result.CheckID,
		HostID:      result.HostID,
		Start:       clock,
		End:         clock,
		Hosts:       make(map[string]State),
	}
}

func (e *Evaluation) Save(db database.Writer) error {
	err := db.Save(e)

	if err == nil {
		cacheLock.Lock()
		cache[e.CheckHostID] = e
		cacheLock.Unlock()
	}

	return err
}

// LatestEvaluation retrieves the latest evaluation if any.
func LatestEvaluation(db database.Reader, result *checks.CheckResult) (*Evaluation, error) {
	if result.CheckHostID == "" {
		return nil, database.ErrNotFound
	}

	cacheLock.RLock()
	eval, found := cache[result.CheckHostID]
	cacheLock.RUnlock()

	if found {
		return eval, nil
	}

	var results []Evaluation
	db.Find("CheckHostID", result.CheckHostID, &results, 1, 0, true)
	if len(results) != 1 {
		return nil, database.ErrNotFound
	}

	return &results[0], nil
}

// Final returns true if the evaluation is for a check and not for a host-part
// of a check.
func (e *Evaluation) Final() bool {
	return e.CheckHostID == checks.CheckHostID(e.CheckID, "")
}
<|endoftext|>


<commit_before>package design

import (
	. "github.com/goadesign/goa/design"
	. "github.com/goadesign/goa/design/apidsl"
)
var resourcePrefix string = "hy_"

//Resources group related API endpoints
// This is request object
// Payload is sending data to server

var _ = Resource(resourcePrefix + "user", func() {

	DefaultMedia(User) //Response Media Type
	BasePath("/user")

	//Actions define a single API endpoint
	// This name should be unique
	Action("UserList", func() {
		Routing(
			GET(""),
		)
		Description("Retrieve all users.")
		//Responses define the shape and status code
		//Response(OK, CollectionOf(User))
		Response(OK, func() {
			Media(CollectionOf(User, func() {
				View("default")
				View("tiny")
			}))
			//	Headers(func() {                // Headers list the response HTTP headers
			//		Header("X-Request-Id")  // Header syntax is identical to Attribute's
			//	})
		})

		//Headers(func() {                  // Headers describe relevant action headers
		//    Header("Authorization", String)
		//    Header("X-Account", Integer)
		//    Required("Authorization", "X-Account")
		//})
	})

	Action("GetUser", func() {
		Routing(
			GET("/:userID"),
		)
		Description("Retrieve user with given id.")
		Params(func() {
			Param("userID", Integer, "User ID", func() {
				Minimum(1)
			})
		})

		Response(OK)
		Response(NotFound)
		Response(BadRequest, ErrorMedia)
	})

	Action("CreateUser", func() {
		Routing(
			POST(""),
		)
		Description("Create new user")
		//Payload(func() {
		//	Member("name")
		//	Required("name")
		//})
		Payload(UserPayload, func() {
			Required("name", "email")
		})

		Response(Created, "/user/[0-9]+") //[??]
		Response(BadRequest, ErrorMedia)
	})

	Action("UpdateUser", func() {
		Routing(
			PUT("/:userID"),
		)
		Description("Change user properties")
		Params(func() {
			Param("userID", Integer, "User ID")
		})
		Payload(UserPayload)

		Response(NoContent)
		Response(NotFound)
		Response(BadRequest, ErrorMedia)
	})

	Action("DeleteUser", func() {
		Routing(
			DELETE("/:userID"),
		)
		Params(func() {
			Param("userID", Integer, "User ID")
		})
		Response(NoContent)
		Response(NotFound)
		Response(BadRequest, ErrorMedia)
	})
})

var _ = Resource(resourcePrefix + "company", func() {

	DefaultMedia(Company)
	BasePath("/company")

	// Parent sets the resource parent
	//Parent("user")  //TODO: how does it work??

	Action("CompanyList", func() {
		Routing(
			GET(""),
		)
		Description("List all companies")
		Response(OK, func() {
			Media(CollectionOf(Company, func() {
				View("default")
				View("tiny")
			}))
		})
		Response(NotFound)
		Response(BadRequest, ErrorMedia)
	})

	Action("GetCompany", func() {
		Routing(
			GET("/:companyID"),
		)
		Description("Retrieve company with given id")
		Params(func() {
			Param("companyID", Integer)
		})
		Response(OK)
		Response(NotFound)
		Response(BadRequest, ErrorMedia)
	})

	//TODO: is it for websocket??
	//Action("watch", func() {
	//	Routing(
	//		GET("/:bottleID/watch"),
	//	)
	//	Scheme("ws")
	//	Description("Retrieve bottle with given id")
	//	Params(func() {
	//		Param("bottleID", Integer)
	//	})
	//	Response(SwitchingProtocols)
	//	Response(BadRequest, ErrorMedia)
	//})

	Action("CreateCompany", func() {
		Routing(
			POST(""),
		)
		Description("Record new company")
		Payload(CompanyPayload, func() {
			Required("name", "country", "address")
		})
		Response(Created, "^/user/[0-9]+/company/[0-9]+$")
		Response(NotFound)
		Response(BadRequest, ErrorMedia)
	})

	Action("UpdateCompany", func() {
		Routing(
			PUT("/:companyID"),
		)
		Params(func() {
			Param("companyID", Integer)
		})
		Payload(CompanyPayload)
		Response(NoContent)
		Response(NotFound)
		Response(BadRequest, ErrorMedia)
	})

	//TODO:somohow confliction error occur
	//=> type miss
	Action("DeleteCompany", func() {
		Routing(
			DELETE("/:companyID"),
		)
		Params(func() {
			Param("companyID", Integer)
		})
		Response(NoContent)
		Response(NotFound)
		Response(BadRequest, ErrorMedia)
	})
})

var _ = Resource("public", func() {
	Origin("*", func() {
		Methods("GET", "OPTIONS")
	})
	Files("/ui", "public/html/index.html")
})

var _ = Resource("js", func() {
	Origin("*", func() {
		Methods("GET", "OPTIONS")
	})
	Files("/js/*filepath", "public/js")
})

var _ = Resource("swagger", func() {
	Origin("*", func() {
		Methods("GET", "OPTIONS")
	})
	Files("/swagger.json", "public/swagger/swagger.json")
})
<commit_msg>added minor commneted out<commit_after>package design

import (
	. "github.com/goadesign/goa/design"
	. "github.com/goadesign/goa/design/apidsl"
)

var resourcePrefix string = "hy_"

//Resources group related API endpoints
// This is request object
// Payload is sending data to server

var _ = Resource(resourcePrefix+"user", func() {

	DefaultMedia(User) //Response Media Type
	BasePath("/user")

	//Actions define a single API endpoint
	// This name should be unique
	Action("UserList", func() {
		Routing(
			GET(""),
		)
		Description("Retrieve all users.")
		//Responses define the shape and status code
		//Response(OK, CollectionOf(User))
		Response(OK, func() {
			Media(CollectionOf(User, func() {
				View("default")
				View("tiny")
			}))
			//	Headers(func() {                // Headers list the response HTTP headers
			//		Header("X-Request-Id")  // Header syntax is identical to Attribute's
			//      Required("X-Request-Id")
			//	})
		})

		//HTTP Request Header
		//Headers(func() {                  // Headers describe relevant action headers
		//    Header("Authorization", String)
		//    Header("X-Account", Integer)
		//    Required("Authorization", "X-Account")
		//})
	})

	Action("GetUser", func() {
		Routing(
			GET("/:userID"),
		)
		Description("Retrieve user with given id.")
		Params(func() {
			Param("userID", Integer, "User ID", func() {
				Minimum(1)
			})
		})

		Response(OK)
		Response(NotFound)
		Response(BadRequest, ErrorMedia)
	})

	Action("CreateUser", func() {
		Routing(
			POST(""),
		)
		Description("Create new user")
		//Payload(func() {
		//	Member("name")
		//	Required("name")
		//})
		Payload(UserPayload, func() {
			Required("name", "email")
		})

		Response(Created, "/user/[0-9]+") //[??]
		Response(BadRequest, ErrorMedia)
	})

	Action("UpdateUser", func() {
		Routing(
			PUT("/:userID"),
		)
		Description("Change user properties")
		Params(func() {
			Param("userID", Integer, "User ID")
		})
		Payload(UserPayload)

		Response(NoContent)
		Response(NotFound)
		Response(BadRequest, ErrorMedia)
	})

	Action("DeleteUser", func() {
		Routing(
			DELETE("/:userID"),
		)
		Params(func() {
			Param("userID", Integer, "User ID")
		})
		Response(NoContent)
		Response(NotFound)
		Response(BadRequest, ErrorMedia)
	})
})

var _ = Resource(resourcePrefix+"company", func() {

	DefaultMedia(Company)
	BasePath("/company")

	// Parent sets the resource parent
	//Parent("user")  //TODO: how does it work??

	Action("CompanyList", func() {
		Routing(
			GET(""),
		)
		Description("List all companies")
		Response(OK, func() {
			Media(CollectionOf(Company, func() {
				View("default")
				View("tiny")
			}))
		})
		Response(NotFound)
		Response(BadRequest, ErrorMedia)
	})

	Action("GetCompany", func() {
		Routing(
			GET("/:companyID"),
		)
		Description("Retrieve company with given id")
		Params(func() {
			Param("companyID", Integer)
		})
		Response(OK)
		Response(NotFound)
		Response(BadRequest, ErrorMedia)
	})

	//TODO: is it for websocket??
	//Action("watch", func() {
	//	Routing(
	//		GET("/:bottleID/watch"),
	//	)
	//	Scheme("ws")
	//	Description("Retrieve bottle with given id")
	//	Params(func() {
	//		Param("bottleID", Integer)
	//	})
	//	Response(SwitchingProtocols)
	//	Response(BadRequest, ErrorMedia)
	//})

	Action("CreateCompany", func() {
		Routing(
			POST(""),
		)
		Description("Record new company")
		Payload(CompanyPayload, func() {
			Required("name", "country", "address")
		})
		Response(Created, "^/user/[0-9]+/company/[0-9]+$")
		Response(NotFound)
		Response(BadRequest, ErrorMedia)
	})

	Action("UpdateCompany", func() {
		Routing(
			PUT("/:companyID"),
		)
		Params(func() {
			Param("companyID", Integer)
		})
		Payload(CompanyPayload)
		Response(NoContent)
		Response(NotFound)
		Response(BadRequest, ErrorMedia)
	})

	//TODO:somohow confliction error occur
	//=> type miss
	Action("DeleteCompany", func() {
		Routing(
			DELETE("/:companyID"),
		)
		Params(func() {
			Param("companyID", Integer)
		})
		Response(NoContent)
		Response(NotFound)
		Response(BadRequest, ErrorMedia)
	})
})

var _ = Resource("public", func() {
	Origin("*", func() {
		Methods("GET", "OPTIONS")
	})
	Files("/ui", "public/html/index.html")
})

var _ = Resource("js", func() {
	Origin("*", func() {
		Methods("GET", "OPTIONS")
	})
	Files("/js/*filepath", "public/js")
})

var _ = Resource("swagger", func() {
	Origin("*", func() {
		Methods("GET", "OPTIONS")
	})
	Files("/swagger.json", "public/swagger/swagger.json")
})
<|endoftext|>


<commit_before>package fakku

import (
	"testing"
)

func TestContentGetSimple_1(t *testing.T) {
	// Test the manga listed in the API docs
	compareName := "Right now, while cleaning the pool"
	compareUrl := "http://www.fakku.net/manga/right-now-while-cleaning-the-pool"
	compareTag := "Vanilla"
	c, err := GetContentInformation("manga", "right-now-while-cleaning-the-pool")
	if err != nil {
		t.Error(err)
	}
	if c.Name != compareName {
		t.Errorf("Expected: %s, Got: %s", compareName, c.Name)
	}
	if c.Url != compareUrl {
		t.Errorf("Expected: %s, Got: %s", compareUrl, c.Url)
	}
	foundTag := false
	for i := 0; i < len(c.Tags); i++ {
		if c.Tags[i].Attribute == compareTag {
			foundTag = true
			break
		}
	}
	if !foundTag {
		t.Errorf("Didn't find tag %s but should have!", compareTag)
	}
}

func TestContentGetComments_1(t *testing.T) {
	/* tests to make sure that the functions work */
	_, err := GetContentComments("manga", "right-now-while-cleaning-the-pool")
	if err != nil {
		t.Error(err)
	}
}

func TestContentGetReadOnline_1(t *testing.T) {
	onlineContent, err := GetContentReadOnline("manga", "right-now-while-cleaning-the-pool")
	if err != nil {
		t.Error(err)
	}

	if len(onlineContent.Pages) != 25 {
		t.Errorf("Expected 25 pages, got %d pages", len(onlineContent.Pages))
	}
}

func TestGetContentDownloads_1(t *testing.T) {
	downloadPoster := "Jacob"
	downloads, err := GetContentDownloads("manga", "right-now-while-cleaning-the-pool")
	if err != nil {
		t.Error(err)
	}
	if downloads.Total != 1 {
		t.Errorf("Expected 1 page, got %d pages", downloads.Total)
	} else {
		if len(downloads.Downloads) != 1 {
			t.Errorf("Expected 1 download, got %d downloads", len(downloads.Downloads))
		} else {
			if downloads.Downloads[0].Poster != downloadPoster {
				t.Errorf("Expected download poster: %s, got %s", downloadPoster, downloads.Downloads[0].Poster)
			}
		}
	}
}
<commit_msg>Streamlined content_test.go with consts<commit_after>package fakku

import (
	"testing"
)

const (
	// Test the manga listed in the API docs
	ContentTestingCategory    = "manga"
	ContentTestingName        = "right-now-while-cleaning-the-pool"
	ContentTestingUrl         = "http://www.fakku.net/manga/right-now-while-cleaning-the-pool"
	ContentTestingTag         = "Vanilla"
	ContentTestingDisplayName = "Right now, while cleaning the pool"
	ContentTestingPoster      = "Jacob"
)

func TestContentGetSimple_1(t *testing.T) {
	c, err := GetContentInformation(ContentTestingCategory, ContentTestingName)
	if err != nil {
		t.Error(err)
	}
	if c.Name != ContentTestingDisplayName {
		t.Errorf("Expected: %s, Got: %s", ContentTestingDisplayName, c.Name)
	}
	if c.Url != ContentTestingUrl {
		t.Errorf("Expected: %s, Got: %s", ContentTestingUrl, c.Url)
	}
	foundTag := false
	for i := 0; i < len(c.Tags); i++ {
		if c.Tags[i].Attribute == ContentTestingTag {
			foundTag = true
			break
		}
	}
	if !foundTag {
		t.Errorf("Didn't find tag %s but should have!", ContentTestingTag)
	}
}

func TestContentGetComments_1(t *testing.T) {
	/* tests to make sure that the functions work */
	_, err := GetContentComments(ContentTestingCategory, ContentTestingName)
	if err != nil {
		t.Error(err)
	}
}

func TestContentGetReadOnline_1(t *testing.T) {
	onlineContent, err := GetContentReadOnline(ContentTestingCategory, ContentTestingName)
	if err != nil {
		t.Error(err)
	}

	if len(onlineContent.Pages) != 25 {
		t.Errorf("Expected 25 pages, got %d pages", len(onlineContent.Pages))
	}
}

func TestGetContentDownloads_1(t *testing.T) {
	downloads, err := GetContentDownloads(ContentTestingCategory, ContentTestingName)
	if err != nil {
		t.Error(err)
	}
	if downloads.Total != 1 {
		t.Errorf("Expected 1 page, got %d pages", downloads.Total)
	} else {
		if len(downloads.Downloads) != 1 {
			t.Errorf("Expected 1 download, got %d downloads", len(downloads.Downloads))
		} else {
			if downloads.Downloads[0].Poster != ContentTestingPoster {
				t.Errorf("Expected download poster: %s, got %s", ContentTestingPoster, downloads.Downloads[0].Poster)
			}
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package context

import (
	"net/http"
	"testing"
)

type keyType int

const (
	key1 keyType = iota
	key2
)

func TestContext(t *testing.T) {
	assertEqual := func(val interface{}, exp interface{}) {
		if val != exp {
			t.Errorf("Expected %v, got %v.", exp, val)
		}
	}

	r, _ := http.NewRequest("GET", "http://localhost:8080/", nil)
	emptyR, _ := http.NewRequest("GET", "http://localhost:8080/", nil)

	// Get()
	assertEqual(Get(r, key1), nil)

	// Set()
	Set(r, key1, "1")
	assertEqual(Get(r, key1), "1")
	assertEqual(len(data[r]), 1)

	Set(r, key2, "2")
	assertEqual(Get(r, key2), "2")
	assertEqual(len(data[r]), 2)

	//GetOk
	value, ok := GetOk(r, key1)
	assertEqual(value, "1")
	assertEqual(ok, true)

	value, ok = GetOk(r, "not exists")
	assertEqual(value, nil)
	assertEqual(ok, false)

	Set(r, "nil value", nil)
	value, ok = GetOk(r, "nil value")
	assertEqual(value, nil)
	assertEqual(ok, true)

	// GetAll()
	values := GetAll(r)
	assertEqual(len(values), 3)

	// GetAll() for empty request
	values = GetAll(emptyR)
	if values != nil {
		t.Error("GetAll didn't return nil value for invalid request")
	}

	// GetAllOk()
	values, ok = GetAllOk(r)
	assertEqual(len(values), 3)
	assertEqual(ok, true)

	// GetAllOk() for empty request
	values, ok = GetAllOk(emptyR)
	assertEqual(value, nil)
	assertEqual(ok, false)

	// Delete()
	Delete(r, key1)
	assertEqual(Get(r, key1), nil)
	assertEqual(len(data[r]), 2)

	Delete(r, key2)
	assertEqual(Get(r, key2), nil)
	assertEqual(len(data[r]), 1)

	// Clear()
	Clear(r)
	assertEqual(len(data), 0)
}

func parallelReader(r *http.Request, key string, iterations int, wait, done chan struct{}) {
	<-wait
	for i := 0; i < iterations; i++ {
		Get(r, key)
	}
	done <- struct{}{}

}

func parallelWriter(r *http.Request, key, value string, iterations int, wait, done chan struct{}) {
	<-wait
	for i := 0; i < iterations; i++ {
		Set(r, key, value)
	}
	done <- struct{}{}

}

func benchmarkMutex(b *testing.B, numReaders, numWriters, iterations int) {

	b.StopTimer()
	r, _ := http.NewRequest("GET", "http://localhost:8080/", nil)
	done := make(chan struct{})
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		wait := make(chan struct{})

		for i := 0; i < numReaders; i++ {
			go parallelReader(r, "test", iterations, wait, done)
		}

		for i := 0; i < numWriters; i++ {
			go parallelWriter(r, "test", "123", iterations, wait, done)
		}

		close(wait)

		for i := 0; i < numReaders+numWriters; i++ {
			<-done
		}

	}

}

func BenchmarkMutexSameReadWrite1(b *testing.B) {
	benchmarkMutex(b, 1, 1, 32)
}
func BenchmarkMutexSameReadWrite2(b *testing.B) {
	benchmarkMutex(b, 2, 2, 32)
}
func BenchmarkMutexSameReadWrite4(b *testing.B) {
	benchmarkMutex(b, 4, 4, 32)
}
func BenchmarkMutex1(b *testing.B) {
	benchmarkMutex(b, 2, 8, 32)
}
func BenchmarkMutex2(b *testing.B) {
	benchmarkMutex(b, 16, 4, 64)
}
func BenchmarkMutex3(b *testing.B) {
	benchmarkMutex(b, 1, 2, 128)
}
func BenchmarkMutex4(b *testing.B) {
	benchmarkMutex(b, 128, 32, 256)
}
func BenchmarkMutex5(b *testing.B) {
	benchmarkMutex(b, 1024, 2048, 64)
}
func BenchmarkMutex6(b *testing.B) {
	benchmarkMutex(b, 2048, 1024, 512)
}
<commit_msg>Fix typo<commit_after>// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package context

import (
	"net/http"
	"testing"
)

type keyType int

const (
	key1 keyType = iota
	key2
)

func TestContext(t *testing.T) {
	assertEqual := func(val interface{}, exp interface{}) {
		if val != exp {
			t.Errorf("Expected %v, got %v.", exp, val)
		}
	}

	r, _ := http.NewRequest("GET", "http://localhost:8080/", nil)
	emptyR, _ := http.NewRequest("GET", "http://localhost:8080/", nil)

	// Get()
	assertEqual(Get(r, key1), nil)

	// Set()
	Set(r, key1, "1")
	assertEqual(Get(r, key1), "1")
	assertEqual(len(data[r]), 1)

	Set(r, key2, "2")
	assertEqual(Get(r, key2), "2")
	assertEqual(len(data[r]), 2)

	//GetOk
	value, ok := GetOk(r, key1)
	assertEqual(value, "1")
	assertEqual(ok, true)

	value, ok = GetOk(r, "not exists")
	assertEqual(value, nil)
	assertEqual(ok, false)

	Set(r, "nil value", nil)
	value, ok = GetOk(r, "nil value")
	assertEqual(value, nil)
	assertEqual(ok, true)

	// GetAll()
	values := GetAll(r)
	assertEqual(len(values), 3)

	// GetAll() for empty request
	values = GetAll(emptyR)
	if values != nil {
		t.Error("GetAll didn't return nil value for invalid request")
	}

	// GetAllOk()
	values, ok = GetAllOk(r)
	assertEqual(len(values), 3)
	assertEqual(ok, true)

	// GetAllOk() for empty request
	values, ok = GetAllOk(emptyR)
	assertEqual(len(values), 0)
	assertEqual(ok, false)

	// Delete()
	Delete(r, key1)
	assertEqual(Get(r, key1), nil)
	assertEqual(len(data[r]), 2)

	Delete(r, key2)
	assertEqual(Get(r, key2), nil)
	assertEqual(len(data[r]), 1)

	// Clear()
	Clear(r)
	assertEqual(len(data), 0)
}

func parallelReader(r *http.Request, key string, iterations int, wait, done chan struct{}) {
	<-wait
	for i := 0; i < iterations; i++ {
		Get(r, key)
	}
	done <- struct{}{}

}

func parallelWriter(r *http.Request, key, value string, iterations int, wait, done chan struct{}) {
	<-wait
	for i := 0; i < iterations; i++ {
		Set(r, key, value)
	}
	done <- struct{}{}

}

func benchmarkMutex(b *testing.B, numReaders, numWriters, iterations int) {

	b.StopTimer()
	r, _ := http.NewRequest("GET", "http://localhost:8080/", nil)
	done := make(chan struct{})
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		wait := make(chan struct{})

		for i := 0; i < numReaders; i++ {
			go parallelReader(r, "test", iterations, wait, done)
		}

		for i := 0; i < numWriters; i++ {
			go parallelWriter(r, "test", "123", iterations, wait, done)
		}

		close(wait)

		for i := 0; i < numReaders+numWriters; i++ {
			<-done
		}

	}

}

func BenchmarkMutexSameReadWrite1(b *testing.B) {
	benchmarkMutex(b, 1, 1, 32)
}
func BenchmarkMutexSameReadWrite2(b *testing.B) {
	benchmarkMutex(b, 2, 2, 32)
}
func BenchmarkMutexSameReadWrite4(b *testing.B) {
	benchmarkMutex(b, 4, 4, 32)
}
func BenchmarkMutex1(b *testing.B) {
	benchmarkMutex(b, 2, 8, 32)
}
func BenchmarkMutex2(b *testing.B) {
	benchmarkMutex(b, 16, 4, 64)
}
func BenchmarkMutex3(b *testing.B) {
	benchmarkMutex(b, 1, 2, 128)
}
func BenchmarkMutex4(b *testing.B) {
	benchmarkMutex(b, 128, 32, 256)
}
func BenchmarkMutex5(b *testing.B) {
	benchmarkMutex(b, 1024, 2048, 64)
}
func BenchmarkMutex6(b *testing.B) {
	benchmarkMutex(b, 2048, 1024, 512)
}
<|endoftext|>


<commit_before>package telegram

import (
	"encoding/base64"
	"fmt"
	"strconv"
	"strings"

	"github.com/everpcpc/yubari/elasticsearch"

	tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api"
)

var (
	page = 5
)

func onSearch(b *Bot, message *tgbotapi.Message) {
	idx := getIndex(message)
	q := message.CommandArguments()

	exists, err := elasticsearch.CheckIndexExist(b.es, idx)
	if err != nil {
		b.logger.Errorf("check es index exists error: %+v", err)
		return
	}
	if !exists {
		msg := tgbotapi.NewMessage(message.Chat.ID, "还没有启用哦")
		msg.ReplyToMessageID = message.MessageID
		b.Client.Send(msg)
	}
	res, err := elasticsearch.SearchMessage(b.es, idx, q, 0, page)
	if err != nil {
		b.logger.Errorf("es search error: %+v", err)
		return
	}

	msg := tgbotapi.NewMessage(message.Chat.ID, buildSearchResponse(res, 0))
	msg.ParseMode = tgbotapi.ModeHTML
	msg.ReplyMarkup = buildSearchResponseButton(res, 0, q)

	_, err = b.Client.Send(msg)
	if err != nil {
		b.logger.Errorf("%+v", err)
	}
}

func getQueryCacheKey(msg *tgbotapi.Message) string {
	return fmt.Sprintf("search_query_%d", msg.MessageID)
}

func getIndex(message *tgbotapi.Message) string {
	return fmt.Sprintf("%s-%d", message.Chat.Type, message.Chat.ID)
}

func buildSearchResponse(res *elasticsearch.SearchResponse, from int) string {
	total := res.Hits.Total.Value
	respond := fmt.Sprintf("%d results：\n", total)
	for i, hit := range res.Hits.Hits {
		var content string
		if len(hit.Highlight.Content) == 0 {
			content = hit.Source.Content[:15]
		} else {
			content = hit.Highlight.Content[0]
		}
		respond += fmt.Sprintf("%d. %s\n", from+i+1, hit.Source.MessageID, content)
	}
	respond += fmt.Sprintf("duration %.3fs", float64(res.Took)/1000)
	return respond
}

func buildSearchResponseButton(res *elasticsearch.SearchResponse, from int, q string) tgbotapi.InlineKeyboardMarkup {
	total := res.Hits.Total.Value
	encodedQ := base64.StdEncoding.EncodeToString([]byte(q))
	row := tgbotapi.NewInlineKeyboardRow(
		tgbotapi.NewInlineKeyboardButtonData("⬅", fmt.Sprintf("search:%d:%s", max(int64(from-5), 0), encodedQ)),
		tgbotapi.NewInlineKeyboardButtonData("❎", fmt.Sprintf("search:%d:%s", -1, encodedQ)),
		tgbotapi.NewInlineKeyboardButtonData("➡", fmt.Sprintf("search:%d:%s", min(int64(from+5), int64(total)), encodedQ)),
	)
	return tgbotapi.NewInlineKeyboardMarkup(row)
}

func onReactionSearch(b *Bot, callbackQuery *tgbotapi.CallbackQuery) {
	token := strings.Split(callbackQuery.Data, ":")
	if len(token) != 3 {
		b.logger.Errorf("react data error: %s", callbackQuery.Data)
		return
	}

	from, err := strconv.ParseInt(token[1], 10, 0)
	if err != nil {
		b.logger.Errorf("react search from error: %s", callbackQuery.Data)
	}
	if from < 0 {
		delMsg := tgbotapi.DeleteMessageConfig{
			ChatID:    callbackQuery.Message.Chat.ID,
			MessageID: callbackQuery.Message.MessageID,
		}
		_, err = b.Client.DeleteMessage(delMsg)
		if err != nil {
			b.logger.Errorf("delete search error: %s", callbackQuery.Data)
		}
		return
	}

	q, err := base64.StdEncoding.DecodeString(token[2])
	if err != nil {
		b.logger.Errorf("react search q error: %s", callbackQuery.Data)
		return
	}

	idx := getIndex(callbackQuery.Message)

	res, err := elasticsearch.SearchMessage(b.es, idx, string(q), int(from), page)
	if err != nil {
		b.logger.Errorf("es search error: %+v", err)
		return
	}

	msg := tgbotapi.NewEditMessageText(
		callbackQuery.Message.Chat.ID,
		callbackQuery.Message.MessageID,
		buildSearchResponse(res, int(from)),
	)
	button := buildSearchResponseButton(res, int(from), string(q))
	msg.ParseMode = tgbotapi.ModeHTML
	msg.ReplyMarkup = &button

	_, err = b.Client.Send(msg)
	if err != nil {
		b.logger.Errorf("search reaction reply error: %+v", err)
	}
}
<commit_msg>always show 1 result<commit_after>package telegram

import (
	"encoding/base64"
	"fmt"
	"strconv"
	"strings"

	"github.com/everpcpc/yubari/elasticsearch"

	tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api"
)

var (
	page = 5
)

func onSearch(b *Bot, message *tgbotapi.Message) {
	idx := getIndex(message)
	q := message.CommandArguments()

	exists, err := elasticsearch.CheckIndexExist(b.es, idx)
	if err != nil {
		b.logger.Errorf("check es index exists error: %+v", err)
		return
	}
	if !exists {
		msg := tgbotapi.NewMessage(message.Chat.ID, "还没有启用哦")
		msg.ReplyToMessageID = message.MessageID
		b.Client.Send(msg)
	}
	res, err := elasticsearch.SearchMessage(b.es, idx, q, 0, page)
	if err != nil {
		b.logger.Errorf("es search error: %+v", err)
		return
	}

	msg := tgbotapi.NewMessage(message.Chat.ID, buildSearchResponse(res, 0))
	msg.ParseMode = tgbotapi.ModeHTML
	msg.ReplyMarkup = buildSearchResponseButton(res, 0, q)

	_, err = b.Client.Send(msg)
	if err != nil {
		b.logger.Errorf("%+v", err)
	}
}

func getIndex(message *tgbotapi.Message) string {
	return fmt.Sprintf("%s-%d", message.Chat.Type, message.Chat.ID)
}

func buildSearchResponse(res *elasticsearch.SearchResponse, from int) string {
	total := res.Hits.Total.Value
	respond := fmt.Sprintf("(%d) results: \n", total)
	for i, hit := range res.Hits.Hits {
		var content string
		if len(hit.Highlight.Content) == 0 {
			content = hit.Source.Content[:15]
		} else {
			content = hit.Highlight.Content[0]
		}
		respond += fmt.Sprintf("%d. %s\n", from+i+1, hit.Source.MessageID, content)
	}
	respond += fmt.Sprintf("duration %.3fs", float64(res.Took)/1000)
	return respond
}

func buildSearchResponseButton(res *elasticsearch.SearchResponse, from int, q string) tgbotapi.InlineKeyboardMarkup {
	total := res.Hits.Total.Value
	if total == 0 {
		total = 1
	}
	encodedQ := base64.StdEncoding.EncodeToString([]byte(q))
	row := tgbotapi.NewInlineKeyboardRow(
		tgbotapi.NewInlineKeyboardButtonData("⬅", fmt.Sprintf("search:%d:%s", max(int64(from-5), 0), encodedQ)),
		tgbotapi.NewInlineKeyboardButtonData("❎", fmt.Sprintf("search:%d:%s", -1, encodedQ)),
		tgbotapi.NewInlineKeyboardButtonData("➡", fmt.Sprintf("search:%d:%s", min(int64(from+5), int64(total-1)), encodedQ)),
	)
	return tgbotapi.NewInlineKeyboardMarkup(row)
}

func onReactionSearch(b *Bot, callbackQuery *tgbotapi.CallbackQuery) {
	token := strings.Split(callbackQuery.Data, ":")
	if len(token) != 3 {
		b.logger.Errorf("react data error: %s", callbackQuery.Data)
		return
	}

	from, err := strconv.ParseInt(token[1], 10, 0)
	if err != nil {
		b.logger.Errorf("react search from error: %s", callbackQuery.Data)
	}
	if from < 0 {
		delMsg := tgbotapi.DeleteMessageConfig{
			ChatID:    callbackQuery.Message.Chat.ID,
			MessageID: callbackQuery.Message.MessageID,
		}
		_, err = b.Client.DeleteMessage(delMsg)
		if err != nil {
			b.logger.Errorf("delete search error: %s", callbackQuery.Data)
		}
		return
	}

	q, err := base64.StdEncoding.DecodeString(token[2])
	if err != nil {
		b.logger.Errorf("react search q error: %s", callbackQuery.Data)
		return
	}

	idx := getIndex(callbackQuery.Message)

	res, err := elasticsearch.SearchMessage(b.es, idx, string(q), int(from), page)
	if err != nil {
		b.logger.Errorf("es search error: %+v", err)
		return
	}

	msg := tgbotapi.NewEditMessageText(
		callbackQuery.Message.Chat.ID,
		callbackQuery.Message.MessageID,
		buildSearchResponse(res, int(from)),
	)
	button := buildSearchResponseButton(res, int(from), string(q))
	msg.ParseMode = tgbotapi.ModeHTML
	msg.ReplyMarkup = &button

	_, err = b.Client.Send(msg)
	if err != nil {
		b.logger.Errorf("search reaction reply error: %+v", err)
	}
}
<|endoftext|>


<commit_before>package build

import (
	"encoding/json"
	"fmt"
	"regexp"
	"strconv"
)

// Parse attempts to create a version based on a given string
func Parse(raw string) (ver ProtocolVersion, err error) {
	parts := versionReg.FindStringSubmatch(raw)
	if len(parts) != 5 {
		err = InvalidVersionError(raw)
		return
	}

	// because of the regexp we can be sure that this will always succeed
	// an empty string automatically results in 0, even though it also returns an err
	major, _ := strconv.ParseUint(parts[1], 10, 8)
	minor, _ := strconv.ParseUint(parts[2], 10, 8)
	patch, _ := strconv.ParseUint(parts[3], 10, 8)

	ver = NewPrereleaseVersion(uint8(major), uint8(minor), uint8(patch), parts[4])
	return
}

// MustParse creates a version based on a given string,
// panics in case the given string is invalid
func MustParse(raw string) ProtocolVersion {
	version, err := Parse(raw)
	if err != nil {
		panic(err)
	}
	return version
}

// NewVersion creates a new protocol version
func NewVersion(major, minor, patch uint8) ProtocolVersion {
	return NewPrereleaseVersion(major, minor, patch, "")
}

// NewPrereleaseVersion creates a new protocol prerelease version
func NewPrereleaseVersion(major, minor, patch uint8, prerelease string) ProtocolVersion {
	var v ProtocolVersion
	v.Version = (uint32(major) << 24) | (uint32(minor) << 16) | (uint32(patch) << 8)
	copy(v.Prerelease[:], prerelease[:])
	return v
}

// ProtocolVersion defines the protocol version that a node uses
type ProtocolVersion struct {
	Version    uint32  // Semantic Versioning
	Prerelease [8]byte // Any pre-release tag (eg. 'alpha' or the git commit hash)
}

// InvalidVersionError indicates a protocol version is invalid.
type InvalidVersionError string

// Error implements the error interface for InvalidVersionError.
func (e InvalidVersionError) Error() string {
	if len(e) == 0 {
		return "invalid version: <nil>"
	}

	return "invalid version: " + string(e)
}

// Compare returns an integer comparing this version with another version.
func (pv *ProtocolVersion) Compare(other ProtocolVersion) int {
	if pv.Version < other.Version {
		return -1
	} else if pv.Version > other.Version {
		return 1
	}

	isAPrerelease := pv.Prerelease != nilPreRelease
	isBPrerelease := other.Prerelease != nilPreRelease

	if !isAPrerelease && isBPrerelease {
		return 1
	} else if isAPrerelease && !isBPrerelease {
		return -1
	}

	// NOTE: if the 2 versions both have a prerelease defined we count them as equal,
	//       regardless if those prerelease values are different or not

	return 0
}

// String returns the string version of this ProtocolVersion
func (pv *ProtocolVersion) String() string {
	str := fmt.Sprintf("%d.%d.%d",
		(pv.Version>>24)&0xFF, // major
		(pv.Version>>16)&0xFF, // minor
		(pv.Version>>8)&0xFF,  // patch
	)

	// optional prerelease
	if pv.Prerelease != nilPreRelease {
		str += "-" + string(pv.Prerelease[:])
	}

	return str
}

// MarshalJSON implements json.Marshaler.MarshalJSON
func (pv ProtocolVersion) MarshalJSON() ([]byte, error) {
	return json.Marshal(pv.String())
}

// UnmarshalJSON implements json.Unmarshaler.UnmarshalJSON
func (pv *ProtocolVersion) UnmarshalJSON(b []byte) error {
	var raw string
	if err := json.Unmarshal(b, &raw); err != nil {
		return InvalidVersionError(string(b))
	}

	result, err := Parse(raw)
	if err != nil {
		return err
	}

	pv.Version = result.Version
	pv.Prerelease = result.Prerelease
	return nil
}

var (
	nilPreRelease = [8]byte{0, 0, 0, 0, 0, 0, 0, 0}
)

var (
	// rawVersion used to generate rivine's protocol version
	rawVersion = "v1.0.4"
	// Version is the current version of rivined.
	Version ProtocolVersion
)

const (
	// EncodedVersionLength is the static length of a sia-encoded ProtocolVersion.
	EncodedVersionLength = 16 // sizeof(uint32==64) + sizeof([8]uint8)
)

const versionRe = `^v?(0{0,2}[0-9]|[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])(?:\.(0{0,2}[0-9]|[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5]))?(?:\.(0{0,2}[0-9]|[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5]))?(?:-(.+?))?$`

// contains the regexp for all valid versions
var versionReg = regexp.MustCompile(versionRe)

func init() {
	Version = MustParse(rawVersion)
}
<commit_msg>bump protocol version to v1.0.5-alpha<commit_after>package build

import (
	"encoding/json"
	"fmt"
	"regexp"
	"strconv"
)

// Parse attempts to create a version based on a given string
func Parse(raw string) (ver ProtocolVersion, err error) {
	parts := versionReg.FindStringSubmatch(raw)
	if len(parts) != 5 {
		err = InvalidVersionError(raw)
		return
	}

	// because of the regexp we can be sure that this will always succeed
	// an empty string automatically results in 0, even though it also returns an err
	major, _ := strconv.ParseUint(parts[1], 10, 8)
	minor, _ := strconv.ParseUint(parts[2], 10, 8)
	patch, _ := strconv.ParseUint(parts[3], 10, 8)

	ver = NewPrereleaseVersion(uint8(major), uint8(minor), uint8(patch), parts[4])
	return
}

// MustParse creates a version based on a given string,
// panics in case the given string is invalid
func MustParse(raw string) ProtocolVersion {
	version, err := Parse(raw)
	if err != nil {
		panic(err)
	}
	return version
}

// NewVersion creates a new protocol version
func NewVersion(major, minor, patch uint8) ProtocolVersion {
	return NewPrereleaseVersion(major, minor, patch, "")
}

// NewPrereleaseVersion creates a new protocol prerelease version
func NewPrereleaseVersion(major, minor, patch uint8, prerelease string) ProtocolVersion {
	var v ProtocolVersion
	v.Version = (uint32(major) << 24) | (uint32(minor) << 16) | (uint32(patch) << 8)
	copy(v.Prerelease[:], prerelease[:])
	return v
}

// ProtocolVersion defines the protocol version that a node uses
type ProtocolVersion struct {
	Version    uint32  // Semantic Versioning
	Prerelease [8]byte // Any pre-release tag (eg. 'alpha' or the git commit hash)
}

// InvalidVersionError indicates a protocol version is invalid.
type InvalidVersionError string

// Error implements the error interface for InvalidVersionError.
func (e InvalidVersionError) Error() string {
	if len(e) == 0 {
		return "invalid version: <nil>"
	}

	return "invalid version: " + string(e)
}

// Compare returns an integer comparing this version with another version.
func (pv *ProtocolVersion) Compare(other ProtocolVersion) int {
	if pv.Version < other.Version {
		return -1
	} else if pv.Version > other.Version {
		return 1
	}

	isAPrerelease := pv.Prerelease != nilPreRelease
	isBPrerelease := other.Prerelease != nilPreRelease

	if !isAPrerelease && isBPrerelease {
		return 1
	} else if isAPrerelease && !isBPrerelease {
		return -1
	}

	// NOTE: if the 2 versions both have a prerelease defined we count them as equal,
	//       regardless if those prerelease values are different or not

	return 0
}

// String returns the string version of this ProtocolVersion
func (pv *ProtocolVersion) String() string {
	str := fmt.Sprintf("%d.%d.%d",
		(pv.Version>>24)&0xFF, // major
		(pv.Version>>16)&0xFF, // minor
		(pv.Version>>8)&0xFF,  // patch
	)

	// optional prerelease
	if pv.Prerelease != nilPreRelease {
		str += "-" + string(pv.Prerelease[:])
	}

	return str
}

// MarshalJSON implements json.Marshaler.MarshalJSON
func (pv ProtocolVersion) MarshalJSON() ([]byte, error) {
	return json.Marshal(pv.String())
}

// UnmarshalJSON implements json.Unmarshaler.UnmarshalJSON
func (pv *ProtocolVersion) UnmarshalJSON(b []byte) error {
	var raw string
	if err := json.Unmarshal(b, &raw); err != nil {
		return InvalidVersionError(string(b))
	}

	result, err := Parse(raw)
	if err != nil {
		return err
	}

	pv.Version = result.Version
	pv.Prerelease = result.Prerelease
	return nil
}

var (
	nilPreRelease = [8]byte{0, 0, 0, 0, 0, 0, 0, 0}
)

var (
	// rawVersion used to generate rivine's protocol version
	rawVersion = "v1.0.5-alpha"
	// Version is the current version of rivined.
	Version ProtocolVersion
)

const (
	// EncodedVersionLength is the static length of a sia-encoded ProtocolVersion.
	EncodedVersionLength = 16 // sizeof(uint32==64) + sizeof([8]uint8)
)

const versionRe = `^v?(0{0,2}[0-9]|[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])(?:\.(0{0,2}[0-9]|[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5]))?(?:\.(0{0,2}[0-9]|[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5]))?(?:-(.+?))?$`

// contains the regexp for all valid versions
var versionReg = regexp.MustCompile(versionRe)

func init() {
	Version = MustParse(rawVersion)
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
)

// benchHash benchmarks a single commit.
func (b *Builder) benchHash(hash string, benchs []string) error {
	if *verbose {
		log.Println(b.name, "benchmarking", hash)
	}

	res := &PerfResult{Hash: hash, Benchmark: "meta-done"}

	// Create place in which to do work.
	workpath := filepath.Join(*buildroot, b.name+"-"+hash[:12])
	// Prepare a workpath if we don't have one we can reuse.
	update := false
	if b.lastWorkpath != workpath {
		if err := os.Mkdir(workpath, mkdirPerm); err != nil {
			return err
		}
		buildLog, _, err := b.buildRepoOnHash(workpath, hash, makeCmd)
		if err != nil {
			removePath(workpath)
			// record failure
			res.Artifacts = append(res.Artifacts, PerfArtifact{"log", buildLog})
			return b.recordPerfResult(res)
		}
		b.lastWorkpath = workpath
		update = true
	}

	// Build the benchmark binary.
	benchBin, buildLog, err := b.buildBenchmark(workpath, update)
	if err != nil {
		// record failure
		res.Artifacts = append(res.Artifacts, PerfArtifact{"log", buildLog})
		return b.recordPerfResult(res)
	}

	benchmark, procs, affinity, last := chooseBenchmark(benchBin, benchs)
	if benchmark != "" {
		res.Benchmark = fmt.Sprintf("%v-%v", benchmark, procs)
		res.Metrics, res.Artifacts, res.OK = b.executeBenchmark(workpath, hash, benchBin, benchmark, procs, affinity)
		if err = b.recordPerfResult(res); err != nil {
			return fmt.Errorf("recordResult: %s", err)
		}
	}

	if last {
		// All benchmarks have beed executed, don't need workpath anymore.
		removePath(b.lastWorkpath)
		b.lastWorkpath = ""
		// Notify the app.
		res = &PerfResult{Hash: hash, Benchmark: "meta-done", OK: true}
		if err = b.recordPerfResult(res); err != nil {
			return fmt.Errorf("recordResult: %s", err)
		}
	}

	return nil
}

// buildBenchmark builds the benchmark binary.
func (b *Builder) buildBenchmark(workpath string, update bool) (benchBin, log string, err error) {
	goroot := filepath.Join(workpath, "go")
	gobin := filepath.Join(goroot, "bin", "go") + exeExt
	gopath := filepath.Join(*buildroot, "gopath")
	env := append([]string{
		"GOROOT=" + goroot,
		"GOPATH=" + gopath},
		b.envv()...)
	// First, download without installing.
	args := []string{"get", "-d"}
	if update {
		args = append(args, "-u")
	}
	args = append(args, *benchPath)
	var buildlog bytes.Buffer
	runOpts := []runOpt{runTimeout(*buildTimeout), runEnv(env), allOutput(&buildlog), runDir(workpath)}
	err = run(exec.Command(gobin, args...), runOpts...)
	if err != nil {
		fmt.Fprintf(&buildlog, "go get -d %s failed: %s", *benchPath, err)
		return "", buildlog.String(), err
	}
	// Then, build into workpath.
	benchBin = filepath.Join(workpath, "benchbin") + exeExt
	args = []string{"build", "-o", benchBin, *benchPath}
	buildlog.Reset()
	err = run(exec.Command(gobin, args...), runOpts...)
	if err != nil {
		fmt.Fprintf(&buildlog, "go build %s failed: %s", *benchPath, err)
		return "", buildlog.String(), err
	}
	return benchBin, "", nil
}

// chooseBenchmark chooses the next benchmark to run
// based on the list of available benchmarks, already executed benchmarks
// and -benchcpu list.
func chooseBenchmark(benchBin string, doneBenchs []string) (bench string, procs, affinity int, last bool) {
	out, err := exec.Command(benchBin).CombinedOutput()
	if err != nil {
		log.Printf("Failed to query benchmark list: %v\n%s", err, out)
		last = true
		return
	}
	outStr := string(out)
	nlIdx := strings.Index(outStr, "\n")
	if nlIdx < 0 {
		log.Printf("Failed to parse benchmark list (no new line): %s", outStr)
		last = true
		return
	}
	localBenchs := strings.Split(outStr[:nlIdx], ",")
	benchsMap := make(map[string]bool)
	for _, b := range doneBenchs {
		benchsMap[b] = true
	}
	cnt := 0
	// We want to run all benchmarks with GOMAXPROCS=1 first.
	for i, procs1 := range benchCPU {
		for _, bench1 := range localBenchs {
			if benchsMap[fmt.Sprintf("%v-%v", bench1, procs1)] {
				continue
			}
			cnt++
			if cnt == 1 {
				bench = bench1
				procs = procs1
				if i < len(benchAffinity) {
					affinity = benchAffinity[i]
				}
			}
		}
	}
	last = cnt <= 1
	return
}

// executeBenchmark runs a single benchmark and parses its output.
func (b *Builder) executeBenchmark(workpath, hash, benchBin, bench string, procs, affinity int) (metrics []PerfMetric, artifacts []PerfArtifact, ok bool) {
	// Benchmarks runs mutually exclusive with other activities.
	benchMutex.RUnlock()
	defer benchMutex.RLock()
	benchMutex.Lock()
	defer benchMutex.Unlock()

	log.Printf("%v executing benchmark %v-%v on %v", b.name, bench, procs, hash)

	// The benchmark executes 'go build'/'go tool',
	// so we need properly setup env.
	env := append([]string{
		"GOROOT=" + filepath.Join(workpath, "go"),
		"PATH=" + filepath.Join(workpath, "go", "bin") + string(os.PathListSeparator) + os.Getenv("PATH"),
		"GODEBUG=gctrace=1", // since Go1.2
		"GOGCTRACE=1",       // before Go1.2
		fmt.Sprintf("GOMAXPROCS=%v", procs)},
		b.envv()...)
	args := []string{
		"-bench", bench,
		"-benchmem", strconv.Itoa(*benchMem),
		"-benchtime", benchTime.String(),
		"-benchnum", strconv.Itoa(*benchNum),
		"-tmpdir", workpath}
	if affinity != 0 {
		args = append(args, "-affinity", strconv.Itoa(affinity))
	}
	benchlog := new(bytes.Buffer)
	err := run(exec.Command(benchBin, args...), runEnv(env), allOutput(benchlog), runDir(workpath))
	if strip := benchlog.Len() - 512<<10; strip > 0 {
		// Leave the last 512K, that part contains metrics.
		benchlog = bytes.NewBuffer(benchlog.Bytes()[strip:])
	}
	artifacts = []PerfArtifact{{Type: "log", Body: benchlog.String()}}
	if err != nil {
		if err != nil {
			log.Printf("Failed to execute benchmark '%v': %v", bench, err)
			ok = false
		}
		return
	}

	metrics1, artifacts1, err := parseBenchmarkOutput(benchlog)
	if err != nil {
		log.Printf("Failed to parse benchmark output: %v", err)
		ok = false
		return
	}
	metrics = metrics1
	artifacts = append(artifacts, artifacts1...)
	return
}

// parseBenchmarkOutput fetches metrics and artifacts from benchmark output.
func parseBenchmarkOutput(out io.Reader) (metrics []PerfMetric, artifacts []PerfArtifact, err error) {
	s := bufio.NewScanner(out)
	metricRe := regexp.MustCompile("^GOPERF-METRIC:([a-z,0-9,-]+)=([0-9]+)$")
	fileRe := regexp.MustCompile("^GOPERF-FILE:([a-z,0-9,-]+)=(.+)$")
	for s.Scan() {
		ln := s.Text()
		if ss := metricRe.FindStringSubmatch(ln); ss != nil {
			var v uint64
			v, err = strconv.ParseUint(ss[2], 10, 64)
			if err != nil {
				err = fmt.Errorf("Failed to parse metric '%v=%v': %v", ss[1], ss[2], err)
				return
			}
			metrics = append(metrics, PerfMetric{Type: ss[1], Val: v})
		} else if ss := fileRe.FindStringSubmatch(ln); ss != nil {
			var buf []byte
			buf, err = ioutil.ReadFile(ss[2])
			if err != nil {
				err = fmt.Errorf("Failed to read file '%v': %v", ss[2], err)
				return
			}
			artifacts = append(artifacts, PerfArtifact{ss[1], string(buf)})
		}
	}
	return
}

// needsBenchmarking determines whether the commit needs benchmarking.
func needsBenchmarking(log *HgLog) bool {
	// Do not benchmark branch commits, they are usually not interesting
	// and fall out of the trunk succession.
	if log.Branch != "" {
		return false
	}
	// Do not benchmark commits that do not touch source files (e.g. CONTRIBUTORS).
	for _, f := range strings.Split(log.Files, " ") {
		if (strings.HasPrefix(f, "include") || strings.HasPrefix(f, "src")) &&
			!strings.HasSuffix(f, "_test.go") && !strings.Contains(f, "testdata") {
			return true
		}
	}
	return false
}
<commit_msg>dashboard: fix perf builder One of these "how it ever worked?.." moments.<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
)

// benchHash benchmarks a single commit.
func (b *Builder) benchHash(hash string, benchs []string) error {
	if *verbose {
		log.Println(b.name, "benchmarking", hash)
	}

	res := &PerfResult{Hash: hash, Benchmark: "meta-done"}

	// Create place in which to do work.
	workpath := filepath.Join(*buildroot, b.name+"-"+hash[:12])
	// Prepare a workpath if we don't have one we can reuse.
	update := false
	if b.lastWorkpath != workpath {
		if err := os.Mkdir(workpath, mkdirPerm); err != nil {
			return err
		}
		buildLog, _, err := b.buildRepoOnHash(workpath, hash, makeCmd)
		if err != nil {
			removePath(workpath)
			// record failure
			res.Artifacts = append(res.Artifacts, PerfArtifact{"log", buildLog})
			return b.recordPerfResult(res)
		}
		b.lastWorkpath = workpath
		update = true
	}

	// Build the benchmark binary.
	benchBin, buildLog, err := b.buildBenchmark(workpath, update)
	if err != nil {
		// record failure
		res.Artifacts = append(res.Artifacts, PerfArtifact{"log", buildLog})
		return b.recordPerfResult(res)
	}

	benchmark, procs, affinity, last := chooseBenchmark(benchBin, benchs)
	if benchmark != "" {
		res.Benchmark = fmt.Sprintf("%v-%v", benchmark, procs)
		res.Metrics, res.Artifacts, res.OK = b.executeBenchmark(workpath, hash, benchBin, benchmark, procs, affinity)
		if err = b.recordPerfResult(res); err != nil {
			return fmt.Errorf("recordResult: %s", err)
		}
	}

	if last {
		// All benchmarks have beed executed, don't need workpath anymore.
		removePath(b.lastWorkpath)
		b.lastWorkpath = ""
		// Notify the app.
		res = &PerfResult{Hash: hash, Benchmark: "meta-done", OK: true}
		if err = b.recordPerfResult(res); err != nil {
			return fmt.Errorf("recordResult: %s", err)
		}
	}

	return nil
}

// buildBenchmark builds the benchmark binary.
func (b *Builder) buildBenchmark(workpath string, update bool) (benchBin, log string, err error) {
	goroot := filepath.Join(workpath, "go")
	gobin := filepath.Join(goroot, "bin", "go") + exeExt
	gopath := filepath.Join(*buildroot, "gopath")
	env := append([]string{
		"GOROOT=" + goroot,
		"GOPATH=" + gopath},
		b.envv()...)
	// First, download without installing.
	args := []string{"get", "-d"}
	if update {
		args = append(args, "-u")
	}
	args = append(args, *benchPath)
	var buildlog bytes.Buffer
	runOpts := []runOpt{runTimeout(*buildTimeout), runEnv(env), allOutput(&buildlog), runDir(workpath)}
	err = run(exec.Command(gobin, args...), runOpts...)
	if err != nil {
		fmt.Fprintf(&buildlog, "go get -d %s failed: %s", *benchPath, err)
		return "", buildlog.String(), err
	}
	// Then, build into workpath.
	benchBin = filepath.Join(workpath, "benchbin") + exeExt
	args = []string{"build", "-o", benchBin, *benchPath}
	buildlog.Reset()
	err = run(exec.Command(gobin, args...), runOpts...)
	if err != nil {
		fmt.Fprintf(&buildlog, "go build %s failed: %s", *benchPath, err)
		return "", buildlog.String(), err
	}
	return benchBin, "", nil
}

// chooseBenchmark chooses the next benchmark to run
// based on the list of available benchmarks, already executed benchmarks
// and -benchcpu list.
func chooseBenchmark(benchBin string, doneBenchs []string) (bench string, procs, affinity int, last bool) {
	out, err := exec.Command(benchBin).CombinedOutput()
	if err != nil {
		log.Printf("Failed to query benchmark list: %v\n%s", err, out)
		last = true
		return
	}
	outStr := string(out)
	nlIdx := strings.Index(outStr, "\n")
	if nlIdx < 0 {
		log.Printf("Failed to parse benchmark list (no new line): %s", outStr)
		last = true
		return
	}
	localBenchs := strings.Split(outStr[:nlIdx], ",")
	benchsMap := make(map[string]bool)
	for _, b := range doneBenchs {
		benchsMap[b] = true
	}
	cnt := 0
	// We want to run all benchmarks with GOMAXPROCS=1 first.
	for i, procs1 := range benchCPU {
		for _, bench1 := range localBenchs {
			if benchsMap[fmt.Sprintf("%v-%v", bench1, procs1)] {
				continue
			}
			cnt++
			if cnt == 1 {
				bench = bench1
				procs = procs1
				if i < len(benchAffinity) {
					affinity = benchAffinity[i]
				}
			}
		}
	}
	last = cnt <= 1
	return
}

// executeBenchmark runs a single benchmark and parses its output.
func (b *Builder) executeBenchmark(workpath, hash, benchBin, bench string, procs, affinity int) (metrics []PerfMetric, artifacts []PerfArtifact, ok bool) {
	// Benchmarks runs mutually exclusive with other activities.
	benchMutex.RUnlock()
	defer benchMutex.RLock()
	benchMutex.Lock()
	defer benchMutex.Unlock()

	log.Printf("%v executing benchmark %v-%v on %v", b.name, bench, procs, hash)

	// The benchmark executes 'go build'/'go tool',
	// so we need properly setup env.
	env := append([]string{
		"GOROOT=" + filepath.Join(workpath, "go"),
		"PATH=" + filepath.Join(workpath, "go", "bin") + string(os.PathListSeparator) + os.Getenv("PATH"),
		"GODEBUG=gctrace=1", // since Go1.2
		"GOGCTRACE=1",       // before Go1.2
		fmt.Sprintf("GOMAXPROCS=%v", procs)},
		b.envv()...)
	args := []string{
		"-bench", bench,
		"-benchmem", strconv.Itoa(*benchMem),
		"-benchtime", benchTime.String(),
		"-benchnum", strconv.Itoa(*benchNum),
		"-tmpdir", workpath}
	if affinity != 0 {
		args = append(args, "-affinity", strconv.Itoa(affinity))
	}
	benchlog := new(bytes.Buffer)
	err := run(exec.Command(benchBin, args...), runEnv(env), allOutput(benchlog), runDir(workpath))
	if strip := benchlog.Len() - 512<<10; strip > 0 {
		// Leave the last 512K, that part contains metrics.
		benchlog = bytes.NewBuffer(benchlog.Bytes()[strip:])
	}
	artifacts = []PerfArtifact{{Type: "log", Body: benchlog.String()}}
	if err != nil {
		if err != nil {
			log.Printf("Failed to execute benchmark '%v': %v", bench, err)
			ok = false
		}
		return
	}

	metrics1, artifacts1, err := parseBenchmarkOutput(benchlog)
	if err != nil {
		log.Printf("Failed to parse benchmark output: %v", err)
		ok = false
		return
	}
	metrics = metrics1
	artifacts = append(artifacts, artifacts1...)
	ok = true
	return
}

// parseBenchmarkOutput fetches metrics and artifacts from benchmark output.
func parseBenchmarkOutput(out io.Reader) (metrics []PerfMetric, artifacts []PerfArtifact, err error) {
	s := bufio.NewScanner(out)
	metricRe := regexp.MustCompile("^GOPERF-METRIC:([a-z,0-9,-]+)=([0-9]+)$")
	fileRe := regexp.MustCompile("^GOPERF-FILE:([a-z,0-9,-]+)=(.+)$")
	for s.Scan() {
		ln := s.Text()
		if ss := metricRe.FindStringSubmatch(ln); ss != nil {
			var v uint64
			v, err = strconv.ParseUint(ss[2], 10, 64)
			if err != nil {
				err = fmt.Errorf("Failed to parse metric '%v=%v': %v", ss[1], ss[2], err)
				return
			}
			metrics = append(metrics, PerfMetric{Type: ss[1], Val: v})
		} else if ss := fileRe.FindStringSubmatch(ln); ss != nil {
			var buf []byte
			buf, err = ioutil.ReadFile(ss[2])
			if err != nil {
				err = fmt.Errorf("Failed to read file '%v': %v", ss[2], err)
				return
			}
			artifacts = append(artifacts, PerfArtifact{ss[1], string(buf)})
		}
	}
	return
}

// needsBenchmarking determines whether the commit needs benchmarking.
func needsBenchmarking(log *HgLog) bool {
	// Do not benchmark branch commits, they are usually not interesting
	// and fall out of the trunk succession.
	if log.Branch != "" {
		return false
	}
	// Do not benchmark commits that do not touch source files (e.g. CONTRIBUTORS).
	for _, f := range strings.Split(log.Files, " ") {
		if (strings.HasPrefix(f, "include") || strings.HasPrefix(f, "src")) &&
			!strings.HasSuffix(f, "_test.go") && !strings.Contains(f, "testdata") {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package builder

// nginx
const (
	NginxVersion           = "1.11.1"
	NginxDownloadURLPrefix = "http://nginx.org/download"
)

// pcre
const (
	PcreVersion           = "8.38"
	PcreDownloadURLPrefix = "http://ftp.csx.cam.ac.uk/pub/software/programming/pcre"
)

// openssl
const (
	OpenSSLVersion           = "1.0.2h"
	OpenSSLDownloadURLPrefix = "http://www.openssl.org/source"
)

// zlib
const (
	ZlibVersion           = "1.2.8"
	ZlibDownloadURLPrefix = "http://zlib.net"
)

// openResty
const (
	OpenRestyVersion           = "1.9.15.1"
	OpenRestyDownloadURLPrefix = "https://openresty.org/download"
)

// tengine
const (
	TengineVersion           = "2.1.2"
	TengineDownloadURLPrefix = "http://tengine.taobao.org/download"
)

// component enumerations
const (
	ComponentNginx = iota
	ComponentOpenResty
	ComponentTengine
	ComponentPcre
	ComponentOpenSSL
	ComponentZlib
	ComponentMax
)
<commit_msg>bumped pcre version to 8.39.<commit_after>package builder

// nginx
const (
	NginxVersion           = "1.11.1"
	NginxDownloadURLPrefix = "http://nginx.org/download"
)

// pcre
const (
	PcreVersion           = "8.39"
	PcreDownloadURLPrefix = "http://ftp.csx.cam.ac.uk/pub/software/programming/pcre"
)

// openssl
const (
	OpenSSLVersion           = "1.0.2h"
	OpenSSLDownloadURLPrefix = "http://www.openssl.org/source"
)

// zlib
const (
	ZlibVersion           = "1.2.8"
	ZlibDownloadURLPrefix = "http://zlib.net"
)

// openResty
const (
	OpenRestyVersion           = "1.9.15.1"
	OpenRestyDownloadURLPrefix = "https://openresty.org/download"
)

// tengine
const (
	TengineVersion           = "2.1.2"
	TengineDownloadURLPrefix = "http://tengine.taobao.org/download"
)

// component enumerations
const (
	ComponentNginx = iota
	ComponentOpenResty
	ComponentTengine
	ComponentPcre
	ComponentOpenSSL
	ComponentZlib
	ComponentMax
)
<|endoftext|>


<commit_before>// Package telegrafstatsd implement telegraf extended StatsD protocol for github.com/rs/xstats
package telegraf

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"strings"
	"time"

	"github.com/rs/xstats"
)

// Inspired by https://github.com/streadway/handy statsd package

type sender chan string

// MaxPacketLen is the number of bytes filled before a packet is flushed before
// the reporting interval.
const maxPacketLen = 2 ^ 15

var tick = time.Tick

// New creates a telegraf statsd sender that emit observations in the statsd
// protocol to the passed writer. Observations are buffered for the report
// interval or until the buffer exceeds a max packet size, whichever comes
// first.
func New(w io.Writer, reportInterval time.Duration) xstats.Sender {
	s := make(chan string)
	go fwd(w, reportInterval, s)
	return sender(s)
}

// Gauge implements xstats.Sender interface
func (s sender) Gauge(stat string, value float64, tags ...string) {
	s <- fmt.Sprintf("%s,%s:%f|g\n", stat, t(tags), value)
}

// Count implements xstats.Sender interface
func (s sender) Count(stat string, count float64, tags ...string) {
	s <- fmt.Sprintf("%s,%s:%f|c\n", stat, t(tags), count)
}

// Histogram implements xstats.Sender interface
func (s sender) Histogram(stat string, value float64, tags ...string) {
	s <- fmt.Sprintf("%s,%s:%f|h\n", stat, t(tags), value)
}

// Timing implements xstats.Sender interface
func (s sender) Timing(stat string, duration time.Duration, tags ...string) {
	s <- fmt.Sprintf("%s,%s:%f|ms\n", stat, t(tags), duration.Seconds())
}

// Generate a telegraf tag suffix
func t(tags []string) string {
	for i, v := range tags {
		tags[i] = strings.Replace(v, ":", "=", 1)
	}
	t := ""
	if len(tags) > 0 {
		t = "" + strings.Join(tags, ",")
	}
	return t
}

func fwd(w io.Writer, reportInterval time.Duration, c <-chan string) {
	buf := &bytes.Buffer{}
	tick := tick(reportInterval)
	for {
		select {
		case s := <-c:
			buf.Write([]byte(s))
			if buf.Len() > maxPacketLen {
				flush(w, buf)
			}

		case <-tick:
			flush(w, buf)
		}
	}
}

func flush(w io.Writer, buf *bytes.Buffer) {
	if buf.Len() <= 0 {
		return
	}
	if _, err := w.Write(buf.Bytes()); err != nil {
		log.Printf("error: could not write to statsd: %v", err)
	}
	buf.Reset()
}
<commit_msg>telegraf: add Close that flushes the insternal buffer<commit_after>// Package telegrafstatsd implement telegraf extended StatsD protocol for github.com/rs/xstats
package telegraf

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"strings"
	"time"

	"github.com/rs/xstats"
)

// Inspired by https://github.com/streadway/handy statsd package

type sender struct {
	c    chan string
	quit chan struct{}
	done chan struct{}
}

// MaxPacketLen is the number of bytes filled before a packet is flushed before
// the reporting interval.
const maxPacketLen = 2 ^ 15

var tick = time.Tick

// New creates a telegraf statsd sender that emit observations in the statsd
// protocol to the passed writer. Observations are buffered for the report
// interval or until the buffer exceeds a max packet size, whichever comes
// first.
func New(w io.Writer, reportInterval time.Duration) xstats.Sender {
	s := &sender{
		c:    make(chan string),
		quit: make(chan struct{}),
		done: make(chan struct{}),
	}
	go s.fwd(w, reportInterval)
	return s
}

// Gauge implements xstats.Sender interface
func (s *sender) Gauge(stat string, value float64, tags ...string) {
	s.c <- fmt.Sprintf("%s,%s:%f|g\n", stat, t(tags), value)
}

// Count implements xstats.Sender interface
func (s *sender) Count(stat string, count float64, tags ...string) {
	s.c <- fmt.Sprintf("%s,%s:%f|c\n", stat, t(tags), count)
}

// Histogram implements xstats.Sender interface
func (s *sender) Histogram(stat string, value float64, tags ...string) {
	s.c <- fmt.Sprintf("%s,%s:%f|h\n", stat, t(tags), value)
}

// Timing implements xstats.Sender interface
func (s *sender) Timing(stat string, duration time.Duration, tags ...string) {
	s.c <- fmt.Sprintf("%s,%s:%f|ms\n", stat, t(tags), duration.Seconds())
}

// Close implements xstats.Sender interface
func (s *sender) Close() {
	close(s.quit)
	<-s.done
	close(s.c)
}

// Generate a telegraf tag suffix
func t(tags []string) string {
	for i, v := range tags {
		tags[i] = strings.Replace(v, ":", "=", 1)
	}
	t := ""
	if len(tags) > 0 {
		t = "" + strings.Join(tags, ",")
	}
	return t
}

func (s *sender) fwd(w io.Writer, reportInterval time.Duration) {
	defer close(s.done)

	buf := &bytes.Buffer{}
	tick := tick(reportInterval)
	for {
		select {
		case m := <-s.c:
			buf.Write([]byte(m))
			if buf.Len() > maxPacketLen {
				flush(w, buf)
			}

		case <-tick:
			flush(w, buf)
		case <-s.quit:
			flush(w, buf)
			return
		}
	}
}

func flush(w io.Writer, buf *bytes.Buffer) {
	if buf.Len() <= 0 {
		return
	}
	if _, err := w.Write(buf.Bytes()); err != nil {
		log.Printf("error: could not write to statsd: %v", err)
	}
	buf.Reset()
}
<|endoftext|>