<commit_before>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package cli

import (
	log "github.com/Sirupsen/logrus"
	"github.com/spf13/cobra"

	"mynewt.apache.org/newt/newtmgr/nmutil"
	"mynewt.apache.org/newt/util"
)

var ConnProfileName string
var NewtmgrLogLevel log.Level
var NewtmgrHelp bool

func Commands() *cobra.Command {
	logLevelStr := ""
	nmCmd := &cobra.Command{
		Use:   "newtmgr",
		Short: "Newtmgr helps you manage devices running the Mynewt OS",
		PersistentPreRun: func(cmd *cobra.Command, args []string) {
			NewtmgrLogLevel, err := log.ParseLevel(logLevelStr)
			err = util.Init(NewtmgrLogLevel, "", util.VERBOSITY_DEFAULT)
			if err != nil {
				nmUsage(nil, err)
			}
		},
		Run: func(cmd *cobra.Command, args []string) {
			cmd.HelpFunc()(cmd, args)
		},
	}

	nmCmd.PersistentFlags().StringVarP(&ConnProfileName, "conn", "c", "",
		"connection profile to use")

	nmCmd.PersistentFlags().StringVarP(&logLevelStr, "loglevel", "l", "info",
		"log level to use")

	nmCmd.PersistentFlags().BoolVarP(&nmutil.TraceLogEnabled, "trace", "t",
		false, "print all bytes transmitted and received")

	// Add the help flag so it shows up under Global Flags
	nmCmd.PersistentFlags().BoolVarP(&NewtmgrHelp, "help", "h",
		false, "Help for newtmgr commands")

	nmCmd.AddCommand(configCmd())
	nmCmd.AddCommand(connProfileCmd())
	nmCmd.AddCommand(crashCmd())
	nmCmd.AddCommand(dTimeCmd())
	nmCmd.AddCommand(fsCmd())
	nmCmd.AddCommand(echoCmd())
	nmCmd.AddCommand(imageCmd())
	nmCmd.AddCommand(logsCmd())
	nmCmd.AddCommand(mempoolStatsCmd())
	nmCmd.AddCommand(resetCmd())
	nmCmd.AddCommand(runCmd())
	nmCmd.AddCommand(statsCmd())
	nmCmd.AddCommand(taskStatsCmd())

	return nmCmd
}
<commit_msg>Modified Newtmgr help text. It now says "Newtmgr helps you manage remote devices running the Mynewt OS"<commit_after>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package cli

import (
	log "github.com/Sirupsen/logrus"
	"github.com/spf13/cobra"

	"mynewt.apache.org/newt/newtmgr/nmutil"
	"mynewt.apache.org/newt/util"
)

var ConnProfileName string
var NewtmgrLogLevel log.Level
var NewtmgrHelp bool

func Commands() *cobra.Command {
	logLevelStr := ""
	nmCmd := &cobra.Command{
		Use:   "newtmgr",
		Short: "Newtmgr helps you manage remote devices running the Mynewt OS",
		PersistentPreRun: func(cmd *cobra.Command, args []string) {
			NewtmgrLogLevel, err := log.ParseLevel(logLevelStr)
			err = util.Init(NewtmgrLogLevel, "", util.VERBOSITY_DEFAULT)
			if err != nil {
				nmUsage(nil, err)
			}
		},
		Run: func(cmd *cobra.Command, args []string) {
			cmd.HelpFunc()(cmd, args)
		},
	}

	nmCmd.PersistentFlags().StringVarP(&ConnProfileName, "conn", "c", "",
		"connection profile to use")

	nmCmd.PersistentFlags().StringVarP(&logLevelStr, "loglevel", "l", "info",
		"log level to use")

	nmCmd.PersistentFlags().BoolVarP(&nmutil.TraceLogEnabled, "trace", "t",
		false, "print all bytes transmitted and received")

	// Add the help flag so it shows up under Global Flags
	nmCmd.PersistentFlags().BoolVarP(&NewtmgrHelp, "help", "h",
		false, "Help for newtmgr commands")

	nmCmd.AddCommand(configCmd())
	nmCmd.AddCommand(connProfileCmd())
	nmCmd.AddCommand(crashCmd())
	nmCmd.AddCommand(dTimeCmd())
	nmCmd.AddCommand(fsCmd())
	nmCmd.AddCommand(echoCmd())
	nmCmd.AddCommand(imageCmd())
	nmCmd.AddCommand(logsCmd())
	nmCmd.AddCommand(mempoolStatsCmd())
	nmCmd.AddCommand(resetCmd())
	nmCmd.AddCommand(runCmd())
	nmCmd.AddCommand(statsCmd())
	nmCmd.AddCommand(taskStatsCmd())

	return nmCmd
}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"os"
	"path"
	"strings"

	"github.com/Sirupsen/logrus"
	"github.com/huawei-openlab/oct/factory"
	"github.com/huawei-openlab/oct/utils"
	"github.com/huawei-openlab/oct/utils/config"
	"github.com/huawei-openlab/oct/utils/hooks"
)

const TestCacheDir = "./bundles/"
const (
	PASS = "SUCESSFUL"
	FAIL = "FAILED"
)

type TestUnit struct {
	//Case name
	Name string
	//Args is used to generate bundle
	Args string
	//Describle what does this unit test for. It is optional.
	Description string
	//Testopt is the term of OCI specs to be validate, it can be split from Args
	Testopt string

	BundleDir string
	Runtime   factory.Factory
	//success or failed
	Result string
	//when result == failed, ErrInfo is err code, or, ErrInfo is nil
	ErrInfo error
}

type UnitsManager struct {
	TestUnits []*TestUnit
}

var units *UnitsManager = new(UnitsManager)

func (this *UnitsManager) LoadTestUnits(filename string) {

	for key, value := range config.BundleMap {
		//TODO: config.BundleMap should support 'Description'
		unit := NewTestUnit(key, value, "")
		this.TestUnits = append(this.TestUnits, unit)
	}
}

func NewTestUnit(name string, args string, desc string) *TestUnit {

	tu := new(TestUnit)
	tu.Name = name
	tu.Args = args
	tu.Description = desc
	argsslice := strings.Fields(args)
	for i, arg := range argsslice {
		if strings.EqualFold(arg, "--args=./runtimetest") {
			tu.Testopt = strings.TrimPrefix(argsslice[i+1], "--args=")
		}
	}
	return tu
}

//Ouput method, ouput value: err-only or all
func (this *UnitsManager) OutputResult(output string) {

	if output != "err-only" && output != "all" {
		logrus.Fatalf("Error output cmd, output=%v\n", output)
	}

	SuccessCount := 0
	failCount := 0

	//Can not merge into on range, because output should be devided into two parts, successful and
	//failure
	if output == "all" {
		logrus.Println("Sucessful Details:")
		echoDividing()
	}

	for _, tu := range this.TestUnits {
		if tu.Result == PASS {
			SuccessCount++
			if output == "all" {
				tu.EchoSUnit()
			}
		}
	}

	logrus.Println("Failure Details:")
	echoDividing()

	for _, tu := range this.TestUnits {
		if tu.Result == FAIL {
			failCount++
			tu.EchoFUit()
		}
	}

	echoDividing()
	logrus.Printf("Statistics:  %v bundles success, %v bundles failed\n", SuccessCount, failCount)
}

func (unit *TestUnit) EchoSUnit() {

	logrus.Printf("\nBundleName:\n  %v\nBundleDir:\n  %v\nCaseArgs:\n  %v\nTestResult:\n  %v\n",
		unit.Name, unit.BundleDir, unit.Args, unit.Result)
}

func (unit *TestUnit) EchoFUit() {
	logrus.Printf("\nBundleName:\n  %v\nBundleDir:\n  %v\nCaseArgs:\n  %v\nResult:\n  %v\n"+
		"ErrInfo:\n  %v\n", unit.Name, unit.BundleDir, unit.Args, unit.Result, unit.ErrInfo)
}

func echoDividing() {
	logrus.Println("============================================================================" +
		"===================")
}

func (unit *TestUnit) SetResult(result string, err error) {
	unit.Result = result
	if result == PASS {
		unit.ErrInfo = nil
	} else {
		unit.ErrInfo = err
	}
}

//Set runtime
func (unit *TestUnit) SetRuntime(runtime string) error {
	if r, err := factory.CreateRuntime(runtime); err != nil {
		logrus.Printf("Create runtime %v err: %v\n", runtime, err)
		return err
	} else {
		unit.Runtime = r
	}
	return nil
}

func (unit *TestUnit) Run() {
	if unit.Runtime == nil {
		logrus.Fatalf("Set the runtime before run the test")
	}

	unit.GenerateConfigs()
	unit.PrepareBundle()

	out, err := unit.Runtime.StartRT(unit.BundleDir)
	if err != nil {
		unit.SetResult(FAIL, err)
		return
	}

	if err = unit.PostStartHooks(unit.Testopt, out); err != nil {
		unit.SetResult(FAIL, err)
		return
	}

	_ = unit.Runtime.StopRT(unit.Runtime.GetRTID())
	unit.SetResult(PASS, nil)
	return
}

func (unit *TestUnit) PostStartHooks(testopt string, out string) error {
	var err error
	switch testopt {
	case "vna":
		err = hooks.SetPostStartHooks(out, hooks.NamespacePostStart)
	default:
	}
	return err
}

func (unit *TestUnit) PrepareBundle() {
	// Create bundle follder
	unit.BundleDir = path.Join(TestCacheDir, unit.Name)
	err := os.RemoveAll(unit.BundleDir)
	if err != nil {
		logrus.Fatalf("Remove bundle %v err: %v\n", unit.Name, err)
	}

	err = os.Mkdir(unit.BundleDir, os.ModePerm)
	if err != nil {
		logrus.Fatalf("Mkdir bundle %v dir err: %v\n", unit.BundleDir, err)
	}

	// Create rootfs folder to bundle
	rootfs := unit.BundleDir + "/rootfs"
	err = os.Mkdir(rootfs, os.ModePerm)
	if err != nil {
		logrus.Fatalf("Mkdir rootfs for bundle %v err: %v\n", unit.Name, err)
	}

	// Tar rootfs.tar.gz to rootfs
	out, err := utils.ExecCmd("", "tar", "-xf", "rootfs.tar.gz", "-C", rootfs)
	if err != nil {
		logrus.Fatalf("Tar roofs err: %v\n", out)
	}

	// Copy runtimtest from plugins to rootfs
	src := "./plugins/runtimetest"
	dRuntimeTest := rootfs + "/runtimetest"
	err = copy(dRuntimeTest, src)
	if err != nil {
		logrus.Fatalf("Copy runtimetest to rootfs err: %v\n", err)
	}
	err = os.Chmod(dRuntimeTest, os.ModePerm)
	if err != nil {
		logrus.Fatalf("Chmod runtimetest mode err: %v\n", err)
	}

	Mutex.Lock()
	// copy *.json to testroot and rootfs
	csrc := "./plugins/config.json-" + unit.Name
	rsrc := "./plugins/runtime.json-" + unit.Name
	cdest := rootfs + "/config.json"
	rdest := rootfs + "/runtime.json"
	err = copy(cdest, csrc)
	if err != nil {
		logrus.Fatal(err)
	}
	err = copy(rdest, rsrc)
	if err != nil {
		logrus.Fatal(err)
	}

	cdest = unit.BundleDir + "/config.json"
	rdest = unit.BundleDir + "/runtime.json"
	err = copy(cdest, csrc)
	if err != nil {
		logrus.Fatal(err)
	}
	err = copy(rdest, rsrc)
	if err != nil {
		logrus.Fatal(err)
	}
	Mutex.Unlock()
}

func (unit *TestUnit) GenerateConfigs() {
	args := splitArgs(unit.Args)

	logrus.Debugf("Args to the ocitools generate: ")
	for _, a := range args {
		logrus.Debugln(a)
	}
	Mutex.Lock()
	_, err := utils.ExecGenCmd(args)
	if err != nil {
		logrus.Fatalf("Generate *.json err: %v\n", err)
	}

	copy("./plugins/runtime.json-"+unit.Name, "./plugins/runtime.json")
	if err != nil {
		logrus.Fatalf("copy to runtime.json-%v, %v", unit.Name, err)
	}

	copy("./plugins/config.json-"+unit.Name, "./plugins/config.json")
	if err != nil {
		logrus.Fatalf("copy to config.json-%v, %v", unit.Name, err)
	}
	Mutex.Unlock()
}

func splitArgs(args string) []string {

	argsnew := strings.TrimSpace(args)

	argArray := strings.Split(argsnew, "--")

	lenth := len(argArray)
	resArray := make([]string, lenth-1)
	for i, arg := range argArray {
		if i == 0 || i == lenth {
			continue
		} else {
			resArray[i-1] = "--" + strings.TrimSpace(arg)
		}
	}
	return resArray
}

func copy(dst string, src string) error {
	in, err := os.Open(src)
	if err != nil {
		return err
	}
	defer in.Close()
	out, err := os.Create(dst)
	if err != nil {
		return err
	}
	defer out.Close()
	_, err = io.Copy(out, in)
	cerr := out.Close()
	if err != nil {
		return err
	}
	return cerr
}
<commit_msg>fix a typo<commit_after>package main

import (
	"io"
	"os"
	"path"
	"strings"

	"github.com/Sirupsen/logrus"
	"github.com/huawei-openlab/oct/factory"
	"github.com/huawei-openlab/oct/utils"
	"github.com/huawei-openlab/oct/utils/config"
	"github.com/huawei-openlab/oct/utils/hooks"
)

const TestCacheDir = "./bundles/"
const (
	PASS = "SUCCESS"
	FAIL = "FAILED"
)

type TestUnit struct {
	//Case name
	Name string
	//Args is used to generate bundle
	Args string
	//Describle what does this unit test for. It is optional.
	Description string
	//Testopt is the term of OCI specs to be validate, it can be split from Args
	Testopt string

	BundleDir string
	Runtime   factory.Factory
	//success or failed
	Result string
	//when result == failed, ErrInfo is err code, or, ErrInfo is nil
	ErrInfo error
}

type UnitsManager struct {
	TestUnits []*TestUnit
}

var units *UnitsManager = new(UnitsManager)

func (this *UnitsManager) LoadTestUnits(filename string) {

	for key, value := range config.BundleMap {
		//TODO: config.BundleMap should support 'Description'
		unit := NewTestUnit(key, value, "")
		this.TestUnits = append(this.TestUnits, unit)
	}
}

func NewTestUnit(name string, args string, desc string) *TestUnit {

	tu := new(TestUnit)
	tu.Name = name
	tu.Args = args
	tu.Description = desc
	argsslice := strings.Fields(args)
	for i, arg := range argsslice {
		if strings.EqualFold(arg, "--args=./runtimetest") {
			tu.Testopt = strings.TrimPrefix(argsslice[i+1], "--args=")
		}
	}
	return tu
}

//Ouput method, ouput value: err-only or all
func (this *UnitsManager) OutputResult(output string) {

	if output != "err-only" && output != "all" {
		logrus.Fatalf("Error output cmd, output=%v\n", output)
	}

	SuccessCount := 0
	failCount := 0

	//Can not merge into on range, because output should be devided into two parts, successful and
	//failure
	if output == "all" {
		logrus.Println("Successful Details:")
		echoDividing()
	}

	for _, tu := range this.TestUnits {
		if tu.Result == PASS {
			SuccessCount++
			if output == "all" {
				tu.EchoSUnit()
			}
		}
	}

	logrus.Println("Failure Details:")
	echoDividing()

	for _, tu := range this.TestUnits {
		if tu.Result == FAIL {
			failCount++
			tu.EchoFUit()
		}
	}

	echoDividing()
	logrus.Printf("Statistics:  %v bundles success, %v bundles failed\n", SuccessCount, failCount)
}

func (unit *TestUnit) EchoSUnit() {

	logrus.Printf("\nBundleName:\n  %v\nBundleDir:\n  %v\nCaseArgs:\n  %v\nTestResult:\n  %v\n",
		unit.Name, unit.BundleDir, unit.Args, unit.Result)
}

func (unit *TestUnit) EchoFUit() {
	logrus.Printf("\nBundleName:\n  %v\nBundleDir:\n  %v\nCaseArgs:\n  %v\nResult:\n  %v\n"+
		"ErrInfo:\n  %v\n", unit.Name, unit.BundleDir, unit.Args, unit.Result, unit.ErrInfo)
}

func echoDividing() {
	logrus.Println("============================================================================" +
		"===================")
}

func (unit *TestUnit) SetResult(result string, err error) {
	unit.Result = result
	if result == PASS {
		unit.ErrInfo = nil
	} else {
		unit.ErrInfo = err
	}
}

//Set runtime
func (unit *TestUnit) SetRuntime(runtime string) error {
	if r, err := factory.CreateRuntime(runtime); err != nil {
		logrus.Printf("Create runtime %v err: %v\n", runtime, err)
		return err
	} else {
		unit.Runtime = r
	}
	return nil
}

func (unit *TestUnit) Run() {
	if unit.Runtime == nil {
		logrus.Fatalf("Set the runtime before run the test")
	}

	unit.GenerateConfigs()
	unit.PrepareBundle()

	out, err := unit.Runtime.StartRT(unit.BundleDir)
	if err != nil {
		unit.SetResult(FAIL, err)
		return
	}

	if err = unit.PostStartHooks(unit.Testopt, out); err != nil {
		unit.SetResult(FAIL, err)
		return
	}

	_ = unit.Runtime.StopRT(unit.Runtime.GetRTID())
	unit.SetResult(PASS, nil)
	return
}

func (unit *TestUnit) PostStartHooks(testopt string, out string) error {
	var err error
	switch testopt {
	case "vna":
		err = hooks.SetPostStartHooks(out, hooks.NamespacePostStart)
	default:
	}
	return err
}

func (unit *TestUnit) PrepareBundle() {
	// Create bundle follder
	unit.BundleDir = path.Join(TestCacheDir, unit.Name)
	err := os.RemoveAll(unit.BundleDir)
	if err != nil {
		logrus.Fatalf("Remove bundle %v err: %v\n", unit.Name, err)
	}

	err = os.Mkdir(unit.BundleDir, os.ModePerm)
	if err != nil {
		logrus.Fatalf("Mkdir bundle %v dir err: %v\n", unit.BundleDir, err)
	}

	// Create rootfs folder to bundle
	rootfs := unit.BundleDir + "/rootfs"
	err = os.Mkdir(rootfs, os.ModePerm)
	if err != nil {
		logrus.Fatalf("Mkdir rootfs for bundle %v err: %v\n", unit.Name, err)
	}

	// Tar rootfs.tar.gz to rootfs
	out, err := utils.ExecCmd("", "tar", "-xf", "rootfs.tar.gz", "-C", rootfs)
	if err != nil {
		logrus.Fatalf("Tar roofs err: %v\n", out)
	}

	// Copy runtimtest from plugins to rootfs
	src := "./plugins/runtimetest"
	dRuntimeTest := rootfs + "/runtimetest"
	err = copy(dRuntimeTest, src)
	if err != nil {
		logrus.Fatalf("Copy runtimetest to rootfs err: %v\n", err)
	}
	err = os.Chmod(dRuntimeTest, os.ModePerm)
	if err != nil {
		logrus.Fatalf("Chmod runtimetest mode err: %v\n", err)
	}

	Mutex.Lock()
	// copy *.json to testroot and rootfs
	csrc := "./plugins/config.json-" + unit.Name
	rsrc := "./plugins/runtime.json-" + unit.Name
	cdest := rootfs + "/config.json"
	rdest := rootfs + "/runtime.json"
	err = copy(cdest, csrc)
	if err != nil {
		logrus.Fatal(err)
	}
	err = copy(rdest, rsrc)
	if err != nil {
		logrus.Fatal(err)
	}

	cdest = unit.BundleDir + "/config.json"
	rdest = unit.BundleDir + "/runtime.json"
	err = copy(cdest, csrc)
	if err != nil {
		logrus.Fatal(err)
	}
	err = copy(rdest, rsrc)
	if err != nil {
		logrus.Fatal(err)
	}
	Mutex.Unlock()
}

func (unit *TestUnit) GenerateConfigs() {
	args := splitArgs(unit.Args)

	logrus.Debugf("Args to the ocitools generate: ")
	for _, a := range args {
		logrus.Debugln(a)
	}
	Mutex.Lock()
	_, err := utils.ExecGenCmd(args)
	if err != nil {
		logrus.Fatalf("Generate *.json err: %v\n", err)
	}

	copy("./plugins/runtime.json-"+unit.Name, "./plugins/runtime.json")
	if err != nil {
		logrus.Fatalf("copy to runtime.json-%v, %v", unit.Name, err)
	}

	copy("./plugins/config.json-"+unit.Name, "./plugins/config.json")
	if err != nil {
		logrus.Fatalf("copy to config.json-%v, %v", unit.Name, err)
	}
	Mutex.Unlock()
}

func splitArgs(args string) []string {

	argsnew := strings.TrimSpace(args)

	argArray := strings.Split(argsnew, "--")

	lenth := len(argArray)
	resArray := make([]string, lenth-1)
	for i, arg := range argArray {
		if i == 0 || i == lenth {
			continue
		} else {
			resArray[i-1] = "--" + strings.TrimSpace(arg)
		}
	}
	return resArray
}

func copy(dst string, src string) error {
	in, err := os.Open(src)
	if err != nil {
		return err
	}
	defer in.Close()
	out, err := os.Create(dst)
	if err != nil {
		return err
	}
	defer out.Close()
	_, err = io.Copy(out, in)
	cerr := out.Close()
	if err != nil {
		return err
	}
	return cerr
}
<|endoftext|>


<commit_before>package docker

import (
	"context"
	"fmt"
	"log"
	"net/http"
	"regexp"
	"strings"
	"time"

	yaml "gopkg.in/yaml.v2"

	"github.com/ViBiOh/dashboard/auth"
	"github.com/ViBiOh/httputils"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/filters"
	"github.com/docker/docker/api/types/network"
)

const minMemory = 16777216
const maxMemory = 805306368
const defaultTag = `:latest`
const deploySuffix = `_deploy`
const networkMode = `traefik`
const linkSeparator = `:`

var imageTag = regexp.MustCompile(`^\S*?:\S+$`)

type dockerComposeHealthcheck struct {
	Test     []string
	Interval string
	Timeout  string
	Retries  int
}

type dockerComposeService struct {
	Image       string
	Command     []string
	Environment map[string]string
	Labels      map[string]string
	Links       []string
	Ports       []string
	Healthcheck *dockerComposeHealthcheck
	ReadOnly    bool  `yaml:"read_only"`
	CPUShares   int64 `yaml:"cpu_shares"`
	MemoryLimit int64 `yaml:"mem_limit"`
}

type dockerCompose struct {
	Version  string
	Services map[string]dockerComposeService
}

type deployedService struct {
	ID   string
	Name string
}

func getConfig(service *dockerComposeService, user *auth.User, appName string) (*container.Config, error) {
	environments := make([]string, 0, len(service.Environment))
	for key, value := range service.Environment {
		environments = append(environments, key+`=`+value)
	}

	if service.Labels == nil {
		service.Labels = make(map[string]string)
	}

	service.Labels[ownerLabel] = user.Username
	service.Labels[appLabel] = appName

	config := container.Config{
		Image:  service.Image,
		Labels: service.Labels,
		Env:    environments,
	}

	if len(service.Command) != 0 {
		config.Cmd = service.Command
	}

	if service.Healthcheck != nil {
		healthconfig := container.HealthConfig{
			Test:    service.Healthcheck.Test,
			Retries: service.Healthcheck.Retries,
		}

		if service.Healthcheck.Interval != `` {
			interval, err := time.ParseDuration(service.Healthcheck.Interval)
			if err != nil {
				return nil, fmt.Errorf(`Error while parsing healthcheck interval: %v`, err)
			}

			healthconfig.Interval = interval
		}

		if service.Healthcheck.Timeout != `` {
			timeout, err := time.ParseDuration(service.Healthcheck.Timeout)
			if err != nil {
				return nil, fmt.Errorf(`Error while parsing healthcheck timeout: %v`, err)
			}

			healthconfig.Timeout = timeout
		}

		config.Healthcheck = &healthconfig
	}

	return &config, nil
}

func getHostConfig(service *dockerComposeService) *container.HostConfig {
	hostConfig := container.HostConfig{
		LogConfig: container.LogConfig{Type: `json-file`, Config: map[string]string{
			`max-size`: `50m`,
		}},
		NetworkMode:   networkMode,
		RestartPolicy: container.RestartPolicy{Name: `on-failure`, MaximumRetryCount: 5},
		Resources: container.Resources{
			CPUShares: 128,
			Memory:    minMemory,
		},
		SecurityOpt: []string{`no-new-privileges`},
	}

	if service.ReadOnly {
		hostConfig.ReadonlyRootfs = true
	}

	if service.CPUShares != 0 {
		hostConfig.Resources.CPUShares = service.CPUShares
	}

	if service.MemoryLimit != 0 {
		if service.MemoryLimit <= maxMemory {
			hostConfig.Resources.Memory = service.MemoryLimit
		} else {
			hostConfig.Resources.Memory = maxMemory
		}
	}

	return &hostConfig
}

func getNetworkConfig(service *dockerComposeService, deployedServices map[string]*deployedService) *network.NetworkingConfig {
	traefikConfig := network.EndpointSettings{}

	for _, link := range service.Links {
		linkParts := strings.Split(link, linkSeparator)

		target := linkParts[0]
		if linkedService, ok := deployedServices[target]; ok {
			target = getFinalName(linkedService.Name)
		}

		alias := linkParts[0]
		if len(linkParts) > 1 {
			alias = linkParts[1]
		}

		traefikConfig.Links = append(traefikConfig.Links, target+linkSeparator+alias)
	}

	return &network.NetworkingConfig{
		EndpointsConfig: map[string]*network.EndpointSettings{
			networkMode: &traefikConfig,
		},
	}
}

func pullImage(image string, user *auth.User) error {
	if !imageTag.MatchString(image) {
		image = image + defaultTag
	}

	ctx, cancel := getGracefulCtx()
	defer cancel()

	pull, err := docker.ImagePull(ctx, image, types.ImagePullOptions{})
	if err != nil {
		return fmt.Errorf(`Error while pulling image: %v`, err)
	}

	httputils.ReadBody(pull)
	return nil
}

func cleanContainers(containers []types.Container, user *auth.User) {
	for _, container := range containers {
		stopContainer(container.ID)
	}

	for _, container := range containers {
		rmContainer(container.ID)
	}
}

func renameDeployedContainers(containers map[string]*deployedService, user *auth.User) error {
	ctx, cancel := getCtx()
	defer cancel()

	for service, container := range containers {
		if err := docker.ContainerRename(ctx, container.ID, getFinalName(container.Name)); err != nil {
			return fmt.Errorf(`Error while renaming container %s: %v`, service, err)
		}
	}

	return nil
}

func getServiceFullName(app string, service string) string {
	return app + `_` + service + deploySuffix
}

func getFinalName(serviceFullName string) string {
	return strings.TrimSuffix(serviceFullName, deploySuffix)
}

func deleteServices(appName []byte, services map[string]*deployedService, user *auth.User) {
	for service, container := range services {
		if infos, err := inspectContainer(container.ID); err != nil {
			log.Printf(`[%s] [%s] Error while inspecting service %s: %v`, user.Username, appName, service, err)
		} else if infos.State.Health != nil {
			logs := make([]string, 0)

			logs = append(logs, "\n")
			for _, log := range infos.State.Health.Log {
				logs = append(logs, log.Output)
			}

			log.Printf(`[%s] [%s] Healthcheck output for %s: %s`, user.Username, appName, service, logs)
		}

		if err := stopContainer(container.ID); err != nil {
			log.Printf(`[%s] [%s] Error while stopping service %s: %v`, user.Username, appName, service, err)
		}

		if err := rmContainer(container.ID); err != nil {
			log.Printf(`[%s] [%s] Error while deleting service %s: %v`, user.Username, appName, service, err)
		}
	}
}

func startServices(appName []byte, services map[string]*deployedService, user *auth.User) error {
	for service, container := range services {
		if err := startContainer(container.ID); err != nil {
			return fmt.Errorf(`[%s] [%s] Error while starting service %s: %v`, user.Username, appName, service, err)
		}
	}

	return nil
}

func inspectServices(services map[string]*deployedService, user *auth.User) []*types.ContainerJSON {
	containers := make([]*types.ContainerJSON, 0, len(services))

	for service, container := range services {
		infos, err := inspectContainer(container.ID)
		if err != nil {
			log.Printf(`[%s] Error while inspecting container %s: %v`, user.Username, service, err)
		}

		containers = append(containers, &infos)
	}

	return containers
}

func areContainersHealthy(ctx context.Context, user *auth.User, appName []byte, containers []*types.ContainerJSON) bool {
	containersIdsWithHealthcheck := make([]string, 0, len(containers))
	for _, container := range containers {
		if container.Config.Healthcheck != nil && len(container.Config.Healthcheck.Test) != 0 {
			containersIdsWithHealthcheck = append(containersIdsWithHealthcheck, container.ID)
		}
	}

	if len(containersIdsWithHealthcheck) == 0 {
		return true
	}

	filtersArgs := filters.NewArgs()
	healthyStatusFilters(&filtersArgs, containersIdsWithHealthcheck)

	timeoutCtx, cancel := context.WithTimeout(ctx, DeployTimeout)
	defer cancel()

	messages, errors := docker.Events(timeoutCtx, types.EventsOptions{Filters: filtersArgs})
	healthyContainers := make(map[string]bool, len(containersIdsWithHealthcheck))

	for {
		select {
		case <-ctx.Done():
			return false
		case message := <-messages:
			healthyContainers[message.ID] = true
			if len(healthyContainers) == len(containersIdsWithHealthcheck) {
				return true
			}
		case err := <-errors:
			log.Printf(`[%s] [%s] Error while reading healthy events: %v`, user.Username, appName, err)
			return false
		}
	}
}

func finishDeploy(ctx context.Context, cancel context.CancelFunc, user *auth.User, appName []byte, services map[string]*deployedService, oldContainers []types.Container) {
	defer cancel()
	defer func() {
		backgroundMutex.Lock()
		defer backgroundMutex.Unlock()

		delete(backgroundTasks, string(appName))
	}()

	if areContainersHealthy(ctx, user, appName, inspectServices(services, user)) {
		cleanContainers(oldContainers, user)

		if err := renameDeployedContainers(services, user); err != nil {
			log.Printf(`[%s] [%s] Error while renaming deployed containers: %v`, user.Username, appName, err)
		}
	} else {
		deleteServices(appName, services, user)
		log.Printf(`[%s] [%s] Failed to deploy: %v`, user.Username, appName, fmt.Errorf(`[Health check failed`))
	}
}

func createContainer(user *auth.User, appName []byte, serviceName string, services map[string]*deployedService, service *dockerComposeService) (*deployedService, error) {
	if err := pullImage(service.Image, user); err != nil {
		return nil, err
	}

	serviceFullName := getServiceFullName(string(appName), serviceName)

	config, err := getConfig(service, user, string(appName))
	if err != nil {
		return nil, fmt.Errorf(`[%s] [%s] Error while getting config: %v`, user.Username, appName, err)
	}

	ctx, cancel := getCtx()
	defer cancel()

	createdContainer, err := docker.ContainerCreate(ctx, config, getHostConfig(service), getNetworkConfig(service, services), serviceFullName)
	if err != nil {
		return nil, fmt.Errorf(`[%s] [%s] Error while creating service %s: %v`, user.Username, appName, serviceName, err)
	}

	return &deployedService{ID: createdContainer.ID, Name: serviceFullName}, nil
}

func composeFailed(w http.ResponseWriter, user *auth.User, appName []byte, err error) {
	httputils.InternalServer(w, fmt.Errorf(`[%s] [%s] Failed to deploy: %v`, user.Username, appName, err))
}

func composeHandler(w http.ResponseWriter, user *auth.User, appName []byte, composeFile []byte) {
	if user == nil {
		httputils.BadRequest(w, fmt.Errorf(`A user is required`))
		return
	}

	if len(appName) == 0 || len(composeFile) == 0 {
		httputils.BadRequest(w, fmt.Errorf(`[%s] An application name and a compose file are required`, user.Username))
		return
	}

	compose := dockerCompose{}
	if err := yaml.Unmarshal(composeFile, &compose); err != nil {
		httputils.BadRequest(w, fmt.Errorf(`[%s] [%s] Error while unmarshalling compose file: %v`, user.Username, appName, err))
		return
	}

	appNameStr := string(appName)

	backgroundMutex.Lock()
	if _, ok := backgroundTasks[appNameStr]; ok {
		backgroundMutex.Unlock()
		composeFailed(w, user, appName, fmt.Errorf(`[%s] [%s] Application already in deployment`, user.Username, appName))
		return
	}

	backgroundTasks[appNameStr] = true
	backgroundMutex.Unlock()

	oldContainers, err := listContainers(user, appNameStr)
	if err != nil {
		composeFailed(w, user, appName, err)
		return
	}

	if len(oldContainers) > 0 && oldContainers[0].Labels[ownerLabel] != user.Username {
		composeFailed(w, user, appName, fmt.Errorf(`[%s] [%s] Application not owned`, user.Username, appName))
		httputils.Forbidden(w)
	}

	newServices := make(map[string]*deployedService)
	for serviceName, service := range compose.Services {
		if deployedService, err := createContainer(user, appName, serviceName, newServices, &service); err != nil {
			break
		} else {
			newServices[serviceName] = deployedService
		}
	}

	ctx, cancel := context.WithCancel(context.Background())
	go finishDeploy(ctx, cancel, user, appName, newServices, oldContainers)

	if err == nil {
		err = startServices(appName, newServices, user)
	}

	if err != nil {
		cancel()
		composeFailed(w, user, appName, err)
	} else {
		httputils.ResponseArrayJSON(w, newServices)
	}
}
<commit_msg>Restoring variable interpolation fix<commit_after>package docker

import (
	"bytes"
	"context"
	"fmt"
	"log"
	"net/http"
	"regexp"
	"strings"
	"time"

	yaml "gopkg.in/yaml.v2"

	"github.com/ViBiOh/dashboard/auth"
	"github.com/ViBiOh/httputils"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/filters"
	"github.com/docker/docker/api/types/network"
)

const minMemory = 16777216
const maxMemory = 805306368
const defaultTag = `:latest`
const deploySuffix = `_deploy`
const networkMode = `traefik`
const linkSeparator = `:`

var imageTag = regexp.MustCompile(`^\S*?:\S+$`)

type dockerComposeHealthcheck struct {
	Test     []string
	Interval string
	Timeout  string
	Retries  int
}

type dockerComposeService struct {
	Image       string
	Command     []string
	Environment map[string]string
	Labels      map[string]string
	Links       []string
	Ports       []string
	Healthcheck *dockerComposeHealthcheck
	ReadOnly    bool  `yaml:"read_only"`
	CPUShares   int64 `yaml:"cpu_shares"`
	MemoryLimit int64 `yaml:"mem_limit"`
}

type dockerCompose struct {
	Version  string
	Services map[string]dockerComposeService
}

type deployedService struct {
	ID   string
	Name string
}

func getConfig(service *dockerComposeService, user *auth.User, appName string) (*container.Config, error) {
	environments := make([]string, 0, len(service.Environment))
	for key, value := range service.Environment {
		environments = append(environments, key+`=`+value)
	}

	if service.Labels == nil {
		service.Labels = make(map[string]string)
	}

	service.Labels[ownerLabel] = user.Username
	service.Labels[appLabel] = appName

	config := container.Config{
		Image:  service.Image,
		Labels: service.Labels,
		Env:    environments,
	}

	if len(service.Command) != 0 {
		config.Cmd = service.Command
	}

	if service.Healthcheck != nil {
		healthconfig := container.HealthConfig{
			Test:    service.Healthcheck.Test,
			Retries: service.Healthcheck.Retries,
		}

		if service.Healthcheck.Interval != `` {
			interval, err := time.ParseDuration(service.Healthcheck.Interval)
			if err != nil {
				return nil, fmt.Errorf(`Error while parsing healthcheck interval: %v`, err)
			}

			healthconfig.Interval = interval
		}

		if service.Healthcheck.Timeout != `` {
			timeout, err := time.ParseDuration(service.Healthcheck.Timeout)
			if err != nil {
				return nil, fmt.Errorf(`Error while parsing healthcheck timeout: %v`, err)
			}

			healthconfig.Timeout = timeout
		}

		config.Healthcheck = &healthconfig
	}

	return &config, nil
}

func getHostConfig(service *dockerComposeService) *container.HostConfig {
	hostConfig := container.HostConfig{
		LogConfig: container.LogConfig{Type: `json-file`, Config: map[string]string{
			`max-size`: `50m`,
		}},
		NetworkMode:   networkMode,
		RestartPolicy: container.RestartPolicy{Name: `on-failure`, MaximumRetryCount: 5},
		Resources: container.Resources{
			CPUShares: 128,
			Memory:    minMemory,
		},
		SecurityOpt: []string{`no-new-privileges`},
	}

	if service.ReadOnly {
		hostConfig.ReadonlyRootfs = true
	}

	if service.CPUShares != 0 {
		hostConfig.Resources.CPUShares = service.CPUShares
	}

	if service.MemoryLimit != 0 {
		if service.MemoryLimit <= maxMemory {
			hostConfig.Resources.Memory = service.MemoryLimit
		} else {
			hostConfig.Resources.Memory = maxMemory
		}
	}

	return &hostConfig
}

func getNetworkConfig(service *dockerComposeService, deployedServices map[string]*deployedService) *network.NetworkingConfig {
	traefikConfig := network.EndpointSettings{}

	for _, link := range service.Links {
		linkParts := strings.Split(link, linkSeparator)

		target := linkParts[0]
		if linkedService, ok := deployedServices[target]; ok {
			target = getFinalName(linkedService.Name)
		}

		alias := linkParts[0]
		if len(linkParts) > 1 {
			alias = linkParts[1]
		}

		traefikConfig.Links = append(traefikConfig.Links, target+linkSeparator+alias)
	}

	return &network.NetworkingConfig{
		EndpointsConfig: map[string]*network.EndpointSettings{
			networkMode: &traefikConfig,
		},
	}
}

func pullImage(image string, user *auth.User) error {
	if !imageTag.MatchString(image) {
		image = image + defaultTag
	}

	ctx, cancel := getGracefulCtx()
	defer cancel()

	pull, err := docker.ImagePull(ctx, image, types.ImagePullOptions{})
	if err != nil {
		return fmt.Errorf(`Error while pulling image: %v`, err)
	}

	httputils.ReadBody(pull)
	return nil
}

func cleanContainers(containers []types.Container, user *auth.User) {
	for _, container := range containers {
		stopContainer(container.ID)
	}

	for _, container := range containers {
		rmContainer(container.ID)
	}
}

func renameDeployedContainers(containers map[string]*deployedService, user *auth.User) error {
	ctx, cancel := getCtx()
	defer cancel()

	for service, container := range containers {
		if err := docker.ContainerRename(ctx, container.ID, getFinalName(container.Name)); err != nil {
			return fmt.Errorf(`Error while renaming container %s: %v`, service, err)
		}
	}

	return nil
}

func getServiceFullName(app string, service string) string {
	return app + `_` + service + deploySuffix
}

func getFinalName(serviceFullName string) string {
	return strings.TrimSuffix(serviceFullName, deploySuffix)
}

func deleteServices(appName []byte, services map[string]*deployedService, user *auth.User) {
	for service, container := range services {
		if infos, err := inspectContainer(container.ID); err != nil {
			log.Printf(`[%s] [%s] Error while inspecting service %s: %v`, user.Username, appName, service, err)
		} else if infos.State.Health != nil {
			logs := make([]string, 0)

			logs = append(logs, "\n")
			for _, log := range infos.State.Health.Log {
				logs = append(logs, log.Output)
			}

			log.Printf(`[%s] [%s] Healthcheck output for %s: %s`, user.Username, appName, service, logs)
		}

		if err := stopContainer(container.ID); err != nil {
			log.Printf(`[%s] [%s] Error while stopping service %s: %v`, user.Username, appName, service, err)
		}

		if err := rmContainer(container.ID); err != nil {
			log.Printf(`[%s] [%s] Error while deleting service %s: %v`, user.Username, appName, service, err)
		}
	}
}

func startServices(appName []byte, services map[string]*deployedService, user *auth.User) error {
	for service, container := range services {
		if err := startContainer(container.ID); err != nil {
			return fmt.Errorf(`[%s] [%s] Error while starting service %s: %v`, user.Username, appName, service, err)
		}
	}

	return nil
}

func inspectServices(services map[string]*deployedService, user *auth.User) []*types.ContainerJSON {
	containers := make([]*types.ContainerJSON, 0, len(services))

	for service, container := range services {
		infos, err := inspectContainer(container.ID)
		if err != nil {
			log.Printf(`[%s] Error while inspecting container %s: %v`, user.Username, service, err)
		}

		containers = append(containers, &infos)
	}

	return containers
}

func areContainersHealthy(ctx context.Context, user *auth.User, appName []byte, containers []*types.ContainerJSON) bool {
	containersIdsWithHealthcheck := make([]string, 0, len(containers))
	for _, container := range containers {
		if container.Config.Healthcheck != nil && len(container.Config.Healthcheck.Test) != 0 {
			containersIdsWithHealthcheck = append(containersIdsWithHealthcheck, container.ID)
		}
	}

	if len(containersIdsWithHealthcheck) == 0 {
		return true
	}

	filtersArgs := filters.NewArgs()
	healthyStatusFilters(&filtersArgs, containersIdsWithHealthcheck)

	timeoutCtx, cancel := context.WithTimeout(ctx, DeployTimeout)
	defer cancel()

	messages, errors := docker.Events(timeoutCtx, types.EventsOptions{Filters: filtersArgs})
	healthyContainers := make(map[string]bool, len(containersIdsWithHealthcheck))

	for {
		select {
		case <-ctx.Done():
			return false
		case message := <-messages:
			healthyContainers[message.ID] = true
			if len(healthyContainers) == len(containersIdsWithHealthcheck) {
				return true
			}
		case err := <-errors:
			log.Printf(`[%s] [%s] Error while reading healthy events: %v`, user.Username, appName, err)
			return false
		}
	}
}

func finishDeploy(ctx context.Context, cancel context.CancelFunc, user *auth.User, appName []byte, services map[string]*deployedService, oldContainers []types.Container) {
	defer cancel()
	defer func() {
		backgroundMutex.Lock()
		defer backgroundMutex.Unlock()

		delete(backgroundTasks, string(appName))
	}()

	if areContainersHealthy(ctx, user, appName, inspectServices(services, user)) {
		cleanContainers(oldContainers, user)

		if err := renameDeployedContainers(services, user); err != nil {
			log.Printf(`[%s] [%s] Error while renaming deployed containers: %v`, user.Username, appName, err)
		}
	} else {
		deleteServices(appName, services, user)
		log.Printf(`[%s] [%s] Failed to deploy: %v`, user.Username, appName, fmt.Errorf(`[Health check failed`))
	}
}

func createContainer(user *auth.User, appName []byte, serviceName string, services map[string]*deployedService, service *dockerComposeService) (*deployedService, error) {
	if err := pullImage(service.Image, user); err != nil {
		return nil, err
	}

	serviceFullName := getServiceFullName(string(appName), serviceName)

	config, err := getConfig(service, user, string(appName))
	if err != nil {
		return nil, fmt.Errorf(`[%s] [%s] Error while getting config: %v`, user.Username, appName, err)
	}

	ctx, cancel := getCtx()
	defer cancel()

	createdContainer, err := docker.ContainerCreate(ctx, config, getHostConfig(service), getNetworkConfig(service, services), serviceFullName)
	if err != nil {
		return nil, fmt.Errorf(`[%s] [%s] Error while creating service %s: %v`, user.Username, appName, serviceName, err)
	}

	return &deployedService{ID: createdContainer.ID, Name: serviceFullName}, nil
}

func composeFailed(w http.ResponseWriter, user *auth.User, appName []byte, err error) {
	httputils.InternalServer(w, fmt.Errorf(`[%s] [%s] Failed to deploy: %v`, user.Username, appName, err))
}

func composeHandler(w http.ResponseWriter, user *auth.User, appName []byte, composeFile []byte) {
	if user == nil {
		httputils.BadRequest(w, fmt.Errorf(`A user is required`))
		return
	}

	if len(appName) == 0 || len(composeFile) == 0 {
		httputils.BadRequest(w, fmt.Errorf(`[%s] An application name and a compose file are required`, user.Username))
		return
	}

	composeFile = bytes.Replace(composeFile, []byte(`$$`), []byte(`$`), -1)

	compose := dockerCompose{}
	if err := yaml.Unmarshal(composeFile, &compose); err != nil {
		httputils.BadRequest(w, fmt.Errorf(`[%s] [%s] Error while unmarshalling compose file: %v`, user.Username, appName, err))
		return
	}

	appNameStr := string(appName)

	backgroundMutex.Lock()
	if _, ok := backgroundTasks[appNameStr]; ok {
		backgroundMutex.Unlock()
		composeFailed(w, user, appName, fmt.Errorf(`[%s] [%s] Application already in deployment`, user.Username, appName))
		return
	}

	backgroundTasks[appNameStr] = true
	backgroundMutex.Unlock()

	oldContainers, err := listContainers(user, appNameStr)
	if err != nil {
		composeFailed(w, user, appName, err)
		return
	}

	if len(oldContainers) > 0 && oldContainers[0].Labels[ownerLabel] != user.Username {
		composeFailed(w, user, appName, fmt.Errorf(`[%s] [%s] Application not owned`, user.Username, appName))
		httputils.Forbidden(w)
	}

	newServices := make(map[string]*deployedService)
	for serviceName, service := range compose.Services {
		if deployedService, err := createContainer(user, appName, serviceName, newServices, &service); err != nil {
			break
		} else {
			newServices[serviceName] = deployedService
		}
	}

	ctx, cancel := context.WithCancel(context.Background())
	go finishDeploy(ctx, cancel, user, appName, newServices, oldContainers)

	if err == nil {
		err = startServices(appName, newServices, user)
	}

	if err != nil {
		cancel()
		composeFailed(w, user, appName, err)
	} else {
		httputils.ResponseArrayJSON(w, newServices)
	}
}
<|endoftext|>


<commit_before>// Copyright 2012 The Walk Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package declarative

import (
	"fmt"
	"reflect"
	"strings"
)

import (
	"github.com/lxn/walk"
)

var (
	conditionsByName = make(map[string]walk.Condition)
	imagesByFilePath = make(map[string]walk.Image)
)

func imageFromFile(filePath string) (walk.Image, error) {
	if image, ok := imagesByFilePath[filePath]; ok {
		return image, nil
	}

	image, err := walk.NewImageFromFile(filePath)
	if err != nil {
		return nil, err
	}

	imagesByFilePath[filePath] = image

	return image, nil
}

func MustRegisterCondition(name string, condition walk.Condition) {
	if name == "" {
		panic(`name == ""`)
	}
	if condition == nil {
		panic("condition == nil")
	}
	if _, ok := conditionsByName[name]; ok {
		panic("name already registered")
	}

	conditionsByName[name] = condition
}

type declWidget struct {
	d Widget
	w walk.Window
}

type Builder struct {
	level                    int
	columns                  int
	row                      int
	col                      int
	parent                   walk.Container
	declWidgets              []declWidget
	name2Window              map[string]walk.Window
	deferredFuncs            []func() error
	knownCompositeConditions map[string]walk.Condition
}

func NewBuilder(parent walk.Container) *Builder {
	return &Builder{
		parent:                   parent,
		name2Window:              make(map[string]walk.Window),
		knownCompositeConditions: make(map[string]walk.Condition),
	}
}

func (b *Builder) Parent() walk.Container {
	return b.parent
}

func (b *Builder) Defer(f func() error) {
	b.deferredFuncs = append(b.deferredFuncs, f)
}

func (b *Builder) deferBuildMenuActions(menu *walk.Menu, items []MenuItem) {
	if len(items) > 0 {
		b.Defer(func() error {
			for _, item := range items {
				if _, err := item.createAction(b, menu); err != nil {
					return err
				}
			}

			return nil
		})
	}
}

func (b *Builder) deferBuildActions(actionList *walk.ActionList, items []MenuItem) {
	if len(items) > 0 {
		b.Defer(func() error {
			for _, item := range items {
				action, err := item.createAction(b, nil)
				if err != nil {
					return err
				}
				if err := actionList.Add(action); err != nil {
					return err
				}
			}

			return nil
		})
	}
}

func (b *Builder) InitWidget(d Widget, w walk.Window, customInit func() error) error {
	b.level++
	defer func() {
		b.level--
	}()

	var succeeded bool
	defer func() {
		if !succeeded {
			w.Dispose()
		}
	}()

	b.declWidgets = append(b.declWidgets, declWidget{d, w})

	// Widget
	name, _, _, font, toolTipText, minSize, maxSize, stretchFactor, row, rowSpan, column, columnSpan, alwaysConsumeSpace, contextMenuItems, onKeyDown, onKeyPress, onKeyUp, onMouseDown, onMouseMove, onMouseUp, onSizeChanged := d.WidgetInfo()

	w.SetName(name)

	if name != "" {
		b.name2Window[name] = w
	}

	if toolTipText != "" {
		if widget, ok := w.(walk.Widget); ok {
			if err := widget.SetToolTipText(toolTipText); err != nil {
				return err
			}
		}
	}

	if err := w.SetMinMaxSize(minSize.toW(), maxSize.toW()); err != nil {
		return err
	}

	if len(contextMenuItems) > 0 {
		cm, err := walk.NewMenu()
		if err != nil {
			return err
		}

		b.deferBuildMenuActions(cm, contextMenuItems)

		w.SetContextMenu(cm)
	}

	if onKeyDown != nil {
		w.KeyDown().Attach(onKeyDown)
	}

	if onKeyPress != nil {
		w.KeyPress().Attach(onKeyPress)
	}

	if onKeyUp != nil {
		w.KeyUp().Attach(onKeyUp)
	}

	if onMouseDown != nil {
		w.MouseDown().Attach(onMouseDown)
	}

	if onMouseMove != nil {
		w.MouseMove().Attach(onMouseMove)
	}

	if onMouseUp != nil {
		w.MouseUp().Attach(onMouseUp)
	}

	if onSizeChanged != nil {
		w.SizeChanged().Attach(onSizeChanged)
	}

	if widget, ok := w.(walk.Widget); ok {
		if err := widget.SetAlwaysConsumeSpace(alwaysConsumeSpace); err != nil {
			return err
		}

		type SetStretchFactorer interface {
			SetStretchFactor(widget walk.Widget, factor int) error
		}

		if p := widget.Parent(); p != nil {
			switch l := p.Layout().(type) {
			case SetStretchFactorer:
				if stretchFactor < 1 {
					stretchFactor = 1
				}
				if err := l.SetStretchFactor(widget, stretchFactor); err != nil {
					return err
				}

			case *walk.GridLayout:
				if rowSpan < 1 {
					rowSpan = 1
				}
				if columnSpan < 1 {
					columnSpan = 1
				}

				if b.columns > 0 && row == 0 && column == 0 {
					if b.col+columnSpan > b.columns {
						b.row++
						b.col = 0
					}

					row = b.row
					column = b.col

					b.col += columnSpan
				}

				r := walk.Rectangle{column, row, columnSpan, rowSpan}

				if err := l.SetRange(widget, r); err != nil {
					return err
				}
			}
		}
	}

	oldParent := b.parent

	// Container
	var db *walk.DataBinder
	if dc, ok := d.(Container); ok {
		if wc, ok := w.(walk.Container); ok {
			dataBinder, layout, children := dc.ContainerInfo()

			if layout != nil {
				l, err := layout.Create()
				if err != nil {
					return err
				}

				if err := wc.SetLayout(l); err != nil {
					return err
				}
			}

			b.parent = wc
			defer func() {
				b.parent = oldParent
			}()

			if g, ok := layout.(Grid); ok {
				columns := b.columns
				defer func() {
					b.columns, b.row, b.col = columns, row, column+columnSpan
				}()

				b.columns = g.Columns
				b.row = 0
				b.col = 0
			}

			for _, child := range children {
				if err := child.Create(b); err != nil {
					return err
				}
			}

			if dataBinder.AssignTo != nil || dataBinder.DataSource != nil {
				if dataB, err := dataBinder.create(); err != nil {
					return err
				} else {
					db = dataB
				}
			}
		}
	}

	// Custom
	if customInit != nil {
		if err := customInit(); err != nil {
			return err
		}
	}

	b.parent = oldParent

	// Widget continued
	if font != nil {
		if f, err := font.Create(); err != nil {
			return err
		} else if f != nil {
			w.SetFont(f)
		}
	}

	if b.level == 1 {
		if err := b.initProperties(); err != nil {
			return err
		}
	}

	// Call Reset on DataBinder after customInit, so a Dialog gets a chance to first
	// wire up its DefaultButton to the CanSubmitChanged event of a DataBinder.
	if db != nil {
		if _, ok := d.(Container); ok {
			if wc, ok := w.(walk.Container); ok {
				b.Defer(func() error {
					// FIXME: Currently SetDataBinder must be called after initProperties.
					wc.SetDataBinder(db)

					if db.DataSource() == nil {
						return nil
					}

					return db.Reset()
				})
			}
		}
	}

	if b.level == 1 {
		for _, f := range b.deferredFuncs {
			if err := f(); err != nil {
				return err
			}
		}
	}

	succeeded = true

	return nil
}

func (b *Builder) initProperties() error {
	for _, dw := range b.declWidgets {
		d, w := dw.d, dw.w

		sv := reflect.ValueOf(d)
		st := sv.Type()
		if st.Kind() != reflect.Struct {
			panic("d must be a struct value")
		}

		wb := w.AsWindowBase()

		fieldCount := st.NumField()
		for i := 0; i < fieldCount; i++ {
			sf := st.Field(i)

			prop := wb.Property(sf.Name)

			switch val := sv.Field(i).Interface().(type) {
			case nil:
				// nop

			case bindData:
				if prop == nil {
					panic(sf.Name + " is not a property")
				}

				src := b.conditionOrProperty(val)

				if src == nil {
					// No luck so far, so we assume the expression refers to
					// something in the data source.
					src = val.expression

					if val.validator != nil {
						validator, err := val.validator.Create()
						if err != nil {
							return err
						}
						if err := prop.SetValidator(validator); err != nil {
							return err
						}
					}
				}

				if err := prop.SetSource(src); err != nil {
					return err
				}

			case walk.Condition:
				if prop == nil {
					panic(sf.Name + " is not a property")
				}

				if err := prop.SetSource(val); err != nil {
					return err
				}

			default:
				if prop == nil {
					continue
				}

				v := prop.Get()
				valt, vt := reflect.TypeOf(val), reflect.TypeOf(v)

				if v != nil && valt != vt {
					panic(fmt.Sprintf("cannot assign value %v of type %T to property %s of type %T", val, val, sf.Name, v))
				}
				if err := prop.Set(val); err != nil {
					return err
				}
			}
		}
	}

	return nil
}

func (b *Builder) conditionOrProperty(data Property) interface{} {
	switch val := data.(type) {
	case bindData:
		if c, ok := b.knownCompositeConditions[val.expression]; ok {
			return c
		} else if conds := strings.Split(val.expression, "&&"); len(conds) > 1 {
			// This looks like a composite condition.
			for i, s := range conds {
				conds[i] = strings.TrimSpace(s)
			}

			var conditions []walk.Condition

			for _, cond := range conds {
				if p := b.property(cond); p != nil {
					conditions = append(conditions, p.(walk.Condition))
				} else if c, ok := conditionsByName[cond]; ok {
					conditions = append(conditions, c)
				} else {
					panic("unknown condition or property name: " + cond)
				}
			}

			var condition walk.Condition
			if len(conditions) > 1 {
				condition = walk.NewAllCondition(conditions...)
				b.knownCompositeConditions[val.expression] = condition
			} else {
				condition = conditions[0]
			}

			return condition
		}

		if p := b.property(val.expression); p != nil {
			return p
		}

		return conditionsByName[val.expression]

	case walk.Condition:
		return val
	}

	return nil
}

func (b *Builder) property(expression string) walk.Property {
	if parts := strings.Split(expression, "."); len(parts) == 2 {
		if sw, ok := b.name2Window[parts[0]]; ok {
			return sw.AsWindowBase().Property(parts[1])
		}
	}

	return nil
}
<commit_msg>declarative/Builder: Apply StretchFactor to GridLayouts<commit_after>// Copyright 2012 The Walk Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package declarative

import (
	"fmt"
	"reflect"
	"strings"
)

import (
	"github.com/lxn/walk"
)

var (
	conditionsByName = make(map[string]walk.Condition)
	imagesByFilePath = make(map[string]walk.Image)
)

func imageFromFile(filePath string) (walk.Image, error) {
	if image, ok := imagesByFilePath[filePath]; ok {
		return image, nil
	}

	image, err := walk.NewImageFromFile(filePath)
	if err != nil {
		return nil, err
	}

	imagesByFilePath[filePath] = image

	return image, nil
}

func MustRegisterCondition(name string, condition walk.Condition) {
	if name == "" {
		panic(`name == ""`)
	}
	if condition == nil {
		panic("condition == nil")
	}
	if _, ok := conditionsByName[name]; ok {
		panic("name already registered")
	}

	conditionsByName[name] = condition
}

type declWidget struct {
	d Widget
	w walk.Window
}

type Builder struct {
	level                    int
	columns                  int
	row                      int
	col                      int
	parent                   walk.Container
	declWidgets              []declWidget
	name2Window              map[string]walk.Window
	deferredFuncs            []func() error
	knownCompositeConditions map[string]walk.Condition
}

func NewBuilder(parent walk.Container) *Builder {
	return &Builder{
		parent:                   parent,
		name2Window:              make(map[string]walk.Window),
		knownCompositeConditions: make(map[string]walk.Condition),
	}
}

func (b *Builder) Parent() walk.Container {
	return b.parent
}

func (b *Builder) Defer(f func() error) {
	b.deferredFuncs = append(b.deferredFuncs, f)
}

func (b *Builder) deferBuildMenuActions(menu *walk.Menu, items []MenuItem) {
	if len(items) > 0 {
		b.Defer(func() error {
			for _, item := range items {
				if _, err := item.createAction(b, menu); err != nil {
					return err
				}
			}

			return nil
		})
	}
}

func (b *Builder) deferBuildActions(actionList *walk.ActionList, items []MenuItem) {
	if len(items) > 0 {
		b.Defer(func() error {
			for _, item := range items {
				action, err := item.createAction(b, nil)
				if err != nil {
					return err
				}
				if err := actionList.Add(action); err != nil {
					return err
				}
			}

			return nil
		})
	}
}

func (b *Builder) InitWidget(d Widget, w walk.Window, customInit func() error) error {
	b.level++
	defer func() {
		b.level--
	}()

	var succeeded bool
	defer func() {
		if !succeeded {
			w.Dispose()
		}
	}()

	b.declWidgets = append(b.declWidgets, declWidget{d, w})

	// Widget
	name, _, _, font, toolTipText, minSize, maxSize, stretchFactor, row, rowSpan, column, columnSpan, alwaysConsumeSpace, contextMenuItems, onKeyDown, onKeyPress, onKeyUp, onMouseDown, onMouseMove, onMouseUp, onSizeChanged := d.WidgetInfo()

	w.SetName(name)

	if name != "" {
		b.name2Window[name] = w
	}

	if toolTipText != "" {
		if widget, ok := w.(walk.Widget); ok {
			if err := widget.SetToolTipText(toolTipText); err != nil {
				return err
			}
		}
	}

	if err := w.SetMinMaxSize(minSize.toW(), maxSize.toW()); err != nil {
		return err
	}

	if len(contextMenuItems) > 0 {
		cm, err := walk.NewMenu()
		if err != nil {
			return err
		}

		b.deferBuildMenuActions(cm, contextMenuItems)

		w.SetContextMenu(cm)
	}

	if onKeyDown != nil {
		w.KeyDown().Attach(onKeyDown)
	}

	if onKeyPress != nil {
		w.KeyPress().Attach(onKeyPress)
	}

	if onKeyUp != nil {
		w.KeyUp().Attach(onKeyUp)
	}

	if onMouseDown != nil {
		w.MouseDown().Attach(onMouseDown)
	}

	if onMouseMove != nil {
		w.MouseMove().Attach(onMouseMove)
	}

	if onMouseUp != nil {
		w.MouseUp().Attach(onMouseUp)
	}

	if onSizeChanged != nil {
		w.SizeChanged().Attach(onSizeChanged)
	}

	if widget, ok := w.(walk.Widget); ok {
		if err := widget.SetAlwaysConsumeSpace(alwaysConsumeSpace); err != nil {
			return err
		}

		type SetStretchFactorer interface {
			SetStretchFactor(widget walk.Widget, factor int) error
		}

		if p := widget.Parent(); p != nil {
			if stretchFactor < 1 {
				stretchFactor = 1
			}

			switch l := p.Layout().(type) {
			case SetStretchFactorer:
				if err := l.SetStretchFactor(widget, stretchFactor); err != nil {
					return err
				}

			case *walk.GridLayout:
				csf := l.ColumnStretchFactor(column)
				if csf < stretchFactor {
					csf = stretchFactor
				}
				l.SetColumnStretchFactor(column, csf)

				rsf := l.RowStretchFactor(row)
				if rsf < stretchFactor {
					rsf = stretchFactor
				}
				l.SetRowStretchFactor(row, rsf)

				if rowSpan < 1 {
					rowSpan = 1
				}
				if columnSpan < 1 {
					columnSpan = 1
				}

				if b.columns > 0 && row == 0 && column == 0 {
					if b.col+columnSpan > b.columns {
						b.row++
						b.col = 0
					}

					row = b.row
					column = b.col

					b.col += columnSpan
				}

				r := walk.Rectangle{column, row, columnSpan, rowSpan}

				if err := l.SetRange(widget, r); err != nil {
					return err
				}
			}
		}
	}

	oldParent := b.parent

	// Container
	var db *walk.DataBinder
	if dc, ok := d.(Container); ok {
		if wc, ok := w.(walk.Container); ok {
			dataBinder, layout, children := dc.ContainerInfo()

			if layout != nil {
				l, err := layout.Create()
				if err != nil {
					return err
				}

				if err := wc.SetLayout(l); err != nil {
					return err
				}
			}

			b.parent = wc
			defer func() {
				b.parent = oldParent
			}()

			if g, ok := layout.(Grid); ok {
				columns := b.columns
				defer func() {
					b.columns, b.row, b.col = columns, row, column+columnSpan
				}()

				b.columns = g.Columns
				b.row = 0
				b.col = 0
			}

			for _, child := range children {
				if err := child.Create(b); err != nil {
					return err
				}
			}

			if dataBinder.AssignTo != nil || dataBinder.DataSource != nil {
				if dataB, err := dataBinder.create(); err != nil {
					return err
				} else {
					db = dataB
				}
			}
		}
	}

	// Custom
	if customInit != nil {
		if err := customInit(); err != nil {
			return err
		}
	}

	b.parent = oldParent

	// Widget continued
	if font != nil {
		if f, err := font.Create(); err != nil {
			return err
		} else if f != nil {
			w.SetFont(f)
		}
	}

	if b.level == 1 {
		if err := b.initProperties(); err != nil {
			return err
		}
	}

	// Call Reset on DataBinder after customInit, so a Dialog gets a chance to first
	// wire up its DefaultButton to the CanSubmitChanged event of a DataBinder.
	if db != nil {
		if _, ok := d.(Container); ok {
			if wc, ok := w.(walk.Container); ok {
				b.Defer(func() error {
					// FIXME: Currently SetDataBinder must be called after initProperties.
					wc.SetDataBinder(db)

					if db.DataSource() == nil {
						return nil
					}

					return db.Reset()
				})
			}
		}
	}

	if b.level == 1 {
		for _, f := range b.deferredFuncs {
			if err := f(); err != nil {
				return err
			}
		}
	}

	succeeded = true

	return nil
}

func (b *Builder) initProperties() error {
	for _, dw := range b.declWidgets {
		d, w := dw.d, dw.w

		sv := reflect.ValueOf(d)
		st := sv.Type()
		if st.Kind() != reflect.Struct {
			panic("d must be a struct value")
		}

		wb := w.AsWindowBase()

		fieldCount := st.NumField()
		for i := 0; i < fieldCount; i++ {
			sf := st.Field(i)

			prop := wb.Property(sf.Name)

			switch val := sv.Field(i).Interface().(type) {
			case nil:
				// nop

			case bindData:
				if prop == nil {
					panic(sf.Name + " is not a property")
				}

				src := b.conditionOrProperty(val)

				if src == nil {
					// No luck so far, so we assume the expression refers to
					// something in the data source.
					src = val.expression

					if val.validator != nil {
						validator, err := val.validator.Create()
						if err != nil {
							return err
						}
						if err := prop.SetValidator(validator); err != nil {
							return err
						}
					}
				}

				if err := prop.SetSource(src); err != nil {
					return err
				}

			case walk.Condition:
				if prop == nil {
					panic(sf.Name + " is not a property")
				}

				if err := prop.SetSource(val); err != nil {
					return err
				}

			default:
				if prop == nil {
					continue
				}

				v := prop.Get()
				valt, vt := reflect.TypeOf(val), reflect.TypeOf(v)

				if v != nil && valt != vt {
					panic(fmt.Sprintf("cannot assign value %v of type %T to property %s of type %T", val, val, sf.Name, v))
				}
				if err := prop.Set(val); err != nil {
					return err
				}
			}
		}
	}

	return nil
}

func (b *Builder) conditionOrProperty(data Property) interface{} {
	switch val := data.(type) {
	case bindData:
		if c, ok := b.knownCompositeConditions[val.expression]; ok {
			return c
		} else if conds := strings.Split(val.expression, "&&"); len(conds) > 1 {
			// This looks like a composite condition.
			for i, s := range conds {
				conds[i] = strings.TrimSpace(s)
			}

			var conditions []walk.Condition

			for _, cond := range conds {
				if p := b.property(cond); p != nil {
					conditions = append(conditions, p.(walk.Condition))
				} else if c, ok := conditionsByName[cond]; ok {
					conditions = append(conditions, c)
				} else {
					panic("unknown condition or property name: " + cond)
				}
			}

			var condition walk.Condition
			if len(conditions) > 1 {
				condition = walk.NewAllCondition(conditions...)
				b.knownCompositeConditions[val.expression] = condition
			} else {
				condition = conditions[0]
			}

			return condition
		}

		if p := b.property(val.expression); p != nil {
			return p
		}

		return conditionsByName[val.expression]

	case walk.Condition:
		return val
	}

	return nil
}

func (b *Builder) property(expression string) walk.Property {
	if parts := strings.Split(expression, "."); len(parts) == 2 {
		if sw, ok := b.name2Window[parts[0]]; ok {
			return sw.AsWindowBase().Property(parts[1])
		}
	}

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package model

import (
	"fmt"
	"io/ioutil"
	"regexp"

	"k8s.io/klog"
	"k8s.io/kops/nodeup/pkg/distros"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
)

// NTPBuilder installs and starts NTP, to ensure accurate clock times.
// As well as general log confusion, clock-skew of more than 5 minutes
// causes AWS API calls to fail
type NTPBuilder struct {
	*NodeupModelContext
}

var _ fi.ModelBuilder = &NTPBuilder{}

type ntpDaemon string

var (
	chronyd ntpDaemon = "chronyd"
	ntpd    ntpDaemon = "ntpd"
)

// Build is responsible for configuring NTP
func (b *NTPBuilder) Build(c *fi.ModelBuilderContext) error {
	switch b.Distribution {
	case distros.DistributionContainerOS:
		klog.Infof("Detected ContainerOS; won't install ntp")
		return nil
	case distros.DistributionCoreOS:
		klog.Infof("Detected CoreOS; won't install ntp")
		return nil
	case distros.DistributionFlatcar:
		klog.Infof("Detected Flatcar; won't install ntp")
		return nil
	}

	var ntpIP string
	switch b.Cluster.Spec.CloudProvider {
	case "aws":
		ntpIP = "169.254.169.123"
	case "gce":
		ntpIP = "time.google.com"
	default:
		ntpIP = ""
	}

	if b.Distribution.IsDebianFamily() {
		c.AddTask(&nodetasks.Package{Name: "ntp"})

		if ntpIP != "" {
			bytes, err := updateNtpIP(ntpIP, ntpd)
			if err != nil {
				return err
			}
			c.AddTask(&nodetasks.File{
				Path:     "/etc/ntp.conf",
				Contents: fi.NewBytesResource(bytes),
				Type:     nodetasks.FileType_File,
				Mode:     s("0644"),
			})
		}

		c.AddTask((&nodetasks.Service{Name: "ntp"}).InitDefaults())
	} else if b.Distribution.IsRHELFamily() {
		switch b.Distribution {
		case distros.DistributionCentos8, distros.DistributionRhel8:
			c.AddTask(&nodetasks.Package{Name: "chrony"})

			if ntpIP != "" {
				bytes, err := updateNtpIP(ntpIP, chronyd)
				if err != nil {
					return err
				}
				c.AddTask(&nodetasks.File{
					Path:     "/etc/chrony.conf",
					Contents: fi.NewBytesResource(bytes),
					Type:     nodetasks.FileType_File,
					Mode:     s("0644"),
				})
			}
			c.AddTask((&nodetasks.Service{Name: "chronyd"}).InitDefaults())

		default:
			c.AddTask(&nodetasks.Package{Name: "ntp"})

			if ntpIP != "" {
				bytes, err := updateNtpIP(ntpIP, ntpd)
				if err != nil {
					return err
				}
				c.AddTask(&nodetasks.File{
					Path:     "/etc/ntp.conf",
					Contents: fi.NewBytesResource(bytes),
					Type:     nodetasks.FileType_File,
					Mode:     s("0644"),
				})
			}

			c.AddTask((&nodetasks.Service{Name: "ntpd"}).InitDefaults())
		}
	} else {
		klog.Warningf("unknown distribution, skipping ntp install: %v", b.Distribution)
		return nil
	}
	return nil
}

// updateNtpIP takes a ip and a ntpDaemon and will comment out
// the default server or pool values and append the correct cloud
// ip to the ntp config file.
func updateNtpIP(ip string, daemon ntpDaemon) ([]byte, error) {
	var address string
	var path string
	r := regexp.MustCompile(`(?m)(^pool|^server)\s.*`)
	switch daemon {
	case ntpd:
		address = fmt.Sprintf("server %s prefer iburst\n", ip)
		path = "/etc/ntp.conf"
	case chronyd:
		address = fmt.Sprintf("server %s prefer iburst minpoll 4 maxpoll 4\n", ip)
		path = "/etc/chrony.conf"
	default:
		return nil, fmt.Errorf("%s is not a supported ntp application", daemon)
	}

	f, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}

	new := r.ReplaceAllFunc(f, func(b []byte) []byte {
		return []byte(fmt.Sprintf("#commented out by kops %s", string(b)))
	})
	new = append(new, []byte(address)...)
	return new, nil
}
<commit_msg>Update nodeup/pkg/model/ntp.go<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package model

import (
	"fmt"
	"io/ioutil"
	"regexp"

	"k8s.io/klog"
	"k8s.io/kops/nodeup/pkg/distros"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
)

// NTPBuilder installs and starts NTP, to ensure accurate clock times.
// As well as general log confusion, clock-skew of more than 5 minutes
// causes AWS API calls to fail
type NTPBuilder struct {
	*NodeupModelContext
}

var _ fi.ModelBuilder = &NTPBuilder{}

type ntpDaemon string

var (
	chronyd ntpDaemon = "chronyd"
	ntpd    ntpDaemon = "ntpd"
)

// Build is responsible for configuring NTP
func (b *NTPBuilder) Build(c *fi.ModelBuilderContext) error {
	switch b.Distribution {
	case distros.DistributionContainerOS:
		klog.Infof("Detected ContainerOS; won't install ntp")
		return nil
	case distros.DistributionCoreOS:
		klog.Infof("Detected CoreOS; won't install ntp")
		return nil
	case distros.DistributionFlatcar:
		klog.Infof("Detected Flatcar; won't install ntp")
		return nil
	}

	var ntpIP string
	switch b.Cluster.Spec.CloudProvider {
	case "aws":
		ntpIP = "169.254.169.123"
	case "gce":
		ntpIP = "time.google.com"
	default:
		ntpIP = ""
	}

	if b.Distribution.IsDebianFamily() {
		c.AddTask(&nodetasks.Package{Name: "ntp"})

		if ntpIP != "" {
			bytes, err := updateNtpIP(ntpIP, ntpd)
			if err != nil {
				return err
			}
			c.AddTask(&nodetasks.File{
				Path:     "/etc/ntp.conf",
				Contents: fi.NewBytesResource(bytes),
				Type:     nodetasks.FileType_File,
				Mode:     s("0644"),
			})
		}

		c.AddTask((&nodetasks.Service{Name: "ntp"}).InitDefaults())
	} else if b.Distribution.IsRHELFamily() {
		switch b.Distribution {
		case distros.DistributionCentos8, distros.DistributionRhel8:
			c.AddTask(&nodetasks.Package{Name: "chrony"})

			if ntpIP != "" {
				bytes, err := updateNtpIP(ntpIP, chronyd)
				if err != nil {
					return err
				}
				c.AddTask(&nodetasks.File{
					Path:     "/etc/chrony.conf",
					Contents: fi.NewBytesResource(bytes),
					Type:     nodetasks.FileType_File,
					Mode:     s("0644"),
				})
			}
			c.AddTask((&nodetasks.Service{Name: "chronyd"}).InitDefaults())

		default:
			c.AddTask(&nodetasks.Package{Name: "ntp"})

			if ntpIP != "" {
				bytes, err := updateNtpIP(ntpIP, ntpd)
				if err != nil {
					return err
				}
				c.AddTask(&nodetasks.File{
					Path:     "/etc/ntp.conf",
					Contents: fi.NewBytesResource(bytes),
					Type:     nodetasks.FileType_File,
					Mode:     s("0644"),
				})
			}

			c.AddTask((&nodetasks.Service{Name: "ntpd"}).InitDefaults())
		}
	} else {
		klog.Warningf("unknown distribution, skipping ntp install: %v", b.Distribution)
		return nil
	}
	return nil
}

// updateNtpIP takes a ip and a ntpDaemon and will comment out
// the default server or pool values and append the correct cloud
// ip to the ntp config file.
func updateNtpIP(ip string, daemon ntpDaemon) ([]byte, error) {
	var address string
	var path string
	r := regexp.MustCompile(`(?m)^(?:pool|server)\s.*`)
	switch daemon {
	case ntpd:
		address = fmt.Sprintf("server %s prefer iburst\n", ip)
		path = "/etc/ntp.conf"
	case chronyd:
		address = fmt.Sprintf("server %s prefer iburst minpoll 4 maxpoll 4\n", ip)
		path = "/etc/chrony.conf"
	default:
		return nil, fmt.Errorf("%s is not a supported ntp application", daemon)
	}

	f, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}

	new := r.ReplaceAllFunc(f, func(b []byte) []byte {
		return []byte(fmt.Sprintf("#commented out by kops %s", string(b)))
	})
	new = append(new, []byte(address)...)
	return new, nil
}
<|endoftext|>


<commit_before>package dirnfiles

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/keenstart/keennodes/gopfile"
	"github.com/keenstart/keennodes/khash"
)

const (
	PROCESSROOT = "/Users/garethharris/"
	PROCESSEXT  = ".jpg,.JPG,.PNG,.png" //,.PNG,.png

	BLOBFILE  = "/tmp/blob.bl"
	BLOCKSIZE = 1024
)

type Dirinfo struct {
	Key          int
	Path         string
	Fsize        int64
	Name         string
	Modtime      string
	Mode         string
	FileChecksum uint64
}

type Dirs struct {
	Files map[int]*Dirinfo
}

func NewDirs() *Dirs {
	return &Dirs{Files: make(map[int]*Dirinfo)}
}

func NewDirinfo(key int, path string, fsize int64, name string, modtime string, mode string) *Dirinfo {

	chksm := khash.Hashcrc64(khash.Filebytes(path))

	return &Dirinfo{
		Key:          key,
		Path:         path,
		Fsize:        fsize,
		Name:         name,
		Modtime:      modtime,
		Mode:         mode,
		FileChecksum: chksm,
	}

}

func (d *Dirs) GetDirsfile() error {

	var key int

	err := filepath.Walk(PROCESSROOT, func(path string, f os.FileInfo, err error) error {
		if !f.IsDir() && f.Mode().IsRegular() && f.Size() > BLOCKSIZE {

			if strings.Contains(PROCESSEXT, filepath.Ext(path)) == true &&
				len(filepath.Ext(path)) > 1 {

				fmt.Println("EXT = ", filepath.Ext(path)) //Debug
				//dd := &Dirinfo{path: path, fsize: f.Size(), name: f.Name(), modtime: f.ModTime().String()}
				dd := NewDirinfo(key, path, f.Size(), f.Name(), f.ModTime().String(), f.Mode().String())

				d.Files[key] = dd

				key++

				if err != nil {
					return err
				}
			}
		}
		return nil
	})

	if err != nil {
		return err
	}

	fmt.Println("Total files = ", key) //Debug

	return nil

}

func (d *Dirs) DisplayPath() {
	fmt.Println("Display")
	for _, value := range d.Files {
		fmt.Printf("Key %d == %s with %d bytes. Name = %s, modify time = %s, file mode = %s FileChecksum %x\n",
			value.Key, value.Path, value.Fsize, value.Name, value.Modtime, value.Mode, value.FileChecksum)

	}

}

func (d *Dirs) GetFiles() error {
	err := gopfile.Load(BLOBFILE, d)

	if err != nil {
		return err
	}

	return nil

}

func (d *Dirs) SetFiles() error {
	err := gopfile.Save(BLOBFILE, d)

	if err != nil {
		return err
	}

	return nil

}
<commit_msg>Update dirnfiles.go<commit_after>package dirnfiles

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/keenstart/keennodes/gopfile"
	"github.com/keenstart/keennodes/khash"
)

const (
	PROCESSROOT = "/Users/garethharris/"
	PROCESSEXT  = ".jpg,.JPG,.PNG,.png" //,.PNG,.png

	BLOBFILE  = "/tmp/blob.bl"
	BLOCKSIZE = 1024
)

type Dirinfo struct {
	Key          int
	Path         string
	Fsize        int64
	Name         string //remove
	Modtime      string //remove
	Mode         string // remove
	FileChecksum uint64
	//active       bool  //add - when ative it
			     // means the process is
			     // done on this file
			     // and it is not corrupted or mssing

}

type Dirs struct {
	Files map[int]*Dirinfo
}

func NewDirs() *Dirs {
	return &Dirs{Files: make(map[int]*Dirinfo)}
}

func NewDirinfo(key int, path string, fsize int64, name string, modtime string, mode string) *Dirinfo {

	chksm := khash.Hashcrc64(khash.Filebytes(path))

	return &Dirinfo{
		Key:          key,
		Path:         path,
		Fsize:        fsize,
		Name:         name,
		Modtime:      modtime,
		Mode:         mode,
		FileChecksum: chksm,
	}

}

func (d *Dirs) GetDirsfile() error {

	var key int

	err := filepath.Walk(PROCESSROOT, func(path string, f os.FileInfo, err error) error {
		if !f.IsDir() && f.Mode().IsRegular() && f.Size() > BLOCKSIZE {

			if strings.Contains(PROCESSEXT, filepath.Ext(path)) == true &&
				len(filepath.Ext(path)) > 1 {

				fmt.Println("EXT = ", filepath.Ext(path)) //Debug
				//dd := &Dirinfo{path: path, fsize: f.Size(), name: f.Name(), modtime: f.ModTime().String()}
				dd := NewDirinfo(key, path, f.Size(), f.Name(), f.ModTime().String(), f.Mode().String())

				d.Files[key] = dd

				key++

				if err != nil {
					return err
				}
			}
		}
		return nil
	})

	if err != nil {
		return err
	}

	fmt.Println("Total files = ", key) //Debug

	return nil

}

func (d *Dirs) DisplayPath() {
	fmt.Println("Display")
	for _, value := range d.Files {
		fmt.Printf("Key %d == %s with %d bytes. Name = %s, modify time = %s, file mode = %s FileChecksum %x\n",
			value.Key, value.Path, value.Fsize, value.Name, value.Modtime, value.Mode, value.FileChecksum)

	}

}

func (d *Dirs) GetFiles() error {
	err := gopfile.Load(BLOBFILE, d)

	if err != nil {
		return err
	}

	return nil

}

func (d *Dirs) SetFiles() error {
	err := gopfile.Save(BLOBFILE, d)

	if err != nil {
		return err
	}

	return nil

}
<|endoftext|>


<commit_before>package main

import (
	"strings"
	"syscall/js"

	"github.com/ikawaha/kagome-dict/ipa"
	"github.com/ikawaha/kagome/v2/tokenizer"
)

func igOK(s string, _ bool) string {
	return s
}

func tokenize(_ js.Value, args []js.Value) interface{} {
	if len(args) == 0 {
		return nil
	}
	t, err := tokenizer.New(ipa.Dict(), tokenizer.OmitBosEos())
	if err != nil {
		return nil
	}
	var ret []interface{}
	tokens := t.Tokenize(args[0].String())
	for _, v := range tokens {
		//fmt.Printf("%s\t%+v%v\n", v.Surface, v.POS(), strings.Join(v.Features(), ","))
		ret = append(ret, map[string]interface{}{
			"word_id":       v.ID,
			"word_type":     v.Class.String(),
			"word_position": v.Start,
			"surface_form":  v.Surface,
			"pos":           strings.Join(v.POS(), ","),
			"base_form":     igOK(v.BaseForm()),
			"reading":       igOK(v.Reading()),
			"pronunciation": igOK(v.Pronunciation()),
		})
	}
	return ret
}

func registerCallbacks() {
	_ = ipa.Dict()
	js.Global().Set("kagome_tokenize", js.FuncOf(tokenize))
}

func main() {
	c := make(chan struct{}, 0)
	registerCallbacks()
	println("Kagome Web Assembly Ready")
	<-c
}
<commit_msg>Add go:build ignore<commit_after>//go:build ignore
// +build ignore

package main

import (
	"strings"
	"syscall/js"

	"github.com/ikawaha/kagome-dict/ipa"
	"github.com/ikawaha/kagome/v2/tokenizer"
)

func igOK(s string, _ bool) string {
	return s
}

func tokenize(_ js.Value, args []js.Value) interface{} {
	if len(args) == 0 {
		return nil
	}
	t, err := tokenizer.New(ipa.Dict(), tokenizer.OmitBosEos())
	if err != nil {
		return nil
	}
	var ret []interface{}
	tokens := t.Tokenize(args[0].String())
	for _, v := range tokens {
		//fmt.Printf("%s\t%+v%v\n", v.Surface, v.POS(), strings.Join(v.Features(), ","))
		ret = append(ret, map[string]interface{}{
			"word_id":       v.ID,
			"word_type":     v.Class.String(),
			"word_position": v.Start,
			"surface_form":  v.Surface,
			"pos":           strings.Join(v.POS(), ","),
			"base_form":     igOK(v.BaseForm()),
			"reading":       igOK(v.Reading()),
			"pronunciation": igOK(v.Pronunciation()),
		})
	}
	return ret
}

func registerCallbacks() {
	_ = ipa.Dict()
	js.Global().Set("kagome_tokenize", js.FuncOf(tokenize))
}

func main() {
	c := make(chan struct{}, 0)
	registerCallbacks()
	println("Kagome Web Assembly Ready")
	<-c
}
<|endoftext|>


<commit_before>package effio

import (
	"code.google.com/p/plotinum/plot"
	"code.google.com/p/plotinum/plotter"
	"code.google.com/p/plotinum/vg"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"runtime"
)

type Group struct {
	Name     string
	Tests    Tests
	Grouping *Grouping
}
type Groups map[string]*Group
type Grouping struct {
	Name      string // group name, e.g. "by_fio", "by_media"
	SuitePath string // root of the suite, e.g. /home/atobey/src/effio/suites/-id/
	OutPath   string // writing final graphs in this directory
	Groups    Groups `json:"-"` // e.g. "samsung_840_read_latency" => [ t1, t2, ... ]
	Suite     *Suite `json:"-"` // parent test suite
}

// suite_path must be a fully-qualitifed path or Chdirs will fail and crash
func (suite *Suite) GraphAll(suite_path string, out_path string) {
	// various groupings/pivots that will be graphed
	by_fio := NewGrouping("by_fio_conf", out_path, suite_path, suite)
	by_dev := NewGrouping("by_device", out_path, suite_path, suite)
	by_mda := NewGrouping("by_media", out_path, suite_path, suite)
	by_tst := NewGrouping("by_test", out_path, suite_path, suite)
	all := []Grouping{by_fio, by_dev, by_mda, by_tst}

	// assign tests to groups
	for _, test := range suite.Tests {
		by_fio.AppendGroup(test.FioConfTmpl.Name, test) // e.g. "read_latency_512" => [ t1, t9, .. ]
		by_dev.AppendGroup(test.Device.Name, test)      // e.g. "fusionio_iodriveii" => [ t3, t7, ...]
		by_mda.AppendGroup(test.Device.Media, test)     // e.g. "MLC" => [t1, t6, ...]
		by_tst.AppendGroup(test.Name, test)             // ends up 1:1 name => [t1]
	}

	for _, gg := range all {
		for _, g := range gg.Groups {
			// generate a latency logfile size graph for every group
			g.barFileSizes()

			// load the CSV on demand
			// at one point this cached loaded tests between runs, but as long
			// as plotinum is taking minutes to generate graphs with lots of data
			// points, the file loading doesn't cost enough to matter
			for _, test := range g.Tests {
				test.LatRecs = LoadCSV(test.LatLogPath(g.Grouping.SuitePath))
				test.LatData = test.LatRecs.Summarize(10000, 10)

				// release the memory used by loading the raw data then force a GC
				// otherwise some of the CSV files easily OOM a 16G machine
				test.LatRecs = nil
				runtime.GC()

				test.LatData.WriteFiles(gg.OutPath, fmt.Sprintf("%s-%s", gg.Name, g.Name))
			}

			// generate output
			g.scatterPlot(true)
			g.scatterPlot(false)
			g.barChart(true)
			g.barChart(false)

			// write metadata for the group/grouping as json
			g.writeJson()
		}
	}
}

func NewGrouping(name string, out_path string, suite_path string, suite *Suite) Grouping {
	mbrs := make(Groups)
	return Grouping{name, suite_path, out_path, mbrs, suite}
}

func (gg *Grouping) AppendGroup(key string, test *Test) {
	if g, ok := gg.Groups[key]; ok {
		g.Tests = append(gg.Groups[key].Tests, test)
	} else {
		gg.Groups[key] = &Group{key, Tests{test}, gg}
	}
}

func (g *Group) barChart(logscale bool) {
	p, err := plot.New()
	if err != nil {
		log.Fatalf("Error creating new plot: %s\n", err)
	}

	// TODO: human names for test groups
	p.Title.Text = fmt.Sprintf("Latency Distribution: %s", g.Name)
	p.X.Label.Text = "Time Offset"
	p.Y.Label.Text = "Latency (usec)"
	p.Add(plotter.NewGrid())
	p.Legend.Top = true
	w := vg.Points(20)

	for i, test := range g.Tests {
		fmt.Printf("Histogram for %s: %v\n", test.Name, test.LatData.Histogram)
		bars, err := plotter.NewBarChart(test.LatData.Histogram, w)
		if err != nil {
			log.Printf("Failed to create new barchart for test %s: %s\n", test.Name, err)
			return
		}
		bars.Color = CustomColors[i]
		p.Add(bars)
		p.Legend.Add(fmt.Sprintf("read: %s ", test.Device.Name), bars)
	}

	if logscale {
		p.Y.Scale = plot.LogScale
		p.Y.Label.Text = "Latency (usec log(10))"

		// defer the savegraph functions so panics can be recovered
		// plotinum will panic on zero values when LogScale is enabled
		// BUG/TODO: somewhere in latency.go histograms are getting
		// entries with values of 0 which should be impossible on latency data
		defer func() {
			if r := recover(); r != nil {
				fmt.Println("Recovered from g.saveGraph()", r)
			}
		}()
		defer g.saveGraph(p, "histogram_bars-logscale")
	} else {
		g.saveGraph(p, "histogram_bars")
	}
}

func (g *Group) scatterPlot(logscale bool) {
	p, err := plot.New()
	if err != nil {
		log.Fatalf("Error creating new plot: %s\n", err)
	}

	// TODO: human names for test groups
	p.Title.Text = fmt.Sprintf("Latency Distribution: %s", g.Name)
	p.X.Label.Text = "Time Offset"
	p.Y.Label.Text = "Latency (usec)"
	p.Add(plotter.NewGrid())
	p.Legend.Top = true

	for i, test := range g.Tests {
		if len(test.LatData.RRecSm) > 0 {
			// reads get circles
			rsp, err := plotter.NewScatter(test.LatData.RRecSm)
			if err != nil {
				log.Printf("Failed to create new scatter plot for test %s: %s\n", test.Name, err)
				return
			}
			rsp.Shape = plot.CircleGlyph{}
			rsp.Radius = vg.Points(3)
			rsp.GlyphStyle.Color = CustomColors[i]
			p.Add(rsp)
			p.Legend.Add(fmt.Sprintf("read: %s ", test.Device.Name), rsp)
		}

		if len(test.LatData.WRecSm) > 0 {
			// writes get pyramids, same color
			wsp, err := plotter.NewScatter(test.LatData.WRecSm)
			if err != nil {
				log.Printf("Failed to create new scatter plot for test %s: %s\n", test.Name, err)
				return
			}
			wsp.Shape = plot.PyramidGlyph{}
			wsp.Radius = vg.Points(3)
			wsp.GlyphStyle.Color = CustomColors[i]
			p.Add(wsp)
			p.Legend.Add(fmt.Sprintf("write: %s ", test.Device.Name), wsp)
		}
	}

	if logscale {
		p.Y.Scale = plot.LogScale
		p.Y.Label.Text = "Latency (usec log(10))"
		g.saveGraph(p, "scatter-logscale")
	} else {
		g.saveGraph(p, "scatter")
	}
}

// draws a bar graph displaying the sizes of the lat_lat.log files across
// all tests
// TODO: figure out how to make the bar width respond to the graph width
func (g *Group) barFileSizes() {
	sizes := make([]int64, len(g.Tests))
	for i, test := range g.Tests {
		fi, err := os.Stat(test.LatLogPath(g.Grouping.SuitePath))
		if err != nil {
			sizes[i] = 0
			continue
		}
		sizes[i] = fi.Size()
	}

	p, err := plot.New()
	if err != nil {
		log.Fatalf("Error creating new plot: %s\n", err)
	}

	p.Title.Text = fmt.Sprintf("Latency Log Sizes: %s", g.Name)
	p.X.Label.Text = "Device + Test"
	p.Y.Label.Text = "Bytes"
	p.Legend.Top = true
	p.Add(plotter.NewGrid())

	// plotinum doesn't offer a way to draw one group of bars
	// with different colors, so each bar is a group with an offset
	var bw float64 = 20.0
	var count float64 = 0
	for i, test := range g.Tests {
		if sizes[i] == 0 {
			continue
		}

		val := plotter.Values{float64(sizes[i])}
		chart, err := plotter.NewBarChart(val, vg.Points(bw))
		if err != nil {
			log.Fatalf("Error adding bar to plot: %s\n", err)
		}

		chart.Color = CustomColors[i]
		chart.Offset = vg.Points(count * bw)

		p.Add(chart)
		p.Legend.Add(test.Name, chart)

		count += 1
	}

	p.X.Min = 0
	p.X.Max = float64(count + 1)

	g.saveGraph(p, "bar-log-size")
}

func (g *Group) writeJson() {
	fname := fmt.Sprintf("group-%s-%s.json", g.Grouping.Name, g.Name)
	outfile := path.Join(g.Grouping.OutPath, fname)

	js, err := json.MarshalIndent(g, "", "  ")
	if err != nil {
		log.Fatalf("Failed to encode group data as JSON: %s\n", err)
	}
	js = append(js, byte('\n'))

	err = ioutil.WriteFile(outfile, js, 0644)
	if err != nil {
		log.Fatalf("Failed to write group JSON data file '%s': %s\n", outfile, err)
	}
}

// e.g. suites/-id/-out/scatter-by_dev-random-read-512b.jpg
func (g *Group) saveGraph(p *plot.Plot, name string) {
	fname := fmt.Sprintf("%s-%s-%s.png", name, g.Grouping.Name, g.Name)
	fpath := path.Join(g.Grouping.OutPath, fname)
	err := p.Save(12, 8, fpath)
	if err != nil {
		log.Fatalf("Failed to save %s: %s\n", fpath, err)
	}
	log.Printf("saved graph: '%s'\n", fpath)
}
<commit_msg>no need to call runtime.GC()<commit_after>package effio

import (
	"code.google.com/p/plotinum/plot"
	"code.google.com/p/plotinum/plotter"
	"code.google.com/p/plotinum/vg"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
)

type Group struct {
	Name     string
	Tests    Tests
	Grouping *Grouping
}
type Groups map[string]*Group
type Grouping struct {
	Name      string // group name, e.g. "by_fio", "by_media"
	SuitePath string // root of the suite, e.g. /home/atobey/src/effio/suites/-id/
	OutPath   string // writing final graphs in this directory
	Groups    Groups `json:"-"` // e.g. "samsung_840_read_latency" => [ t1, t2, ... ]
	Suite     *Suite `json:"-"` // parent test suite
}

// suite_path must be a fully-qualitifed path or Chdirs will fail and crash
func (suite *Suite) GraphAll(suite_path string, out_path string) {
	// various groupings/pivots that will be graphed
	by_fio := NewGrouping("by_fio_conf", out_path, suite_path, suite)
	by_dev := NewGrouping("by_device", out_path, suite_path, suite)
	by_mda := NewGrouping("by_media", out_path, suite_path, suite)
	by_tst := NewGrouping("by_test", out_path, suite_path, suite)
	all := []Grouping{by_fio, by_dev, by_mda, by_tst}

	// assign tests to groups
	for _, test := range suite.Tests {
		by_fio.AppendGroup(test.FioConfTmpl.Name, test) // e.g. "read_latency_512" => [ t1, t9, .. ]
		by_dev.AppendGroup(test.Device.Name, test)      // e.g. "fusionio_iodriveii" => [ t3, t7, ...]
		by_mda.AppendGroup(test.Device.Media, test)     // e.g. "MLC" => [t1, t6, ...]
		by_tst.AppendGroup(test.Name, test)             // ends up 1:1 name => [t1]
	}

	for _, gg := range all {
		for _, g := range gg.Groups {
			// generate a latency logfile size graph for every group
			g.barFileSizes()

			// load the CSV on demand
			// at one point this cached loaded tests between runs, but as long
			// as plotinum is taking minutes to generate graphs with lots of data
			// points, the file loading doesn't cost enough to matter
			for _, test := range g.Tests {
				test.LatRecs = LoadCSV(test.LatLogPath(g.Grouping.SuitePath))
				test.LatData = test.LatRecs.Summarize(10000, 10)

				// release the memory used by loading the raw data
				// otherwise some of the CSV files easily OOM a 16G machine
				test.LatRecs = nil

				test.LatData.WriteFiles(gg.OutPath, fmt.Sprintf("%s-%s", gg.Name, g.Name))
			}

			// generate output
			g.scatterPlot(true)
			g.scatterPlot(false)
			g.barChart(true)
			g.barChart(false)

			// write metadata for the group/grouping as json
			g.writeJson()
		}
	}
}

func NewGrouping(name string, out_path string, suite_path string, suite *Suite) Grouping {
	mbrs := make(Groups)
	return Grouping{name, suite_path, out_path, mbrs, suite}
}

func (gg *Grouping) AppendGroup(key string, test *Test) {
	if g, ok := gg.Groups[key]; ok {
		g.Tests = append(gg.Groups[key].Tests, test)
	} else {
		gg.Groups[key] = &Group{key, Tests{test}, gg}
	}
}

func (g *Group) barChart(logscale bool) {
	p, err := plot.New()
	if err != nil {
		log.Fatalf("Error creating new plot: %s\n", err)
	}

	// TODO: human names for test groups
	p.Title.Text = fmt.Sprintf("Latency Distribution: %s", g.Name)
	p.X.Label.Text = "Time Offset"
	p.Y.Label.Text = "Latency (usec)"
	p.Add(plotter.NewGrid())
	p.Legend.Top = true
	w := vg.Points(20)

	for i, test := range g.Tests {
		fmt.Printf("Histogram for %s: %v\n", test.Name, test.LatData.Histogram)
		bars, err := plotter.NewBarChart(test.LatData.Histogram, w)
		if err != nil {
			log.Printf("Failed to create new barchart for test %s: %s\n", test.Name, err)
			return
		}
		bars.Color = CustomColors[i]
		p.Add(bars)
		p.Legend.Add(fmt.Sprintf("read: %s ", test.Device.Name), bars)
	}

	if logscale {
		p.Y.Scale = plot.LogScale
		p.Y.Label.Text = "Latency (usec log(10))"

		// defer the savegraph functions so panics can be recovered
		// plotinum will panic on zero values when LogScale is enabled
		// BUG/TODO: somewhere in latency.go histograms are getting
		// entries with values of 0 which should be impossible on latency data
		defer func() {
			if r := recover(); r != nil {
				fmt.Println("Recovered from g.saveGraph()", r)
			}
		}()
		defer g.saveGraph(p, "histogram_bars-logscale")
	} else {
		g.saveGraph(p, "histogram_bars")
	}
}

func (g *Group) scatterPlot(logscale bool) {
	p, err := plot.New()
	if err != nil {
		log.Fatalf("Error creating new plot: %s\n", err)
	}

	// TODO: human names for test groups
	p.Title.Text = fmt.Sprintf("Latency Distribution: %s", g.Name)
	p.X.Label.Text = "Time Offset"
	p.Y.Label.Text = "Latency (usec)"
	p.Add(plotter.NewGrid())
	p.Legend.Top = true

	for i, test := range g.Tests {
		if len(test.LatData.RRecSm) > 0 {
			// reads get circles
			rsp, err := plotter.NewScatter(test.LatData.RRecSm)
			if err != nil {
				log.Printf("Failed to create new scatter plot for test %s: %s\n", test.Name, err)
				return
			}
			rsp.Shape = plot.CircleGlyph{}
			rsp.Radius = vg.Points(3)
			rsp.GlyphStyle.Color = CustomColors[i]
			p.Add(rsp)
			p.Legend.Add(fmt.Sprintf("read: %s ", test.Device.Name), rsp)
		}

		if len(test.LatData.WRecSm) > 0 {
			// writes get pyramids, same color
			wsp, err := plotter.NewScatter(test.LatData.WRecSm)
			if err != nil {
				log.Printf("Failed to create new scatter plot for test %s: %s\n", test.Name, err)
				return
			}
			wsp.Shape = plot.PyramidGlyph{}
			wsp.Radius = vg.Points(3)
			wsp.GlyphStyle.Color = CustomColors[i]
			p.Add(wsp)
			p.Legend.Add(fmt.Sprintf("write: %s ", test.Device.Name), wsp)
		}
	}

	if logscale {
		p.Y.Scale = plot.LogScale
		p.Y.Label.Text = "Latency (usec log(10))"
		g.saveGraph(p, "scatter-logscale")
	} else {
		g.saveGraph(p, "scatter")
	}
}

// draws a bar graph displaying the sizes of the lat_lat.log files across
// all tests
// TODO: figure out how to make the bar width respond to the graph width
func (g *Group) barFileSizes() {
	sizes := make([]int64, len(g.Tests))
	for i, test := range g.Tests {
		fi, err := os.Stat(test.LatLogPath(g.Grouping.SuitePath))
		if err != nil {
			sizes[i] = 0
			continue
		}
		sizes[i] = fi.Size()
	}

	p, err := plot.New()
	if err != nil {
		log.Fatalf("Error creating new plot: %s\n", err)
	}

	p.Title.Text = fmt.Sprintf("Latency Log Sizes: %s", g.Name)
	p.X.Label.Text = "Device + Test"
	p.Y.Label.Text = "Bytes"
	p.Legend.Top = true
	p.Add(plotter.NewGrid())

	// plotinum doesn't offer a way to draw one group of bars
	// with different colors, so each bar is a group with an offset
	var bw float64 = 20.0
	var count float64 = 0
	for i, test := range g.Tests {
		if sizes[i] == 0 {
			continue
		}

		val := plotter.Values{float64(sizes[i])}
		chart, err := plotter.NewBarChart(val, vg.Points(bw))
		if err != nil {
			log.Fatalf("Error adding bar to plot: %s\n", err)
		}

		chart.Color = CustomColors[i]
		chart.Offset = vg.Points(count * bw)

		p.Add(chart)
		p.Legend.Add(test.Name, chart)

		count += 1
	}

	p.X.Min = 0
	p.X.Max = float64(count + 1)

	g.saveGraph(p, "bar-log-size")
}

func (g *Group) writeJson() {
	fname := fmt.Sprintf("group-%s-%s.json", g.Grouping.Name, g.Name)
	outfile := path.Join(g.Grouping.OutPath, fname)

	js, err := json.MarshalIndent(g, "", "  ")
	if err != nil {
		log.Fatalf("Failed to encode group data as JSON: %s\n", err)
	}
	js = append(js, byte('\n'))

	err = ioutil.WriteFile(outfile, js, 0644)
	if err != nil {
		log.Fatalf("Failed to write group JSON data file '%s': %s\n", outfile, err)
	}
}

// e.g. suites/-id/-out/scatter-by_dev-random-read-512b.jpg
func (g *Group) saveGraph(p *plot.Plot, name string) {
	fname := fmt.Sprintf("%s-%s-%s.png", name, g.Grouping.Name, g.Name)
	fpath := path.Join(g.Grouping.OutPath, fname)
	err := p.Save(12, 8, fpath)
	if err != nil {
		log.Fatalf("Failed to save %s: %s\n", fpath, err)
	}
	log.Printf("saved graph: '%s'\n", fpath)
}
<|endoftext|>


<commit_before><commit_msg>update with gse example code<commit_after><|endoftext|>


<commit_before>package vindinium

// State represents the current state of a game of Vindinium.
type State struct {
	Game    Game
	Hero    Hero
	Token   string
	ViewUrl string
	PlayUrl string
}
<commit_msg>State: Make PlayURL and ViewURL uppercase to adhere to Go conventions<commit_after>package vindinium

// State represents the current state of a game of Vindinium.
type State struct {
	Game    Game
	Hero    Hero
	Token   string
	ViewURL string
	PlayURL string
}
<|endoftext|>


<commit_before>package virtualbox

import (
	"bufio"
	"net"
	"strings"
)

// DHCP server info.
type DHCP struct {
	NetworkName string
	IPv4        net.IPNet
	LowerIP     net.IP
	UpperIP     net.IP
	Enabled     bool
}

func addDHCP(kind, name string, d DHCP) error {
	args := []string{"dhcpserver", "add",
		kind, name,
		"--ip", d.IPv4.IP.String(),
		"--netmask", net.IP(d.IPv4.Mask).String(),
		"--lowerip", d.LowerIP.String(),
		"--upperip", d.UpperIP.String(),
	}
	if d.Enabled {
		args = append(args, "--enable")
	} else {
		args = append(args, "--disable")
	}
	return vbm(args...)
}

// AddInternalDHCP adds a DHCP server to an internal network.
func AddInternalDHCP(netname string, d DHCP) error {
	return addDHCP("--netname", netname, d)
}

// AddHostonlyDHCP adds a DHCP server to a host-only network.
func AddHostonlyDHCP(ifname string, d DHCP) error {
	return addDHCP("--ifname", ifname, d)
}

// DHCPs gets all DHCP server settings in a map keyed by DHCP.NetworkName.
func DHCPs() (map[string]*DHCP, error) {
	out, err := vbmOut("list", "dhcpservers")
	if err != nil {
		return nil, err
	}
	s := bufio.NewScanner(strings.NewReader(out))
	m := map[string]*DHCP{}
	dhcp := &DHCP{}
	for s.Scan() {
		line := s.Text()
		if line == "" {
			m[dhcp.NetworkName] = dhcp
			dhcp = &DHCP{}
			continue
		}
		res := reColonLine.FindStringSubmatch(line)
		if res == nil {
			continue
		}
		switch key, val := res[1], res[2]; key {
		case "NetworkName":
			dhcp.NetworkName = val
		case "IP":
			dhcp.IPv4.IP = net.ParseIP(val)
		case "upperIPAddress":
			dhcp.UpperIP = net.ParseIP(val)
		case "lowerIPAddress":
			dhcp.LowerIP = net.ParseIP(val)
		case "NetworkMask":
			dhcp.IPv4.Mask = ParseIPv4Mask(val)
		case "Enabled":
			dhcp.Enabled = (val == "Yes")
		}
	}
	if err := s.Err(); err != nil {
		return nil, err
	}
	return m, nil
}
<commit_msg>check to see if the hostonlyif created the dhcp server and if it did, modify<commit_after>package virtualbox

import (
	"bufio"
	"net"
	"strings"
)

// DHCP server info.
type DHCP struct {
	NetworkName string
	IPv4        net.IPNet
	LowerIP     net.IP
	UpperIP     net.IP
	Enabled     bool
}

func addDHCP(kind, name string, d DHCP) error {
	command := "modify"

	// On some platforms (OSX), creating a hostonlyinterface adds a default dhcpserver
	// While on others (Windows?) it does not.
	dhcps, err := DHCPs()
	if err != nil {
		return err
	}

	if _, ok := dhcps[name]; !ok {
		command = "add"
	}

	args := []string{"dhcpserver", command,
		kind, name,
		"--ip", d.IPv4.IP.String(),
		"--netmask", net.IP(d.IPv4.Mask).String(),
		"--lowerip", d.LowerIP.String(),
		"--upperip", d.UpperIP.String(),
	}
	if d.Enabled {
		args = append(args, "--enable")
	} else {
		args = append(args, "--disable")
	}
	return vbm(args...)
}

// AddInternalDHCP adds a DHCP server to an internal network.
func AddInternalDHCP(netname string, d DHCP) error {
	return addDHCP("--netname", netname, d)
}

// AddHostonlyDHCP adds a DHCP server to a host-only network.
func AddHostonlyDHCP(ifname string, d DHCP) error {
	return addDHCP("--netname", "HostInterfaceNetworking-"+ifname, d)
}

// DHCPs gets all DHCP server settings in a map keyed by DHCP.NetworkName.
func DHCPs() (map[string]*DHCP, error) {
	out, err := vbmOut("list", "dhcpservers")
	if err != nil {
		return nil, err
	}
	s := bufio.NewScanner(strings.NewReader(out))
	m := map[string]*DHCP{}
	dhcp := &DHCP{}
	for s.Scan() {
		line := s.Text()
		if line == "" {
			m[dhcp.NetworkName] = dhcp
			dhcp = &DHCP{}
			continue
		}
		res := reColonLine.FindStringSubmatch(line)
		if res == nil {
			continue
		}
		switch key, val := res[1], res[2]; key {
		case "NetworkName":
			dhcp.NetworkName = val
		case "IP":
			dhcp.IPv4.IP = net.ParseIP(val)
		case "upperIPAddress":
			dhcp.UpperIP = net.ParseIP(val)
		case "lowerIPAddress":
			dhcp.LowerIP = net.ParseIP(val)
		case "NetworkMask":
			dhcp.IPv4.Mask = ParseIPv4Mask(val)
		case "Enabled":
			dhcp.Enabled = (val == "Yes")
		}
	}
	if err := s.Err(); err != nil {
		return nil, err
	}
	return m, nil
}
<|endoftext|>


<commit_before><commit_msg>Run go fmt<commit_after><|endoftext|>