<commit_before>// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"strconv"
	"strings"
	"time"

	"os"

	"github.com/Sirupsen/logrus"
	"github.com/intelsdi-x/swan/experiments/memcached-sensitivity-profile/common"
	"github.com/intelsdi-x/swan/pkg/conf"
	"github.com/intelsdi-x/swan/pkg/executor"
	"github.com/intelsdi-x/swan/pkg/experiment"
	"github.com/intelsdi-x/swan/pkg/experiment/logger"
	"github.com/intelsdi-x/swan/pkg/experiment/sensitivity"
	"github.com/intelsdi-x/swan/pkg/experiment/sensitivity/validate"
	"github.com/intelsdi-x/swan/pkg/isolation"
	"github.com/intelsdi-x/swan/pkg/isolation/topo"
	"github.com/intelsdi-x/swan/pkg/snap"
	"github.com/intelsdi-x/swan/pkg/snap/sessions/mutilate"
	"github.com/intelsdi-x/swan/pkg/snap/sessions/use"
	"github.com/intelsdi-x/swan/pkg/utils/errutil"
	_ "github.com/intelsdi-x/swan/pkg/utils/unshare"
	"github.com/intelsdi-x/swan/pkg/utils/uuid"
	"github.com/intelsdi-x/swan/pkg/workloads/memcached"
)

var (
	appName             = os.Args[0]
	useCorePinningFlag  = conf.NewBoolFlag("use_core_pinning", "Enables core pinning of memcached threads", false)
	maxThreadsFlag      = conf.NewIntFlag("max_threads", "Scale memcached up to cores (default to number of physical cores).", 0)
	useUSECollectorFlag = conf.NewBoolFlag("use_USE_collector", "Collects USE (Utilization, Saturation, Errors) metrics.", false)
)

func main() {
	experimentStart := time.Now()

	// Preparing application - setting name, help, parsing flags etc.
	experiment.Configure()

	// Generate an experiment ID and start the metadata session.
	uid := uuid.New()

	// Initialize logger.
	logger.Initialize(appName, uid)

	// connect to metadata database
	metadata, err := experiment.NewMetadata(uid, experiment.MetadataConfigFromFlags())
	errutil.CheckWithContext(err, "Cannot connect to metadata database")

	// Save experiment runtime environment (configuration, environmental variables, etc).
	err = metadata.RecordRuntimeEnv(experimentStart)
	errutil.CheckWithContext(err, "Cannot save runtime environment")

	// Read configuration.
	loadDuration := sensitivity.LoadDurationFlag.Value()
	loadPoints := sensitivity.LoadPointsCountFlag.Value()
	useCorePinning := useCorePinningFlag.Value()
	peakLoad := sensitivity.PeakLoadFlag.Value()
	if peakLoad == 0 {
		logrus.Fatalf("peak load have to be != 0!")
	}

	// Record metadata.
	records := map[string]string{
		"command_arguments": strings.Join(os.Args, ","),
		"experiment_name":   appName,
		"repetitions":       "1",
		"load_duration":     loadDuration.String(),
		"load_points":       strconv.Itoa(loadPoints),
		"use_core_pinning":  strconv.FormatBool(useCorePinning),
		"peak_load":         strconv.Itoa(peakLoad),
	}
	err = metadata.RecordMap(records)
	errutil.CheckWithContext(err, "Cannot save metadata")

	// Validate preconditions.
	validate.OS()

	// Discover CPU topology.
	topology, err := topo.Discover()
	errutil.CheckWithContext(err, "Cannot discover CPU topology")
	physicalCores := topology.AvailableCores()
	allSoftwareThreds := topology.AvailableThreads()

	maxThreads := maxThreadsFlag.Value()
	if maxThreads == 0 {
		maxThreads = len(physicalCores)
	}

	// Launch Kubernetes cluster if necessary.
	var cleanup func() error
	if sensitivity.RunOnKubernetesFlag.Value() && !sensitivity.RunOnExistingKubernetesFlag.Value() {
		cleanup, err = sensitivity.LaunchKubernetesCluster()
		errutil.CheckWithContext(err, "Cannot launch Kubernetes cluster")
		defer cleanup()
	}

	// Create mutilate snap session launcher.
	mutilateSnapSession, err := mutilatesession.NewSessionLauncherDefault()
	errutil.CheckWithContext(err, "Cannot create Mutilate snap session")

	// Create USE Collector session launcher.
	useUSECollector := useUSECollectorFlag.Value()
	var useSession snap.SessionLauncher
	if useUSECollector {
		useSession, err = use.NewSessionLauncherDefault()
		errutil.CheckWithContext(err, "Cannot create USE snap session")
	}

	// Calculate value to increase QPS by on every iteration.
	qpsDelta := int(peakLoad / loadPoints)
	logrus.Debugf("Increasing QPS by %d every iteration up to peak load %d to achieve %d load points", qpsDelta, peakLoad, loadPoints)

	// Iterate over all physical cores available.
	for numberOfThreads := 1; numberOfThreads <= maxThreads; numberOfThreads++ {
		// Iterate over load points that user requested.
		for qps := qpsDelta; qps <= peakLoad; qps += qpsDelta {
			func() {
				logrus.Infof("Running %d threads of memcached with load of %d QPS", numberOfThreads, qps)

				// Check if core pinning should be enabled and set phase name.
				var isolators isolation.Decorators
				phaseName := fmt.Sprintf("memcached -t %d", numberOfThreads)
				if useCorePinning {
					var threads isolation.IntSet
					if numberOfThreads > len(physicalCores) {
						threads, err = allSoftwareThreds.Take(numberOfThreads)
						errutil.PanicWithContext(err, "Cannot take %d software threads for memcached")
					} else {
						// We have enough physcial threads - take them.
						threads, err = physicalCores.Take(numberOfThreads)
						errutil.PanicWithContext(err, "Cannot take %d hardware threads (cores) for memcached")
					}
					logrus.Infof("Threads pinning enabled, using threads %q", threads.AsRangeString())
					isolators = append(isolators, isolation.Taskset{CPUList: threads})
					phaseName = isolators.Decorate(phaseName)
				}
				logrus.Debugf("Running phase: %q", phaseName)

				// Create directory where output of all the tasks will be stored.
				err := experiment.CreateRepetitionDir(appName, uid, phaseName, 0)
				errutil.PanicWithContext(err, "Cannot create repetition directory")

				// Create memcached executor.
				var memcachedExecutor executor.Executor
				if sensitivity.RunOnKubernetesFlag.Value() {
					memcachedExecutor, err = sensitivity.CreateKubernetesHpExecutor(isolators)
					errutil.PanicWithContext(err, "Cannot create Kubernetes executor")
				} else {
					memcachedExecutor = executor.NewLocalIsolated(isolators)
				}

				// Create memcached launcher and start memcached
				memcachedConfiguration := memcached.DefaultMemcachedConfig()
				memcachedConfiguration.NumThreads = numberOfThreads
				memcachedLauncher := executor.ServiceLauncher{Launcher: memcached.New(memcachedExecutor, memcachedConfiguration)}
				memcachedTask, err := memcachedLauncher.Launch()
				errutil.PanicWithContext(err, "Memcached has not been launched successfully")
				defer memcachedTask.Stop()

				// Create mutilate load generator.
				loadGenerator, err := common.PrepareMutilateGenerator(memcachedConfiguration.IP, memcachedConfiguration.Port)
				errutil.PanicWithContext(err, "Cannot create mutilate load generator")

				// Populate memcached.
				err = loadGenerator.Populate()
				errutil.PanicWithContext(err, "Memcached cannot be populated")

				// Create tags to be used on Snap metrics.
				phase := strings.Replace(phaseName, ",", "'", -1)
				aggressor := "No aggressor " + strings.Replace(phaseName, ",", "'", -1)

				snapTags := make(map[string]interface{})
				snapTags[experiment.ExperimentKey] = uid
				snapTags[experiment.PhaseKey] = phase
				snapTags[experiment.RepetitionKey] = 0
				snapTags[experiment.LoadPointQPSKey] = qps
				snapTags[experiment.AggressorNameKey] = aggressor
				snapTags["number_of_cores"] = numberOfThreads // For backward compatibility.
				snapTags["number_of_threads"] = numberOfThreads

				var useSessionHandle snap.SessionHandle
				// Start USE Collection.
				if useUSECollector {
					useSessionHandle, err := useSession.LaunchSession(nil, snapTags)
					errutil.PanicWithContext(err, "Cannot launch Snap USE Collection session")
					defer useSessionHandle.Stop()
				}

				// Start sending traffic from mutilate cluster to memcached.
				mutilateHandle, err := loadGenerator.Load(qps, loadDuration)
				errutil.PanicWithContext(err, "Cannot start load generator")
				mutilateClusterMaxExecution := sensitivity.LoadGeneratorWaitTimeoutFlag.Value()
				if !mutilateHandle.Wait(mutilateClusterMaxExecution) {
					msg := fmt.Sprintf("Mutilate cluster failed to stop on its own in %s. Attempting to stop...", mutilateClusterMaxExecution)
					err := mutilateHandle.Stop()
					errutil.PanicWithContext(err, msg+" Stopping mutilate cluster errored")
					logrus.Panic(msg)
				}

				// Make sure that mutilate exited with 0 status.
				exitCode, _ := mutilateHandle.ExitCode()
				if exitCode != 0 {
					logrus.Panicf("Mutilate cluster has not stopped properly. Exit status: %d.", exitCode)
				}

				if useUSECollector {
					err = useSessionHandle.Stop()
					errutil.PanicWithContext(err, "Cannot stop Snap USE Collector session")
				}

				// Launch and stop Snap task to collect mutilate metrics.
				mutilateSnapSessionHandle, err := mutilateSnapSession.LaunchSession(mutilateHandle, snapTags)
				errutil.PanicWithContext(err, "Snap mutilate session has not been started successfully")
				defer func() {
					err = mutilateSnapSessionHandle.Stop()
					if err != nil {
						logrus.Errorf("Cannot stop mutilate session: %v", err)
					}
				}()
				err = mutilateSnapSessionHandle.Wait()
				errutil.PanicWithContext(err, "Snap mutilate session has not collected metrics!")

				// It is ugly but there is no other way to make sure that data is written to Cassandra as of now.
				time.Sleep(10 * time.Second)
			}()
		}
	}
}
<commit_msg>Nil pointer exception fix (#641)<commit_after>// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"strconv"
	"strings"
	"time"

	"os"

	"github.com/Sirupsen/logrus"
	"github.com/intelsdi-x/swan/experiments/memcached-sensitivity-profile/common"
	"github.com/intelsdi-x/swan/pkg/conf"
	"github.com/intelsdi-x/swan/pkg/executor"
	"github.com/intelsdi-x/swan/pkg/experiment"
	"github.com/intelsdi-x/swan/pkg/experiment/logger"
	"github.com/intelsdi-x/swan/pkg/experiment/sensitivity"
	"github.com/intelsdi-x/swan/pkg/experiment/sensitivity/validate"
	"github.com/intelsdi-x/swan/pkg/isolation"
	"github.com/intelsdi-x/swan/pkg/isolation/topo"
	"github.com/intelsdi-x/swan/pkg/snap"
	"github.com/intelsdi-x/swan/pkg/snap/sessions/mutilate"
	"github.com/intelsdi-x/swan/pkg/snap/sessions/use"
	"github.com/intelsdi-x/swan/pkg/utils/errutil"
	_ "github.com/intelsdi-x/swan/pkg/utils/unshare"
	"github.com/intelsdi-x/swan/pkg/utils/uuid"
	"github.com/intelsdi-x/swan/pkg/workloads/memcached"
)

var (
	appName             = os.Args[0]
	useCorePinningFlag  = conf.NewBoolFlag("use_core_pinning", "Enables core pinning of memcached threads", false)
	maxThreadsFlag      = conf.NewIntFlag("max_threads", "Scale memcached up to cores (default to number of physical cores).", 0)
	useUSECollectorFlag = conf.NewBoolFlag("use_USE_collector", "Collects USE (Utilization, Saturation, Errors) metrics.", false)
)

func main() {
	experimentStart := time.Now()

	// Preparing application - setting name, help, parsing flags etc.
	experiment.Configure()

	// Generate an experiment ID and start the metadata session.
	uid := uuid.New()

	// Initialize logger.
	logger.Initialize(appName, uid)

	// connect to metadata database
	metadata, err := experiment.NewMetadata(uid, experiment.MetadataConfigFromFlags())
	errutil.CheckWithContext(err, "Cannot connect to metadata database")

	// Save experiment runtime environment (configuration, environmental variables, etc).
	err = metadata.RecordRuntimeEnv(experimentStart)
	errutil.CheckWithContext(err, "Cannot save runtime environment")

	// Read configuration.
	loadDuration := sensitivity.LoadDurationFlag.Value()
	loadPoints := sensitivity.LoadPointsCountFlag.Value()
	useCorePinning := useCorePinningFlag.Value()
	peakLoad := sensitivity.PeakLoadFlag.Value()
	if peakLoad == 0 {
		logrus.Fatalf("peak load have to be != 0!")
	}

	// Record metadata.
	records := map[string]string{
		"command_arguments": strings.Join(os.Args, ","),
		"experiment_name":   appName,
		"repetitions":       "1",
		"load_duration":     loadDuration.String(),
		"load_points":       strconv.Itoa(loadPoints),
		"use_core_pinning":  strconv.FormatBool(useCorePinning),
		"peak_load":         strconv.Itoa(peakLoad),
	}
	err = metadata.RecordMap(records)
	errutil.CheckWithContext(err, "Cannot save metadata")

	// Validate preconditions.
	validate.OS()

	// Discover CPU topology.
	topology, err := topo.Discover()
	errutil.CheckWithContext(err, "Cannot discover CPU topology")
	physicalCores := topology.AvailableCores()
	allSoftwareThreds := topology.AvailableThreads()

	maxThreads := maxThreadsFlag.Value()
	if maxThreads == 0 {
		maxThreads = len(physicalCores)
	}

	// Launch Kubernetes cluster if necessary.
	var cleanup func() error
	if sensitivity.RunOnKubernetesFlag.Value() && !sensitivity.RunOnExistingKubernetesFlag.Value() {
		cleanup, err = sensitivity.LaunchKubernetesCluster()
		errutil.CheckWithContext(err, "Cannot launch Kubernetes cluster")
		defer cleanup()
	}

	// Create mutilate snap session launcher.
	mutilateSnapSession, err := mutilatesession.NewSessionLauncherDefault()
	errutil.CheckWithContext(err, "Cannot create Mutilate snap session")

	// Create USE Collector session launcher.
	useUSECollector := useUSECollectorFlag.Value()
	var useSession snap.SessionLauncher
	if useUSECollector {
		useSession, err = use.NewSessionLauncherDefault()
		errutil.CheckWithContext(err, "Cannot create USE snap session")
	}

	// Calculate value to increase QPS by on every iteration.
	qpsDelta := int(peakLoad / loadPoints)
	logrus.Debugf("Increasing QPS by %d every iteration up to peak load %d to achieve %d load points", qpsDelta, peakLoad, loadPoints)

	// Iterate over all physical cores available.
	for numberOfThreads := 1; numberOfThreads <= maxThreads; numberOfThreads++ {
		// Iterate over load points that user requested.
		for qps := qpsDelta; qps <= peakLoad; qps += qpsDelta {
			func() {
				logrus.Infof("Running %d threads of memcached with load of %d QPS", numberOfThreads, qps)

				// Check if core pinning should be enabled and set phase name.
				var isolators isolation.Decorators
				phaseName := fmt.Sprintf("memcached -t %d", numberOfThreads)
				if useCorePinning {
					var threads isolation.IntSet
					if numberOfThreads > len(physicalCores) {
						threads, err = allSoftwareThreds.Take(numberOfThreads)
						errutil.PanicWithContext(err, "Cannot take %d software threads for memcached")
					} else {
						// We have enough physcial threads - take them.
						threads, err = physicalCores.Take(numberOfThreads)
						errutil.PanicWithContext(err, "Cannot take %d hardware threads (cores) for memcached")
					}
					logrus.Infof("Threads pinning enabled, using threads %q", threads.AsRangeString())
					isolators = append(isolators, isolation.Taskset{CPUList: threads})
					phaseName = isolators.Decorate(phaseName)
				}
				logrus.Debugf("Running phase: %q", phaseName)

				// Create directory where output of all the tasks will be stored.
				err := experiment.CreateRepetitionDir(appName, uid, phaseName, 0)
				errutil.PanicWithContext(err, "Cannot create repetition directory")

				// Create memcached executor.
				var memcachedExecutor executor.Executor
				if sensitivity.RunOnKubernetesFlag.Value() {
					memcachedExecutor, err = sensitivity.CreateKubernetesHpExecutor(isolators)
					errutil.PanicWithContext(err, "Cannot create Kubernetes executor")
				} else {
					memcachedExecutor = executor.NewLocalIsolated(isolators)
				}

				// Create memcached launcher and start memcached
				memcachedConfiguration := memcached.DefaultMemcachedConfig()
				memcachedConfiguration.NumThreads = numberOfThreads
				memcachedLauncher := executor.ServiceLauncher{Launcher: memcached.New(memcachedExecutor, memcachedConfiguration)}
				memcachedTask, err := memcachedLauncher.Launch()
				errutil.PanicWithContext(err, "Memcached has not been launched successfully")
				defer memcachedTask.Stop()

				// Create mutilate load generator.
				loadGenerator, err := common.PrepareMutilateGenerator(memcachedConfiguration.IP, memcachedConfiguration.Port)
				errutil.PanicWithContext(err, "Cannot create mutilate load generator")

				// Populate memcached.
				err = loadGenerator.Populate()
				errutil.PanicWithContext(err, "Memcached cannot be populated")

				// Create tags to be used on Snap metrics.
				phase := strings.Replace(phaseName, ",", "'", -1)
				aggressor := "No aggressor " + strings.Replace(phaseName, ",", "'", -1)

				snapTags := make(map[string]interface{})
				snapTags[experiment.ExperimentKey] = uid
				snapTags[experiment.PhaseKey] = phase
				snapTags[experiment.RepetitionKey] = 0
				snapTags[experiment.LoadPointQPSKey] = qps
				snapTags[experiment.AggressorNameKey] = aggressor
				snapTags["number_of_cores"] = numberOfThreads // For backward compatibility.
				snapTags["number_of_threads"] = numberOfThreads

				var useSessionHandle snap.SessionHandle
				// Start USE Collection.
				if useUSECollector {
					useSessionHandle, err = useSession.LaunchSession(nil, snapTags)
					errutil.PanicWithContext(err, "Cannot launch Snap USE Collection session")
					defer useSessionHandle.Stop()
				}

				// Start sending traffic from mutilate cluster to memcached.
				mutilateHandle, err := loadGenerator.Load(qps, loadDuration)
				errutil.PanicWithContext(err, "Cannot start load generator")
				mutilateClusterMaxExecution := sensitivity.LoadGeneratorWaitTimeoutFlag.Value()
				if !mutilateHandle.Wait(mutilateClusterMaxExecution) {
					msg := fmt.Sprintf("Mutilate cluster failed to stop on its own in %s. Attempting to stop...", mutilateClusterMaxExecution)
					err := mutilateHandle.Stop()
					errutil.PanicWithContext(err, msg+" Stopping mutilate cluster errored")
					logrus.Panic(msg)
				}

				// Make sure that mutilate exited with 0 status.
				exitCode, _ := mutilateHandle.ExitCode()
				if exitCode != 0 {
					logrus.Panicf("Mutilate cluster has not stopped properly. Exit status: %d.", exitCode)
				}

				if useUSECollector {
					err = useSessionHandle.Stop()
					errutil.PanicWithContext(err, "Cannot stop Snap USE Collector session")
				}

				// Launch and stop Snap task to collect mutilate metrics.
				mutilateSnapSessionHandle, err := mutilateSnapSession.LaunchSession(mutilateHandle, snapTags)
				errutil.PanicWithContext(err, "Snap mutilate session has not been started successfully")
				defer func() {
					err = mutilateSnapSessionHandle.Stop()
					if err != nil {
						logrus.Errorf("Cannot stop mutilate session: %v", err)
					}
				}()
				err = mutilateSnapSessionHandle.Wait()
				errutil.PanicWithContext(err, "Snap mutilate session has not collected metrics!")

				// It is ugly but there is no other way to make sure that data is written to Cassandra as of now.
				time.Sleep(10 * time.Second)
			}()
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2017, OpenCensus Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Command stackdriver is an example program that collects data for
// video size. Collected data is exported to
// Stackdriver Monitoring.
package main

import (
	"context"
	"fmt"
	"log"
	"time"

	"go.opencensus.io/exporter/stackdriver"
	"go.opencensus.io/stats"
	"go.opencensus.io/stats/view"
)

// Create measures. The program will record measures for the size of
// processed videos and the nubmer of videos marked as spam.
var videoSize = stats.Int64("example.com/measure/video_size", "size of processed videos", stats.UnitBytes)

func main() {
	ctx := context.Background()

	// Collected view data will be reported to Stackdriver Monitoring API
	// via the Stackdriver exporter.
	//
	// In order to use the Stackdriver exporter, enable Stackdriver Monitoring API
	// at https://console.cloud.google.com/apis/dashboard.
	//
	// Once API is enabled, you can use Google Application Default Credentials
	// to setup the authorization.
	// See https://developers.google.com/identity/protocols/application-default-credentials
	// for more details.
	exporter, err := stackdriver.NewExporter(stackdriver.Options{
		ProjectID: "project-id", // Google Cloud Console project ID.
	})
	if err != nil {
		log.Fatal(err)
	}
	view.RegisterExporter(exporter)

	// Set reporting period to report data at every second.
	view.SetReportingPeriod(1 * time.Second)

	// Create view to see the processed video size cumulatively.
	// Subscribe will allow view data to be exported.
	// Once no longer need, you can unsubscribe from the view.
	if err := view.Register(&view.View{
		Name:        "example.com/views/video_size_cum",
		Description: "processed video size over time",
		Measure:     videoSize,
		Aggregation: view.Distribution(0, 1<<16, 1<<32),
	}); err != nil {
		log.Fatalf("Cannot register the view: %v", err)
	}

	processVideo(ctx)

	// Wait for a duration longer than reporting duration to ensure the stats
	// library reports the collected data.
	fmt.Println("Wait longer than the reporting duration...")
	time.Sleep(1 * time.Minute)
}

func processVideo(ctx context.Context) {
	// Do some processing and record stats.
	stats.Record(ctx, videoSize.M(25648))
}
<commit_msg>Remove the Stackdriver example (#818)<commit_after><|endoftext|>


<commit_before>package param

const (
	// If set to true, Pod terminations are only logged and pods are
	// not actually killed.
	// Type: bool
	// Default: true
	DryRun = "kubemonkey.dry_run"

	// The timezone to use when scheduling Pod terminations
	// Type: string
	// Default: America/Los_Angeles
	Timezone = "kubemonkey.time_zone"

	// The hour of the weekday when the scheduler should run
	// to schedule terminations
	// Must be less than StartHour, and [0,23]
	// Type: int
	// Default: 8
	RunHour = "kubemonkey.run_hour"

	// The hour beginning at which pod terminations may occur
	// Should be set to a time when service owners are expected
	// to be available
	// Must be less than EndHour, and [0, 23]
	// Type: int
	// Default: 10
	StartHour = "kubemonkey.start_hour"

	// The end hour beyond which no pod terminations will occur
	// Should be set to a time when service owners are expected
	// to be available
	// Must be [0,23]
	// Type: int
	// Default: 16
	EndHour = "kubemonkey.end_hour"

	// The amount of time in seconds a pod is given
	// to shut down gracefully, before Kubernetes does
	// a hard kill
	// Type: int
	// Default: 5
	GracePeriodSec = "kubemonkey.graceperiod_sec"

	// A list of namespaces for which terminations should never
	// be carried out.
	// Type: list
	// Default: [ "kube-system" ]
	BlacklistedNamespaces = "kubemonkey.blacklisted_namespaces"

	// Host URL for Kubernetes cluster APIServer
	// Type: string
	// Default: No default. If not specificed, uses what URL is specified
	// by in-cluster config
	ClusterAPIServerHost = "kubernetes.host"

	// Set to true to enable debug mode
	// Type: bool
	// Default: false
	DebugEnabled = "debug.enabled"

	// Delay duration in sec after kube-monkey is launched
	// after which scheduling is run
	// Use when debugging to run scheduling sooner
	// Type: int
	// Default: 30
	DebugScheduleDelay = "debug.schedule_delay"

	// If set to true, terminations will be guaranteed
	// to be scheduled for all eligible Deployments,
	// i.e., probability of kill = 1
	// Type: bool
	// Default: false
	DebugForceShouldKill = "debug.force_should_kill"

	// If set to true, pod terminations will be scheduled
	// sometime in the next 60 sec to facilitate
	// debugging (instead of the hours specified by
	// StartHour and EndHour)
	// Type: bool
	// Default: false
	DebugScheduleImmediateKill = "debug.schedule_immediate_kill"
)
<commit_msg>Add more detail to api server config param<commit_after>package param

const (
	// If set to true, Pod terminations are only logged and pods are
	// not actually killed.
	// Type: bool
	// Default: true
	DryRun = "kubemonkey.dry_run"

	// The timezone to use when scheduling Pod terminations
	// Type: string
	// Default: America/Los_Angeles
	Timezone = "kubemonkey.time_zone"

	// The hour of the weekday when the scheduler should run
	// to schedule terminations
	// Must be less than StartHour, and [0,23]
	// Type: int
	// Default: 8
	RunHour = "kubemonkey.run_hour"

	// The hour beginning at which pod terminations may occur
	// Should be set to a time when service owners are expected
	// to be available
	// Must be less than EndHour, and [0, 23]
	// Type: int
	// Default: 10
	StartHour = "kubemonkey.start_hour"

	// The end hour beyond which no pod terminations will occur
	// Should be set to a time when service owners are expected
	// to be available
	// Must be [0,23]
	// Type: int
	// Default: 16
	EndHour = "kubemonkey.end_hour"

	// The amount of time in seconds a pod is given
	// to shut down gracefully, before Kubernetes does
	// a hard kill
	// Type: int
	// Default: 5
	GracePeriodSec = "kubemonkey.graceperiod_sec"

	// A list of namespaces for which terminations should never
	// be carried out.
	// Type: list
	// Default: [ "kube-system" ]
	BlacklistedNamespaces = "kubemonkey.blacklisted_namespaces"

	// Host URL for Kubernetes cluster APIServer. Use this config
	// if the apiserver IP address provided by in-cluster config
	// does not work for you because your certificate does not
	// conatain the right SAN
	// Type: string
	// Default: No default. If not specified, URL provided
	// by in-cluster config is used
	ClusterAPIServerHost = "kubernetes.host"

	// Set to true to enable debug mode
	// Type: bool
	// Default: false
	DebugEnabled = "debug.enabled"

	// Delay duration in sec after kube-monkey is launched
	// after which scheduling is run
	// Use when debugging to run scheduling sooner
	// Type: int
	// Default: 30
	DebugScheduleDelay = "debug.schedule_delay"

	// If set to true, terminations will be guaranteed
	// to be scheduled for all eligible Deployments,
	// i.e., probability of kill = 1
	// Type: bool
	// Default: false
	DebugForceShouldKill = "debug.force_should_kill"

	// If set to true, pod terminations will be scheduled
	// sometime in the next 60 sec to facilitate
	// debugging (instead of the hours specified by
	// StartHour and EndHour)
	// Type: bool
	// Default: false
	DebugScheduleImmediateKill = "debug.schedule_immediate_kill"
)
<|endoftext|>


<commit_before>package consent

import (
	"fmt"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"github.com/ory/fosite"
)

func TestToRFCError(t *testing.T) {
	for k, tc := range []struct {
		input  *RequestDeniedError
		expect *fosite.RFC6749Error
	}{
		{
			input: &RequestDeniedError{
				Name:  "not empty",
				valid: true,
			},
			expect: &fosite.RFC6749Error{
				Name:        "not empty",
				Description: "",
				Code:        fosite.ErrInvalidRequest.Code,
				Debug:       "",
			},
		},
		{
			input: &RequestDeniedError{
				Name:        "",
				Description: "not empty",
				valid:       true,
			},
			expect: &fosite.RFC6749Error{
				Name:        "request was denied",
				Description: "not empty",
				Code:        fosite.ErrInvalidRequest.Code,
				Debug:       "",
			},
		},
		{
			input: &RequestDeniedError{valid: true},
			expect: &fosite.RFC6749Error{
				Name:        "request was denied",
				Description: "",
				Hint:        "",
				Code:        fosite.ErrInvalidRequest.Code,
				Debug:       "",
			},
		},
	} {
		t.Run(fmt.Sprintf("case=%d", k), func(t *testing.T) {
			require.EqualValues(t, tc.input.toRFCError(), tc.expect)
		})
	}
}

func TestRequestDeniedError(t *testing.T) {
	var e *RequestDeniedError
	v, err := e.Value()
	require.NoError(t, err)
	assert.EqualValues(t, "{}", fmt.Sprintf("%v", v))
}
<commit_msg>fix: use correct assertion in test<commit_after>package consent

import (
	"fmt"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"github.com/ory/fosite"
)

func TestToRFCError(t *testing.T) {
	for k, tc := range []struct {
		input  *RequestDeniedError
		expect *fosite.RFC6749Error
	}{
		{
			input: &RequestDeniedError{
				Name:  "not empty",
				valid: true,
			},
			expect: &fosite.RFC6749Error{
				Name:        "not empty",
				Description: "",
				Code:        fosite.ErrInvalidRequest.Code,
				Debug:       "",
			},
		},
		{
			input: &RequestDeniedError{
				Name:        "",
				Description: "not empty",
				valid:       true,
			},
			expect: &fosite.RFC6749Error{
				Name:        "request_denied",
				Description: "not empty",
				Code:        fosite.ErrInvalidRequest.Code,
				Debug:       "",
			},
		},
		{
			input: &RequestDeniedError{valid: true},
			expect: &fosite.RFC6749Error{
				Name:        "request_denied",
				Description: "",
				Hint:        "",
				Code:        fosite.ErrInvalidRequest.Code,
				Debug:       "",
			},
		},
	} {
		t.Run(fmt.Sprintf("case=%d", k), func(t *testing.T) {
			require.EqualValues(t, tc.input.toRFCError(), tc.expect)
		})
	}
}

func TestRequestDeniedError(t *testing.T) {
	var e *RequestDeniedError
	v, err := e.Value()
	require.NoError(t, err)
	assert.EqualValues(t, "{}", fmt.Sprintf("%v", v))
}
<|endoftext|>


<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package container

import (
	"io/ioutil"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/loggo/loggo"

	coreCloudinit "launchpad.net/juju-core/cloudinit"
	"launchpad.net/juju-core/environs/cloudinit"
	"launchpad.net/juju-core/utils"
)

var (
	logger         = loggo.GetLogger("juju.container")
	aptHTTPProxyRE = regexp.MustCompile(`(?i)^Acquire::HTTP::Proxy\s+"([^"]+)";$`)
)

func WriteUserData(machineConfig *cloudinit.MachineConfig, directory string) (string, error) {
	userData, err := cloudInitUserData(machineConfig)
	if err != nil {
		logger.Errorf("failed to create user data: %v", err)
		return "", err
	}
	userDataFilename := filepath.Join(directory, "cloud-init")
	if err := ioutil.WriteFile(userDataFilename, userData, 0644); err != nil {
		logger.Errorf("failed to write user data: %v", err)
		return "", err
	}
	return userDataFilename, nil
}

func cloudInitUserData(machineConfig *cloudinit.MachineConfig) ([]byte, error) {
	// consider not having this line hardcoded...
	machineConfig.DataDir = "/var/lib/juju"
	cloudConfig := coreCloudinit.New()
	err := cloudinit.Configure(machineConfig, cloudConfig)
	if err != nil {
		return nil, err
	}

	// Run apt-config to fetch proxy settings from host. If no proxy
	// settings are configured, then we don't set up any proxy information
	// on the container.
	proxyConfig, err := utils.AptConfigProxy()
	if err != nil {
		return nil, err
	}
	if proxyConfig != "" {
		var proxyLines []string
		for _, line := range strings.Split(proxyConfig, "\n") {
			line = strings.TrimSpace(line)
			if len(line) > 0 {
				if m := aptHTTPProxyRE.FindStringSubmatch(line); m != nil {
					cloudConfig.SetAptProxy(m[1])
				} else {
					proxyLines = append(proxyLines, line)
				}
			}
		}
		if len(proxyLines) > 0 {
			cloudConfig.AddFile(
				"/etc/apt/apt.conf.d/99proxy-extra",
				strings.Join(proxyLines, "\n"),
				0644)
		}
	}

	// Run ifconfig to get the addresses of the internal container at least
	// logged in the host.
	cloudConfig.AddRunCmd("ifconfig")

	data, err := cloudConfig.Render()
	if err != nil {
		return nil, err
	}
	return data, nil
}
<commit_msg>Don't treat containers specially for apt proxy settings.<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package container

import (
	"io/ioutil"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/loggo/loggo"

	coreCloudinit "launchpad.net/juju-core/cloudinit"
	"launchpad.net/juju-core/environs/cloudinit"
	"launchpad.net/juju-core/utils"
)

var (
	logger = loggo.GetLogger("juju.container")
)

func WriteUserData(machineConfig *cloudinit.MachineConfig, directory string) (string, error) {
	userData, err := cloudInitUserData(machineConfig)
	if err != nil {
		logger.Errorf("failed to create user data: %v", err)
		return "", err
	}
	userDataFilename := filepath.Join(directory, "cloud-init")
	if err := ioutil.WriteFile(userDataFilename, userData, 0644); err != nil {
		logger.Errorf("failed to write user data: %v", err)
		return "", err
	}
	return userDataFilename, nil
}

func cloudInitUserData(machineConfig *cloudinit.MachineConfig) ([]byte, error) {
	// consider not having this line hardcoded...
	machineConfig.DataDir = "/var/lib/juju"
	cloudConfig := coreCloudinit.New()
	err := cloudinit.Configure(machineConfig, cloudConfig)
	if err != nil {
		return nil, err
	}

	// Run ifconfig to get the addresses of the internal container at least
	// logged in the host.
	cloudConfig.AddRunCmd("ifconfig")

	data, err := cloudConfig.Render()
	if err != nil {
		return nil, err
	}
	return data, nil
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"os"
	"os/exec"
	"strings"
)

var cmdRun = &Command{
	UsageLine: "run [build flags] gofiles... [arguments...]",
	Short:     "compile and run Go program",
	Long: `
Run compiles and runs the main package comprising the named Go source files.

For more about build flags, see 'go help build'.

See also: go build.
	`,
}

func init() {
	cmdRun.Run = runRun // break init loop

	addBuildFlags(cmdRun)
}

func printStderr(args ...interface{}) (int, error) {
	return fmt.Fprint(os.Stderr, args...)
}

func runRun(cmd *Command, args []string) {
	var b builder
	b.init()
	b.print = printStderr
	i := 0
	for i < len(args) && strings.HasSuffix(args[i], ".go") {
		i++
	}
	files, cmdArgs := args[:i], args[i:]
	if len(files) == 0 {
		fatalf("go run: no go files listed")
	}
	p := goFilesPackage(files)
	if p.Error != nil {
		fatalf("%s", p.Error)
	}
	if p.Name != "main" {
		fatalf("go run: cannot run non-main package")
	}
	p.target = "" // must build - not up to date
	a1 := b.action(modeBuild, modeBuild, p)
	a := &action{f: (*builder).runProgram, args: cmdArgs, deps: []*action{a1}}
	b.do(a)
}

// runProgram is the action for running a binary that has already
// been compiled.  We ignore exit status.
func (b *builder) runProgram(a *action) error {
	if buildN || buildX {
		b.showcmd("", "%s %s", a.deps[0].target, strings.Join(a.args, " "))
		if buildN {
			return nil
		}
	}

	runStdin(a.deps[0].target, a.args)
	return nil
}

// runStdin is like run, but connects Stdin.
func runStdin(cmdargs ...interface{}) {
	cmdline := stringList(cmdargs...)
	cmd := exec.Command(cmdline[0], cmdline[1:]...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		errorf("%v", err)
	}
}
<commit_msg>cmd/go: do not ignore DepsErrors in 'go run'<commit_after>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"os"
	"os/exec"
	"strings"
)

var cmdRun = &Command{
	UsageLine: "run [build flags] gofiles... [arguments...]",
	Short:     "compile and run Go program",
	Long: `
Run compiles and runs the main package comprising the named Go source files.

For more about build flags, see 'go help build'.

See also: go build.
	`,
}

func init() {
	cmdRun.Run = runRun // break init loop

	addBuildFlags(cmdRun)
}

func printStderr(args ...interface{}) (int, error) {
	return fmt.Fprint(os.Stderr, args...)
}

func runRun(cmd *Command, args []string) {
	var b builder
	b.init()
	b.print = printStderr
	i := 0
	for i < len(args) && strings.HasSuffix(args[i], ".go") {
		i++
	}
	files, cmdArgs := args[:i], args[i:]
	if len(files) == 0 {
		fatalf("go run: no go files listed")
	}
	p := goFilesPackage(files)
	if p.Error != nil {
		fatalf("%s", p.Error)
	}
	for _, err := range p.DepsErrors {
		errorf("%s", err)
	}
	exitIfErrors()
	if p.Name != "main" {
		fatalf("go run: cannot run non-main package")
	}
	p.target = "" // must build - not up to date
	a1 := b.action(modeBuild, modeBuild, p)
	a := &action{f: (*builder).runProgram, args: cmdArgs, deps: []*action{a1}}
	b.do(a)
}

// runProgram is the action for running a binary that has already
// been compiled.  We ignore exit status.
func (b *builder) runProgram(a *action) error {
	if buildN || buildX {
		b.showcmd("", "%s %s", a.deps[0].target, strings.Join(a.args, " "))
		if buildN {
			return nil
		}
	}

	runStdin(a.deps[0].target, a.args)
	return nil
}

// runStdin is like run, but connects Stdin.
func runStdin(cmdargs ...interface{}) {
	cmdline := stringList(cmdargs...)
	cmd := exec.Command(cmdline[0], cmdline[1:]...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		errorf("%v", err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2011, 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package charm

import (
	"archive/zip"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"
)

// The CharmDir type encapsulates access to data and operations
// on a charm directory.
type CharmDir struct {
	Path     string
	meta     *Meta
	config   *Config
	actions  *Actions
	revision int
}

// Trick to ensure *CharmDir implements the Charm interface.
var _ Charm = (*CharmDir)(nil)

// ReadCharmDir returns a CharmDir representing an expanded charm directory.
func ReadCharmDir(path string) (dir *CharmDir, err error) {
	dir = &CharmDir{Path: path}
	file, err := os.Open(dir.join("metadata.yaml"))
	if err != nil {
		return nil, err
	}
	dir.meta, err = ReadMeta(file)
	file.Close()
	if err != nil {
		return nil, err
	}
	file, err = os.Open(dir.join("config.yaml"))
	if _, ok := err.(*os.PathError); ok {
		dir.config = NewConfig()
	} else if err != nil {
		return nil, err
	} else {
		dir.config, err = ReadConfig(file)
		file.Close()
		if err != nil {
			return nil, err
		}
	}
	file, err = os.Open(dir.join("actions.yaml"))
	if _, ok := err.(*os.PathError); ok {
		dir.actions = NewActions()
	} else if err != nil {
		return nil, err
	} else {
		dir.actions, err = ReadActionsYaml(file)
		file.Close()
		if err != nil {
			return nil, err
		}
	}
	if file, err = os.Open(dir.join("revision")); err == nil {
		_, err = fmt.Fscan(file, &dir.revision)
		file.Close()
		if err != nil {
			return nil, errors.New("invalid revision file")
		}
	} else {
		dir.revision = dir.meta.OldRevision
	}

	return dir, nil
}

// join builds a path rooted at the charm's expanded directory
// path and the extra path components provided.
func (dir *CharmDir) join(parts ...string) string {
	parts = append([]string{dir.Path}, parts...)
	return filepath.Join(parts...)
}

// Revision returns the revision number for the charm
// expanded in dir.
func (dir *CharmDir) Revision() int {
	return dir.revision
}

// Meta returns the Meta representing the metadata.yaml file
// for the charm expanded in dir.
func (dir *CharmDir) Meta() *Meta {
	return dir.meta
}

// Config returns the Config representing the config.yaml file
// for the charm expanded in dir.
func (dir *CharmDir) Config() *Config {
	return dir.config
}

// Actions returns the Actions representing the actions.yaml file
// for the charm expanded in dir.
func (dir *CharmDir) Actions() *Actions {
	return dir.actions
}

// SetRevision changes the charm revision number. This affects
// the revision reported by Revision and the revision of the
// charm archived by ArchiveTo.
// The revision file in the charm directory is not modified.
func (dir *CharmDir) SetRevision(revision int) {
	dir.revision = revision
}

// SetDiskRevision does the same as SetRevision but also changes
// the revision file in the charm directory.
func (dir *CharmDir) SetDiskRevision(revision int) error {
	dir.SetRevision(revision)
	file, err := os.OpenFile(dir.join("revision"), os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		return err
	}
	_, err = file.Write([]byte(strconv.Itoa(revision)))
	file.Close()
	return err
}

// resolveSymlinkedRoot returns the target destination of a
// charm root directory if the root directory is a symlink.
func resolveSymlinkedRoot(rootPath string) (string, error) {
	info, err := os.Lstat(rootPath)
	if err == nil && info.Mode()&os.ModeSymlink != 0 {
		rootPath, err = filepath.EvalSymlinks(rootPath)
		if err != nil {
			return "", fmt.Errorf("cannot read path symlink at %q: %v", rootPath, err)
		}
	}
	return rootPath, nil
}

// ArchiveTo creates a charm file from the charm expanded in dir.
// By convention a charm archive should have a ".charm" suffix.
func (dir *CharmDir) ArchiveTo(w io.Writer) error {
	return writeArchive(w, dir.Path, dir.revision, dir.Meta().Hooks())
}

func writeArchive(w io.Writer, path string, revision int, hooks map[string]bool) error {
	zipw := zip.NewWriter(w)
	defer zipw.Close()

	// The root directory may be symlinked elsewhere so
	// resolve that before creating the zip.
	rootPath, err := resolveSymlinkedRoot(path)
	if err != nil {
		return err
	}
	zp := zipPacker{zipw, rootPath, hooks}
	if revision != -1 {
		zp.AddRevision(revision)
	}
	return filepath.Walk(rootPath, zp.WalkFunc())
}

type zipPacker struct {
	*zip.Writer
	root  string
	hooks map[string]bool
}

func (zp *zipPacker) WalkFunc() filepath.WalkFunc {
	return func(path string, fi os.FileInfo, err error) error {
		return zp.visit(path, fi, err)
	}
}

func (zp *zipPacker) AddRevision(revision int) error {
	h := &zip.FileHeader{Name: "revision"}
	h.SetMode(syscall.S_IFREG | 0644)
	w, err := zp.CreateHeader(h)
	if err == nil {
		_, err = w.Write([]byte(strconv.Itoa(revision)))
	}
	return err
}

func (zp *zipPacker) visit(path string, fi os.FileInfo, err error) error {
	if err != nil {
		return err
	}
	relpath, err := filepath.Rel(zp.root, path)
	if err != nil {
		return err
	}
	method := zip.Deflate
	hidden := len(relpath) > 1 && relpath[0] == '.'
	if fi.IsDir() {
		if relpath == "build" {
			return filepath.SkipDir
		}
		if hidden {
			return filepath.SkipDir
		}
		relpath += "/"
		method = zip.Store
	}

	mode := fi.Mode()
	if err := checkFileType(relpath, mode); err != nil {
		return err
	}
	if mode&os.ModeSymlink != 0 {
		method = zip.Store
	}
	if hidden || relpath == "revision" {
		return nil
	}
	h := &zip.FileHeader{
		Name:   relpath,
		Method: method,
	}

	perm := os.FileMode(0644)
	if mode&os.ModeSymlink != 0 {
		perm = 0777
	} else if mode&0100 != 0 {
		perm = 0755
	}
	if filepath.Dir(relpath) == "hooks" {
		hookName := filepath.Base(relpath)
		if _, ok := zp.hooks[hookName]; ok && !fi.IsDir() && mode&0100 == 0 {
			logger.Warningf("making %q executable in charm", path)
			perm = perm | 0100
		}
	}
	h.SetMode(mode&^0777 | perm)

	w, err := zp.CreateHeader(h)
	if err != nil || fi.IsDir() {
		return err
	}
	var data []byte
	if mode&os.ModeSymlink != 0 {
		target, err := os.Readlink(path)
		if err != nil {
			return err
		}
		if err := checkSymlinkTarget(zp.root, relpath, target); err != nil {
			return err
		}
		data = []byte(target)
	} else {
		data, err = ioutil.ReadFile(path)
		if err != nil {
			return err
		}
	}
	_, err = w.Write(data)
	return err
}

func checkSymlinkTarget(basedir, symlink, target string) error {
	if filepath.IsAbs(target) {
		return fmt.Errorf("symlink %q is absolute: %q", symlink, target)
	}
	p := filepath.Join(filepath.Dir(symlink), target)
	if p == ".." || strings.HasPrefix(p, "../") {
		return fmt.Errorf("symlink %q links out of charm: %q", symlink, target)
	}
	return nil
}

func checkFileType(path string, mode os.FileMode) error {
	e := "file has an unknown type: %q"
	switch mode & os.ModeType {
	case os.ModeDir, os.ModeSymlink, 0:
		return nil
	case os.ModeNamedPipe:
		e = "file is a named pipe: %q"
	case os.ModeSocket:
		e = "file is a socket: %q"
	case os.ModeDevice:
		e = "file is a device: %q"
	}
	return fmt.Errorf(e, path)
}
<commit_msg>Prefer io.Copy to needless allocation<commit_after>// Copyright 2011, 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package charm

import (
	"archive/zip"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"
)

// The CharmDir type encapsulates access to data and operations
// on a charm directory.
type CharmDir struct {
	Path     string
	meta     *Meta
	config   *Config
	actions  *Actions
	revision int
}

// Trick to ensure *CharmDir implements the Charm interface.
var _ Charm = (*CharmDir)(nil)

// ReadCharmDir returns a CharmDir representing an expanded charm directory.
func ReadCharmDir(path string) (dir *CharmDir, err error) {
	dir = &CharmDir{Path: path}
	file, err := os.Open(dir.join("metadata.yaml"))
	if err != nil {
		return nil, err
	}
	dir.meta, err = ReadMeta(file)
	file.Close()
	if err != nil {
		return nil, err
	}
	file, err = os.Open(dir.join("config.yaml"))
	if _, ok := err.(*os.PathError); ok {
		dir.config = NewConfig()
	} else if err != nil {
		return nil, err
	} else {
		dir.config, err = ReadConfig(file)
		file.Close()
		if err != nil {
			return nil, err
		}
	}
	file, err = os.Open(dir.join("actions.yaml"))
	if _, ok := err.(*os.PathError); ok {
		dir.actions = NewActions()
	} else if err != nil {
		return nil, err
	} else {
		dir.actions, err = ReadActionsYaml(file)
		file.Close()
		if err != nil {
			return nil, err
		}
	}
	if file, err = os.Open(dir.join("revision")); err == nil {
		_, err = fmt.Fscan(file, &dir.revision)
		file.Close()
		if err != nil {
			return nil, errors.New("invalid revision file")
		}
	} else {
		dir.revision = dir.meta.OldRevision
	}

	return dir, nil
}

// join builds a path rooted at the charm's expanded directory
// path and the extra path components provided.
func (dir *CharmDir) join(parts ...string) string {
	parts = append([]string{dir.Path}, parts...)
	return filepath.Join(parts...)
}

// Revision returns the revision number for the charm
// expanded in dir.
func (dir *CharmDir) Revision() int {
	return dir.revision
}

// Meta returns the Meta representing the metadata.yaml file
// for the charm expanded in dir.
func (dir *CharmDir) Meta() *Meta {
	return dir.meta
}

// Config returns the Config representing the config.yaml file
// for the charm expanded in dir.
func (dir *CharmDir) Config() *Config {
	return dir.config
}

// Actions returns the Actions representing the actions.yaml file
// for the charm expanded in dir.
func (dir *CharmDir) Actions() *Actions {
	return dir.actions
}

// SetRevision changes the charm revision number. This affects
// the revision reported by Revision and the revision of the
// charm archived by ArchiveTo.
// The revision file in the charm directory is not modified.
func (dir *CharmDir) SetRevision(revision int) {
	dir.revision = revision
}

// SetDiskRevision does the same as SetRevision but also changes
// the revision file in the charm directory.
func (dir *CharmDir) SetDiskRevision(revision int) error {
	dir.SetRevision(revision)
	file, err := os.OpenFile(dir.join("revision"), os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		return err
	}
	_, err = file.Write([]byte(strconv.Itoa(revision)))
	file.Close()
	return err
}

// resolveSymlinkedRoot returns the target destination of a
// charm root directory if the root directory is a symlink.
func resolveSymlinkedRoot(rootPath string) (string, error) {
	info, err := os.Lstat(rootPath)
	if err == nil && info.Mode()&os.ModeSymlink != 0 {
		rootPath, err = filepath.EvalSymlinks(rootPath)
		if err != nil {
			return "", fmt.Errorf("cannot read path symlink at %q: %v", rootPath, err)
		}
	}
	return rootPath, nil
}

// ArchiveTo creates a charm file from the charm expanded in dir.
// By convention a charm archive should have a ".charm" suffix.
func (dir *CharmDir) ArchiveTo(w io.Writer) error {
	return writeArchive(w, dir.Path, dir.revision, dir.Meta().Hooks())
}

func writeArchive(w io.Writer, path string, revision int, hooks map[string]bool) error {
	zipw := zip.NewWriter(w)
	defer zipw.Close()

	// The root directory may be symlinked elsewhere so
	// resolve that before creating the zip.
	rootPath, err := resolveSymlinkedRoot(path)
	if err != nil {
		return err
	}
	zp := zipPacker{zipw, rootPath, hooks}
	if revision != -1 {
		zp.AddRevision(revision)
	}
	return filepath.Walk(rootPath, zp.WalkFunc())
}

type zipPacker struct {
	*zip.Writer
	root  string
	hooks map[string]bool
}

func (zp *zipPacker) WalkFunc() filepath.WalkFunc {
	return func(path string, fi os.FileInfo, err error) error {
		return zp.visit(path, fi, err)
	}
}

func (zp *zipPacker) AddRevision(revision int) error {
	h := &zip.FileHeader{Name: "revision"}
	h.SetMode(syscall.S_IFREG | 0644)
	w, err := zp.CreateHeader(h)
	if err == nil {
		_, err = w.Write([]byte(strconv.Itoa(revision)))
	}
	return err
}

func (zp *zipPacker) visit(path string, fi os.FileInfo, err error) error {
	if err != nil {
		return err
	}
	relpath, err := filepath.Rel(zp.root, path)
	if err != nil {
		return err
	}
	method := zip.Deflate
	hidden := len(relpath) > 1 && relpath[0] == '.'
	if fi.IsDir() {
		if relpath == "build" {
			return filepath.SkipDir
		}
		if hidden {
			return filepath.SkipDir
		}
		relpath += "/"
		method = zip.Store
	}

	mode := fi.Mode()
	if err := checkFileType(relpath, mode); err != nil {
		return err
	}
	if mode&os.ModeSymlink != 0 {
		method = zip.Store
	}
	if hidden || relpath == "revision" {
		return nil
	}
	h := &zip.FileHeader{
		Name:   relpath,
		Method: method,
	}

	perm := os.FileMode(0644)
	if mode&os.ModeSymlink != 0 {
		perm = 0777
	} else if mode&0100 != 0 {
		perm = 0755
	}
	if filepath.Dir(relpath) == "hooks" {
		hookName := filepath.Base(relpath)
		if _, ok := zp.hooks[hookName]; ok && !fi.IsDir() && mode&0100 == 0 {
			logger.Warningf("making %q executable in charm", path)
			perm = perm | 0100
		}
	}
	h.SetMode(mode&^0777 | perm)

	w, err := zp.CreateHeader(h)
	if err != nil || fi.IsDir() {
		return err
	}
	var data []byte
	if mode&os.ModeSymlink != 0 {
		target, err := os.Readlink(path)
		if err != nil {
			return err
		}
		if err := checkSymlinkTarget(zp.root, relpath, target); err != nil {
			return err
		}
		data = []byte(target)
		_, err = w.Write(data)
	} else {
		file, err := os.Open(path)
		if err != nil {
			return err
		}
		_, err = io.Copy(w, file)
	}
	return err
}

func checkSymlinkTarget(basedir, symlink, target string) error {
	if filepath.IsAbs(target) {
		return fmt.Errorf("symlink %q is absolute: %q", symlink, target)
	}
	p := filepath.Join(filepath.Dir(symlink), target)
	if p == ".." || strings.HasPrefix(p, "../") {
		return fmt.Errorf("symlink %q links out of charm: %q", symlink, target)
	}
	return nil
}

func checkFileType(path string, mode os.FileMode) error {
	e := "file has an unknown type: %q"
	switch mode & os.ModeType {
	case os.ModeDir, os.ModeSymlink, 0:
		return nil
	case os.ModeNamedPipe:
		e = "file is a named pipe: %q"
	case os.ModeSocket:
		e = "file is a socket: %q"
	case os.ModeDevice:
		e = "file is a device: %q"
	}
	return fmt.Errorf(e, path)
}
<|endoftext|>


<commit_before>package ntemplate

import (
	"os"
	"path/filepath"
	"strings"

	"github.com/webx-top/com"
	"github.com/webx-top/echo"
)

type PathAliases struct {
	aliases  map[string][]string
	tmplDirs []string
}

func (p *PathAliases) TmplDirs() []string {
	return p.tmplDirs
}

func (p *PathAliases) Aliases() []string {
	aliases := make([]string, len(p.aliases))
	var i int
	for alias := range p.aliases {
		aliases[i] = alias
		i++
	}
	return aliases
}

func (p *PathAliases) AddAllSubdir(absPath string) error {
	fp, err := os.Open(absPath)
	if err != nil {
		return err
	}
	defer fp.Close()
	dirs, err := fp.ReadDir(-1)
	if err != nil {
		return err
	}
	for _, dir := range dirs {
		if strings.HasPrefix(dir.Name(), `.`) {
			continue
		}
		p.Add(dir.Name(), absPath)
	}
	return nil
}

func (p *PathAliases) Add(alias, absPath string) *PathAliases {
	var err error
	absPath, err = filepath.Abs(absPath)
	if err != nil {
		panic(err)
	}
	if !com.InSlice(absPath, p.tmplDirs) {
		p.tmplDirs = append(p.tmplDirs, absPath)
	}
	if p.aliases == nil {
		p.aliases = map[string][]string{}
	}
	if _, ok := p.aliases[alias]; !ok {
		p.aliases[alias] = []string{}
	}
	if !strings.HasSuffix(absPath, echo.FilePathSeparator) {
		absPath += echo.FilePathSeparator
	}
	p.aliases[alias] = append(p.aliases[alias], absPath)
	return p
}

func (p *PathAliases) ParsePrefix(withAliasPrefixPath string) string {
	rpath, _ := p.ParsePrefixOk(withAliasPrefixPath)
	return rpath
}

func (p *PathAliases) ParsePrefixOk(withAliasPrefixPath string) (string, bool) {
	if len(withAliasPrefixPath) < 3 {
		return withAliasPrefixPath, false
	}
	if withAliasPrefixPath[0] == '/' || withAliasPrefixPath[0] == '.' {
		fi, err := os.Stat(withAliasPrefixPath)
		if err == nil && !fi.IsDir() {
			return withAliasPrefixPath, false
		}
		withAliasPrefixPath = withAliasPrefixPath[1:]
	}
	parts := strings.SplitN(withAliasPrefixPath, `/`, 2)
	if len(parts) != 2 {
		return withAliasPrefixPath, false
	}
	alias := parts[0]
	if opaths, ok := p.aliases[alias]; ok {
		if len(opaths) == 1 {
			return filepath.Join(opaths[0], withAliasPrefixPath), true
		}
		for _, opath := range opaths {
			_tmpl := filepath.Join(opath, withAliasPrefixPath)
			fi, err := os.Stat(_tmpl)
			if err == nil && !fi.IsDir() {
				return _tmpl, true
			}
		}
	}
	return withAliasPrefixPath, false
}

func (p *PathAliases) RestorePrefix(fullpath string) string {
	rpath, _ := p.RestorePrefixOk(fullpath)
	return rpath
}

func (p *PathAliases) RestorePrefixOk(fullpath string) (string, bool) {
	for _, absPaths := range p.aliases {
		for _, absPath := range absPaths {
			if strings.HasPrefix(fullpath, absPath) {
				return filepath.ToSlash(fullpath[len(absPath):]), true
			}
		}
	}
	return fullpath, false
}

func (p *PathAliases) Parse(withAliasTagPath string) string {
	rpath, _ := p.ParseOk(withAliasTagPath)
	return rpath
}

func (p *PathAliases) ParseOk(withAliasTagPath string) (string, bool) {
	if len(withAliasTagPath) < 3 || withAliasTagPath[0] != '[' {
		return withAliasTagPath, false
	}
	withAliasTagPath = withAliasTagPath[1:]
	parts := strings.SplitN(withAliasTagPath, `]`, 2)
	if len(parts) != 2 {
		return withAliasTagPath, false
	}
	alias := parts[0]
	rpath := parts[1]
	if opaths, ok := p.aliases[alias]; ok {
		if len(opaths) == 1 {
			return filepath.Join(opaths[0], rpath), true
		}
		for _, opath := range opaths {
			_tmpl := filepath.Join(opath, rpath)
			fi, err := os.Stat(_tmpl)
			if err == nil && !fi.IsDir() {
				return _tmpl, true
			}
		}
	}
	return rpath, false
}

func (p *PathAliases) Restore(fullpath string) string {
	rpath, _ := p.RestoreOk(fullpath)
	return rpath
}

func (p *PathAliases) RestoreOk(fullpath string) (string, bool) {
	for alias, absPaths := range p.aliases {
		for _, absPath := range absPaths {
			if strings.HasPrefix(fullpath, absPath) {
				return `[` + alias + `]` + filepath.ToSlash(fullpath[len(absPath):]), true
			}
		}
	}
	return fullpath, false
}
<commit_msg>update<commit_after>package ntemplate

import (
	"os"
	"path/filepath"
	"strings"

	"github.com/webx-top/com"
	"github.com/webx-top/echo"
)

type PathAliases struct {
	aliases  map[string][]string
	tmplDirs []string
}

func (p *PathAliases) TmplDirs() []string {
	return p.tmplDirs
}

func (p *PathAliases) Aliases() []string {
	aliases := make([]string, len(p.aliases))
	var i int
	for alias := range p.aliases {
		aliases[i] = alias
		i++
	}
	return aliases
}

func (p *PathAliases) Range(fn func(string, string) error) (err error) {
	for alias, templateDirs := range p.aliases {
		for _, templateDir := range templateDirs {
			err = fn(alias, templateDir)
			if err != nil {
				return
			}
		}
	}
	return
}

func (p *PathAliases) AddAllSubdir(absPath string) error {
	fp, err := os.Open(absPath)
	if err != nil {
		return err
	}
	defer fp.Close()
	dirs, err := fp.ReadDir(-1)
	if err != nil {
		return err
	}
	for _, dir := range dirs {
		if strings.HasPrefix(dir.Name(), `.`) {
			continue
		}
		p.Add(dir.Name(), absPath)
	}
	return nil
}

func (p *PathAliases) Add(alias, absPath string) *PathAliases {
	var err error
	absPath, err = filepath.Abs(absPath)
	if err != nil {
		panic(err)
	}
	if !com.InSlice(absPath, p.tmplDirs) {
		p.tmplDirs = append(p.tmplDirs, absPath)
	}
	if p.aliases == nil {
		p.aliases = map[string][]string{}
	}
	if _, ok := p.aliases[alias]; !ok {
		p.aliases[alias] = []string{}
	}
	if !strings.HasSuffix(absPath, echo.FilePathSeparator) {
		absPath += echo.FilePathSeparator
	}
	p.aliases[alias] = append(p.aliases[alias], absPath)
	return p
}

func (p *PathAliases) ParsePrefix(withAliasPrefixPath string) string {
	rpath, _ := p.ParsePrefixOk(withAliasPrefixPath)
	return rpath
}

func (p *PathAliases) ParsePrefixOk(withAliasPrefixPath string) (string, bool) {
	if len(withAliasPrefixPath) < 3 {
		return withAliasPrefixPath, false
	}
	if withAliasPrefixPath[0] == '/' || withAliasPrefixPath[0] == '.' {
		fi, err := os.Stat(withAliasPrefixPath)
		if err == nil && !fi.IsDir() {
			return withAliasPrefixPath, false
		}
		withAliasPrefixPath = withAliasPrefixPath[1:]
	}
	parts := strings.SplitN(withAliasPrefixPath, `/`, 2)
	if len(parts) != 2 {
		return withAliasPrefixPath, false
	}
	alias := parts[0]
	if opaths, ok := p.aliases[alias]; ok {
		if len(opaths) == 1 {
			return filepath.Join(opaths[0], withAliasPrefixPath), true
		}
		for _, opath := range opaths {
			_tmpl := filepath.Join(opath, withAliasPrefixPath)
			fi, err := os.Stat(_tmpl)
			if err == nil && !fi.IsDir() {
				return _tmpl, true
			}
		}
	}
	return withAliasPrefixPath, false
}

func (p *PathAliases) RestorePrefix(fullpath string) string {
	rpath, _ := p.RestorePrefixOk(fullpath)
	return rpath
}

func (p *PathAliases) RestorePrefixOk(fullpath string) (string, bool) {
	for _, absPaths := range p.aliases {
		for _, absPath := range absPaths {
			if strings.HasPrefix(fullpath, absPath) {
				return filepath.ToSlash(fullpath[len(absPath):]), true
			}
		}
	}
	return fullpath, false
}

func (p *PathAliases) Parse(withAliasTagPath string) string {
	rpath, _ := p.ParseOk(withAliasTagPath)
	return rpath
}

func (p *PathAliases) ParseOk(withAliasTagPath string) (string, bool) {
	if len(withAliasTagPath) < 3 || withAliasTagPath[0] != '[' {
		return withAliasTagPath, false
	}
	withAliasTagPath = withAliasTagPath[1:]
	parts := strings.SplitN(withAliasTagPath, `]`, 2)
	if len(parts) != 2 {
		return withAliasTagPath, false
	}
	alias := parts[0]
	rpath := parts[1]
	if opaths, ok := p.aliases[alias]; ok {
		if len(opaths) == 1 {
			return filepath.Join(opaths[0], rpath), true
		}
		for _, opath := range opaths {
			_tmpl := filepath.Join(opath, rpath)
			fi, err := os.Stat(_tmpl)
			if err == nil && !fi.IsDir() {
				return _tmpl, true
			}
		}
	}
	return rpath, false
}

func (p *PathAliases) Restore(fullpath string) string {
	rpath, _ := p.RestoreOk(fullpath)
	return rpath
}

func (p *PathAliases) RestoreOk(fullpath string) (string, bool) {
	for alias, absPaths := range p.aliases {
		for _, absPath := range absPaths {
			if strings.HasPrefix(fullpath, absPath) {
				return `[` + alias + `]` + filepath.ToSlash(fullpath[len(absPath):]), true
			}
		}
	}
	return fullpath, false
}
<|endoftext|>


<commit_before>package checkdisk

import (
	"errors"
	"fmt"
	"os"
	"strconv"
	"strings"
	"syscall"

	"github.com/jessevdk/go-flags"
	"github.com/mackerelio/checkers"
	gpud "github.com/shirou/gopsutil/disk"
)

type diskStatus struct {
	Dev   string
	All   uint64
	Used  uint64
	Free  uint64
	Avail uint64
}

var opts struct {
	Warning  *string `short:"w" long:"warning" value-name:"N, N%" description:"Exit with WARNING status if less than N units or N% of disk are free"`
	Critical *string `short:"c" long:"critical" value-name:"N, N%" description:"Exit with CRITICAL status if less than N units or N% of disk are free"`
	Path     *string `short:"p" long:"path" value-name:"PATH" description:"Mount point or block device as emitted by the mount(8) command"`
	Exclude  *string `short:"x" long:"exclude_device" value-name:"EXCLUDE PATH" description:"Ignore device (only works if -p unspecified)"`
	Units    *string `short:"u" long:"units" value-name:"STRING" description:"Choose bytes, kB, MB, GB, TB (default: MB)"`
}

const (
	b  = float64(1)
	kb = float64(1024) * b
	mb = float64(1024) * kb
	gb = float64(1024) * mb
	tb = float64(1024) * gb
)

type unit struct {
	Name string
	Size float64
}

func getDiskUsage(partition gpud.PartitionStat) (*diskStatus, error) {
	fs := syscall.Statfs_t{}
	err := syscall.Statfs(partition.Mountpoint, &fs)
	if err != nil {
		return nil, err
	}

	disk := &diskStatus{}
	disk.Dev = partition.Device
	disk.All = fs.Blocks * uint64(fs.Bsize)
	disk.Free = fs.Bfree * uint64(fs.Bsize)
	disk.Used = disk.All - disk.Free
	disk.Avail = fs.Bavail * uint64(fs.Bsize)

	return disk, nil
}

func checkStatus(current checkers.Status, threshold string, units float64, disk *diskStatus, status checkers.Status) (checkers.Status, error) {
	avail := float64(disk.Avail) / float64(units)
	freePct := (float64(disk.Avail) * float64(100)) / float64(disk.All)

	if strings.HasSuffix(threshold, "%") {
		v, err := strconv.ParseFloat(strings.TrimRight(threshold, "%"), 64)
		if err != nil {
			return checkers.UNKNOWN, err
		}

		if v > freePct {
			current = status
		}
	} else {
		v, err := strconv.ParseFloat(threshold, 64)
		if err != nil {
			return checkers.UNKNOWN, err
		}

		if v > avail {
			current = status
		}
	}

	return current, nil
}

func genMessage(disk *diskStatus, u unit) string {
	all := float64(disk.All) / u.Size
	used := float64(disk.Used) / u.Size
	free := float64(disk.Free) / u.Size
	avail := float64(disk.Avail) / u.Size
	freePct := (float64(disk.Avail) * float64(100)) / float64(disk.All)

	return fmt.Sprintf("Dev: %v, All: %.2f %v, Used: %.2f %v, Free: %.2f %v, Available: %.2f %v, Free percentage: %.2f", disk.Dev, all, u.Name, used, u.Name, free, u.Name, avail, u.Name, freePct)
}

// Do the plugin
func Do() {
	ckr := run(os.Args[1:])
	ckr.Name = "Disk"
	ckr.Exit()
}

func run(args []string) *checkers.Checker {
	_, err := flags.ParseArgs(&opts, args)
	if err != nil {
		os.Exit(1)
	}

	partitions, err := gpud.Partitions(true)
	if err != nil {
		return checkers.Unknown(fmt.Sprintf("Faild to fetch partitions: %s", err))
	}

	if opts.Path != nil {
		exist := false
		for _, partition := range partitions {
			if *opts.Path == partition.Mountpoint {
				partitions = make([]gpud.PartitionStat, 0)
				partitions = append(partitions, partition)
				exist = true
			}
		}

		if exist == false {
			return checkers.Unknown(fmt.Sprintf("Faild to fetch mountpoint: %s", errors.New("Invalid argument flag '-p, --path'")))
		}
	}

	if opts.Path == nil && opts.Exclude != nil {
		var tmp []gpud.PartitionStat
		for _, partition := range partitions {
			if *opts.Exclude != partition.Mountpoint {
				tmp = append(tmp, partition)
			}
		}
		partitions = tmp
	}

	var disks []*diskStatus

	for _, partition := range partitions {
		disk, err := getDiskUsage(partition)
		if err != nil {
			return checkers.Unknown(fmt.Sprintf("Faild to fetch disk usage: %s", err))
		}

		disks = append(disks, disk)
	}

	u := unit{"MB", mb}
	if opts.Units != nil {
		us := strings.ToLower(*opts.Units)
		if us == "bytes" {
			u = unit{us, b}
		} else if us == "kb" {
			u = unit{us, mb}
		} else if us == "gb" {
			u = unit{us, gb}
		} else if us == "tb" {
			u = unit{us, tb}
		} else {
			return checkers.Unknown(fmt.Sprintf("Faild to check disk status: %s", errors.New("Invalid argument flag '-u, --units'")))
		}
	}

	checkSt := checkers.OK
	if opts.Warning != nil {
		for _, disk := range disks {
			checkSt, err = checkStatus(checkSt, *opts.Warning, u.Size, disk, checkers.WARNING)
			if err != nil {
				return checkers.Unknown(fmt.Sprintf("Faild to check disk status: %s", err))
			}

			if checkSt == checkers.WARNING {
				break
			}
		}
	}

	if opts.Critical != nil {
		for _, disk := range disks {
			checkSt, err = checkStatus(checkSt, *opts.Critical, u.Size, disk, checkers.CRITICAL)
			if err != nil {
				return checkers.Unknown(fmt.Sprintf("Faild to check disk status: %s", err))
			}

			if checkSt == checkers.CRITICAL {
				break
			}
		}
	}

	var msgs []string
	for _, disk := range disks {
		msg := genMessage(disk, u)
		msgs = append(msgs, msg)
	}
	msgss := strings.Join(msgs, ";\n")

	return checkers.NewChecker(checkSt, msgss)
}
<commit_msg>Use gopsutil<commit_after>package checkdisk

import (
	"errors"
	"fmt"
	"os"
	"strconv"
	"strings"

	"github.com/jessevdk/go-flags"
	"github.com/mackerelio/checkers"
	gpud "github.com/shirou/gopsutil/disk"
)

var opts struct {
	Warning  *string `short:"w" long:"warning" value-name:"N, N%" description:"Exit with WARNING status if less than N units or N% of disk are free"`
	Critical *string `short:"c" long:"critical" value-name:"N, N%" description:"Exit with CRITICAL status if less than N units or N% of disk are free"`
	Path     *string `short:"p" long:"path" value-name:"PATH" description:"Mount point or block device as emitted by the mount(8) command"`
	Exclude  *string `short:"x" long:"exclude_device" value-name:"EXCLUDE PATH" description:"Ignore device (only works if -p unspecified)"`
	Units    *string `short:"u" long:"units" value-name:"STRING" description:"Choose bytes, kB, MB, GB, TB (default: MB)"`
}

const (
	b  = float64(1)
	kb = float64(1024) * b
	mb = float64(1024) * kb
	gb = float64(1024) * mb
	tb = float64(1024) * gb
)

type unit struct {
	Name string
	Size float64
}

func checkStatus(current checkers.Status, threshold string, units float64, disk *gpud.UsageStat, status checkers.Status) (checkers.Status, error) {
	if strings.HasSuffix(threshold, "%") {
		v, err := strconv.ParseFloat(strings.TrimRight(threshold, "%"), 64)
		if err != nil {
			return checkers.UNKNOWN, err
		}

		freePct := float64(100) - disk.UsedPercent
		inodesFreePct := float64(100) - disk.InodesUsedPercent

		if v > freePct || v > inodesFreePct {
			current = status
		}
	} else {
		v, err := strconv.ParseFloat(threshold, 64)
		if err != nil {
			return checkers.UNKNOWN, err
		}

		if v > float64(disk.Free) {
			current = status
		}
	}

	return current, nil
}

func genMessage(disk *gpud.UsageStat, u unit) string {
	all := float64(disk.Total) / u.Size
	used := float64(disk.Used) / u.Size
	free := float64(disk.Free) / u.Size
	freePct := float64(100) - disk.UsedPercent
	inodesFreePct := float64(100) - disk.InodesUsedPercent

	return fmt.Sprintf("Path: %v, All: %.2f %v, Used: %.2f %v, Free: %.2f %v, Free percentage: %.2f (inodes: %.2f)", disk.Path, all, u.Name, used, u.Name, free, u.Name, freePct, inodesFreePct)
}

// Do the plugin
func Do() {
	ckr := run(os.Args[1:])
	ckr.Name = "Disk"
	ckr.Exit()
}

func run(args []string) *checkers.Checker {
	_, err := flags.ParseArgs(&opts, args)
	if err != nil {
		os.Exit(1)
	}

	partitions, err := gpud.Partitions(true)
	if err != nil {
		return checkers.Unknown(fmt.Sprintf("Faild to fetch partitions: %s", err))
	}

	if opts.Path != nil {
		exist := false
		for _, partition := range partitions {
			if *opts.Path == partition.Mountpoint {
				partitions = make([]gpud.PartitionStat, 0)
				partitions = append(partitions, partition)
				exist = true
			}
		}

		if exist == false {
			return checkers.Unknown(fmt.Sprintf("Faild to fetch mountpoint: %s", errors.New("Invalid argument flag '-p, --path'")))
		}
	}

	if opts.Path == nil && opts.Exclude != nil {
		var tmp []gpud.PartitionStat
		for _, partition := range partitions {
			if *opts.Exclude != partition.Mountpoint {
				tmp = append(tmp, partition)
			}
		}
		partitions = tmp
	}

	var disks []*gpud.UsageStat

	for _, partition := range partitions {
		disk, err := gpud.Usage(partition.Mountpoint)
		if err != nil {
			return checkers.Unknown(fmt.Sprintf("Faild to fetch disk usage: %s", err))
		}

		disks = append(disks, disk)
	}

	u := unit{"MB", mb}
	if opts.Units != nil {
		us := strings.ToLower(*opts.Units)
		if us == "bytes" {
			u = unit{us, b}
		} else if us == "kb" {
			u = unit{us, mb}
		} else if us == "gb" {
			u = unit{us, gb}
		} else if us == "tb" {
			u = unit{us, tb}
		} else {
			return checkers.Unknown(fmt.Sprintf("Faild to check disk status: %s", errors.New("Invalid argument flag '-u, --units'")))
		}
	}

	checkSt := checkers.OK
	if opts.Warning != nil {
		for _, disk := range disks {
			checkSt, err = checkStatus(checkSt, *opts.Warning, u.Size, disk, checkers.WARNING)
			if err != nil {
				return checkers.Unknown(fmt.Sprintf("Faild to check disk status: %s", err))
			}

			if checkSt == checkers.WARNING {
				break
			}
		}
	}

	if opts.Critical != nil {
		for _, disk := range disks {
			checkSt, err = checkStatus(checkSt, *opts.Critical, u.Size, disk, checkers.CRITICAL)
			if err != nil {
				return checkers.Unknown(fmt.Sprintf("Faild to check disk status: %s", err))
			}

			if checkSt == checkers.CRITICAL {
				break
			}
		}
	}

	var msgs []string
	for _, disk := range disks {
		msg := genMessage(disk, u)
		msgs = append(msgs, msg)
	}
	msgss := strings.Join(msgs, ";\n")

	return checkers.NewChecker(checkSt, msgss)
}
<|endoftext|>


<commit_before>package assets

import (
	"bytes"
	"io/ioutil"
	"sync"
	"testing"
)

// TestEmbeddedDocs makes sure we don't forget to regenerate after documentation change
func TestEmbeddedDocs(t *testing.T) {
	t.Skip("skipping for now, paths are hard")
	testNFiles(initDocPaths, 6, t, "documents")
}

func TestDirIndex(t *testing.T) {
	t.Skip("skipping for now, paths are hard")
	testNFiles(initDirIndex, 2, t, "assets")
}

func testNFiles(fs []string, wantCnt int, t *testing.T, ftype string) {
	if len(fs) < wantCnt {
		t.Fatalf("expected %d %s. got %d", wantCnt, ftype, len(fs))
	}

	var wg sync.WaitGroup
	for _, f := range fs {
		wg.Add(1)
		// compare asset
		go func(f string) {
			defer wg.Done()
			testOneFile(f, t)
		}(f)
	}
	wg.Wait()
}

func testOneFile(f string, t *testing.T) {
	// load data from filesystem (git)
	vcsData, err := ioutil.ReadFile(f)
	if err != nil {
		t.Errorf("asset %s: could not read vcs file: %s", f, err)
		return
	}

	// load data from emdedded source
	embdData, err := Asset(f)
	if err != nil {
		t.Errorf("asset %s: could not read vcs file: %s", f, err)
		return
	}

	if !bytes.Equal(vcsData, embdData) {
		t.Errorf("asset %s: vcs and embedded data isnt equal", f)
		return
	}

	t.Logf("checked %s", f)
}
<commit_msg>note in assets test<commit_after>package assets

import (
	"bytes"
	"io/ioutil"
	"sync"
	"testing"
)

// TestEmbeddedDocs makes sure we don't forget to regenerate after documentation change
func TestEmbeddedDocs(t *testing.T) {
	testNFiles(initDocPaths, 6, t, "documents")
}

func TestDirIndex(t *testing.T) {
	t.Skip("skipping for now, code being tested is currently unused")
	// TODO: import assets during init.
	// this will require figuring out how to set the right paths up for
	// referencing the code from its gx path
	testNFiles(initDirIndex, 2, t, "assets")
}

func testNFiles(fs []string, wantCnt int, t *testing.T, ftype string) {
	if len(fs) < wantCnt {
		t.Fatalf("expected %d %s. got %d", wantCnt, ftype, len(fs))
	}

	var wg sync.WaitGroup
	for _, f := range fs {
		wg.Add(1)
		// compare asset
		go func(f string) {
			defer wg.Done()
			testOneFile(f, t)
		}(f)
	}
	wg.Wait()
}

func testOneFile(f string, t *testing.T) {
	// load data from filesystem (git)
	vcsData, err := ioutil.ReadFile(f)
	if err != nil {
		t.Errorf("asset %s: could not read vcs file: %s", f, err)
		return
	}

	// load data from emdedded source
	embdData, err := Asset(f)
	if err != nil {
		t.Errorf("asset %s: could not read vcs file: %s", f, err)
		return
	}

	if !bytes.Equal(vcsData, embdData) {
		t.Errorf("asset %s: vcs and embedded data isnt equal", f)
		return
	}

	t.Logf("checked %s", f)
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"

	"github.com/goamz/goamz/aws"
	"github.com/goamz/goamz/sqs"
	"gopkg.in/alecthomas/kingpin.v1"
)

var (
	app       = kingpin.New("dead-letter-requeue", "Requeues messages from a SQS dead-letter queue to the active one.")
	queueName = app.Arg("queue-name", "Name of the SQS queue (e.g. prod-mgmt-website-data-www100-jimdo-com).").Required().String()
)

func main() {
	var awsAccessKey = os.Getenv("AWS_ACCESS_KEY_ID")
	if len(awsAccessKey) == 0 {
		log.Fatalf("Please set environment variable AWS_ACCESS_KEY_ID")
	}
	var awsSecretKey = os.Getenv("AWS_SECRET_ACCESS_KEY")
	if len(awsSecretKey) == 0 {
		log.Fatalf("Please set environment variable AWS_SECRET_ACCESS_KEY")
	}

	kingpin.MustParse(app.Parse(os.Args[1:]))

	activeQueueName := *queueName

	var deadLetterQueueName = activeQueueName + "_dead_letter"

	var auth = aws.Auth{
		AccessKey: awsAccessKey,
		SecretKey: awsSecretKey,
	}

	conn := sqs.New(auth, aws.EUWest)

	deadLetterQueue, err := conn.GetQueue(deadLetterQueueName)
	if err != nil {
		log.Fatal(err)
		return
	}

	activeQueue, err := conn.GetQueue(activeQueueName)
	if err != nil {
		log.Fatal(err)
		return
	}

	log.Printf("Looking for messages to requeue.")
	for {
		resp, err := deadLetterQueue.ReceiveMessageWithParameters(
			map[string]string{
				"WaitTimeSeconds":     "20",
				"MaxNumberOfMessages": "10",
				"VisibilityTimeout":   "20"})
		if err != nil {
			log.Fatal(err)
			return
		}

		messages := resp.Messages
		numberOfMessages := len(messages)
		if numberOfMessages == 0 {
			log.Printf("Requeuing messages done.")
			return
		} else {
			log.Printf("Moving %v message(s)...", numberOfMessages)
		}

		_, err = activeQueue.SendMessageBatch(messages)
		if err != nil {
			log.Fatal(err)
			return
		}

		_, err = deadLetterQueue.DeleteMessageBatch(messages)
		if err != nil {
			log.Fatal(err)
			return
		}
	}
}
<commit_msg>Use aws.EnvAuth()<commit_after>package main

import (
	"log"
	"os"

	"github.com/goamz/goamz/aws"
	"github.com/goamz/goamz/sqs"
	"gopkg.in/alecthomas/kingpin.v1"
)

var (
	app       = kingpin.New("dead-letter-requeue", "Requeues messages from a SQS dead-letter queue to the active one.")
	queueName = app.Arg("queue-name", "Name of the SQS queue (e.g. prod-mgmt-website-data-www100-jimdo-com).").Required().String()
)

func main() {
	kingpin.MustParse(app.Parse(os.Args[1:]))

	activeQueueName := *queueName

	var deadLetterQueueName = activeQueueName + "_dead_letter"

	auth, err := aws.EnvAuth()
	if err != nil {
		log.Fatal(err)
		return
	}

	conn := sqs.New(auth, aws.EUWest)

	deadLetterQueue, err := conn.GetQueue(deadLetterQueueName)
	if err != nil {
		log.Fatal(err)
		return
	}

	activeQueue, err := conn.GetQueue(activeQueueName)
	if err != nil {
		log.Fatal(err)
		return
	}

	log.Printf("Looking for messages to requeue.")
	for {
		resp, err := deadLetterQueue.ReceiveMessageWithParameters(
			map[string]string{
				"WaitTimeSeconds":     "20",
				"MaxNumberOfMessages": "10",
				"VisibilityTimeout":   "20"})
		if err != nil {
			log.Fatal(err)
			return
		}

		messages := resp.Messages
		numberOfMessages := len(messages)
		if numberOfMessages == 0 {
			log.Printf("Requeuing messages done.")
			return
		} else {
			log.Printf("Moving %v message(s)...", numberOfMessages)
		}

		_, err = activeQueue.SendMessageBatch(messages)
		if err != nil {
			log.Fatal(err)
			return
		}

		_, err = deadLetterQueue.DeleteMessageBatch(messages)
		if err != nil {
			log.Fatal(err)
			return
		}
	}
}
<|endoftext|>


<commit_before>// EXAMPLE FROM: https://github.com/GoogleCloudPlatform/appengine-angular-gotodos
//
// Copyright 2013 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

// gotodos is an App Engine JSON backend for managing a todo list.
//
// It supports the following commands:
//
// - Create a new todo
// POST /todos
// > {"text": "do this"}
// < {"id": 1, "text": "do this", "created": 1356724843.0, "done": false}
//
// - Update an existing todo
// POST /todos
// > {"id": 1, "text": "do this", "created": 1356724843.0, "done": true}
// < {"id": 1, "text": "do this", "created": 1356724843.0, "done": true}
//
// - List existing todos:
// GET /todos
// >
// < [{"id": 1, "text": "do this", "created": 1356724843.0, "done": true},
//    {"id": 2, "text": "do that", "created": 1356724849.0, "done": false}]
//
// - Delete 'done' todos:
// DELETE /todos
// >
// <

package controllers

import (
	"encoding/json"
	"io"
	"log"
	"net/http"
	"regexp"

	"models"

	"appengine/datastore"
	"appengine/taskqueue"
	"appengine/urlfetch"

	"github.com/PuerkitoBio/goquery"
	"github.com/astaxie/beegae"
)

type IOSAppController struct {
	beegae.Controller
}

func (this *IOSAppController) Get() {
	iosapps := []models.IOSApp{}
	_, err := datastore.NewQuery("IOSApp").Order("-UpdatedAt").GetAll(this.AppEngineCtx, &iosapps)
	if err != nil {
		this.Data["json"] = err
		return
	}
	listDataSet := map[string]interface{}{"items": iosapps}
	this.Data["json"] = listDataSet
}

func (this *IOSAppController) Post() {
	iosapp, err := decodeIOSApp(this.Ctx.Input.Request.Body)
	if err != nil {
		log.Println("decode err")
		this.Data["json"] = err
		return
	}
	i, err := iosapp.Create(this.AppEngineCtx)
	if err != nil {
		this.Data["json"] = err
	} else {
		this.Data["json"] = &i
	}
}

func (this *IOSAppController) GetEntity() {
	keyName := this.Ctx.Input.Param(":key_name")
	key := datastore.NewKey(this.AppEngineCtx, "IOSApp", keyName, 0, nil)
	var iosapp models.IOSApp
	err := datastore.Get(this.AppEngineCtx, key, &iosapp)
	if err != nil {
		this.Data["json"] = err
		return
	} else {
		this.Data["json"] = &iosapp
	}
}
func (this *IOSAppController) UpdateEntity() {
	keyName := this.Ctx.Input.Param(":key_name")
	key := datastore.NewKey(this.AppEngineCtx, "IOSApp", keyName, 0, nil)
	var iosapp models.IOSApp
	err := datastore.Get(this.AppEngineCtx, key, &iosapp)
	if err != nil {
		this.Data["json"] = err
		return
	}
	err = json.NewDecoder(this.Ctx.Input.Request.Body).Decode(&iosapp)
	if err != nil {
		this.Data["json"] = err
		return
	}
	i, err := iosapp.Update(this.AppEngineCtx)
	if err != nil {
		this.Data["json"] = err
	} else {
		this.Data["json"] = &i
	}
}

func (this *IOSAppController) DeleteEntity() {
	keyName := this.Ctx.Input.Param(":key_name")
	key := datastore.NewKey(this.AppEngineCtx, "IOSApp", keyName, 0, nil)
	err := datastore.Delete(this.AppEngineCtx, key)
	if err == nil {
		this.Data["json"] = nil
	} else {
		this.Data["json"] = err
	}
}

func (this *IOSAppController) GetAppReview() {
	keyName := this.Ctx.Input.Param(":key_name")
	key := datastore.NewKey(this.AppEngineCtx, "IOSApp", keyName, 0, nil)
	var iosapp models.IOSApp
	err := datastore.Get(this.AppEngineCtx, key, &iosapp)
	if err != nil {
		this.Data["json"] = err
		return
	}
	client := urlfetch.Client(this.AppEngineCtx)
	req, err := http.NewRequest("GET", "https://itunes.apple.com/WebObjects/MZStore.woa/wa/viewContentsUserReviews?pageNumber=0&sortOrdering=4&onlyLatestVersion=false&type=Purple+Software&id="+keyName, nil)
	req.Header.Add("X-Apple-Store-Front", iosapp.Region)
	req.Header.Add("User-Agent", "iTunes/9.2 (Macintosh; U; Mac OS X 10.6)")
	resp, err := client.Do(req)
	if err != nil {
		this.Data["json"] = err
		return
	}
	defer resp.Body.Close()
	doc, _ := goquery.NewDocumentFromResponse(resp)
	doc.Find("Document View VBoxView View MatrixView VBoxView:nth-child(1) VBoxView VBoxView VBoxView").Each(func(_ int, s *goquery.Selection) {
		title_node := s.Find("HBoxView>TextView>SetFontStyle>b").First()
		title := title_node.Text()
		if title != "" {
			reviewIDURL, idExists := s.Find("HBoxView VBoxView GotoURL").First().Attr("url")
			if idExists {
				regex_str := "([0-9]{4,}$)"
				re, err := regexp.Compile(regex_str)
				if err != nil {
					panic(err)
				}
				reviewID := re.FindString(reviewIDURL)
				var content string
				if len(reviewID) > 4 {
					num := 0
					log.Println(title)
					log.Println(reviewID)
					s.Find("TextView SetFontStyle").Each(func(_ int, sc *goquery.Selection) {
						num = num + 1
						if num == 4 {
							content = sc.Text()
						}
					})
					var appreview models.AppReview
					appreview.AppID = keyName
					appreview.ReviewID = reviewID
					appreview.Title = title
					appreview.Content = content
					_, err = appreview.Create(this.AppEngineCtx)
					if err != nil {
						this.Data["json"] = err
						return
					}
				}

			}
		}

	})
}

func (this *IOSAppController) GetReviews() {
	iosapps := []models.IOSApp{}
	_, err := datastore.NewQuery("IOSApp").Order("-UpdatedAt").GetAll(this.AppEngineCtx, &iosapps)
	if err != nil {
		this.Data["json"] = err
		return
	}
	for i := 0; i < len(iosapps); i++ {
		log.Println(iosapps[i].AppID)
		t := taskqueue.NewPOSTTask("/admin/task/iosapp/getappreview/"+iosapps[i].AppID, nil)
		if _, err := taskqueue.Add(this.AppEngineCtx, t, ""); err != nil {
			this.Data["json"] = err
			return
		}
	}
	listDataSet := map[string]interface{}{"items": iosapps}
	this.Data["json"] = listDataSet
	// TODO set taskque to get app reviews
}

func (this *IOSAppController) Render() error {
	if _, ok := this.Data["json"].(error); ok {
		this.AppEngineCtx.Errorf("iosapp error: %v", this.Data["json"])
	}
	this.ServeJson()
	return nil
}

func decodeIOSApp(r io.ReadCloser) (*models.IOSApp, error) {
	defer r.Close()
	var iosapp models.IOSApp
	err := json.NewDecoder(r).Decode(&iosapp)
	return &iosapp, err
}
<commit_msg>fix regex compile position<commit_after>// EXAMPLE FROM: https://github.com/GoogleCloudPlatform/appengine-angular-gotodos
//
// Copyright 2013 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

// gotodos is an App Engine JSON backend for managing a todo list.
//
// It supports the following commands:
//
// - Create a new todo
// POST /todos
// > {"text": "do this"}
// < {"id": 1, "text": "do this", "created": 1356724843.0, "done": false}
//
// - Update an existing todo
// POST /todos
// > {"id": 1, "text": "do this", "created": 1356724843.0, "done": true}
// < {"id": 1, "text": "do this", "created": 1356724843.0, "done": true}
//
// - List existing todos:
// GET /todos
// >
// < [{"id": 1, "text": "do this", "created": 1356724843.0, "done": true},
//    {"id": 2, "text": "do that", "created": 1356724849.0, "done": false}]
//
// - Delete 'done' todos:
// DELETE /todos
// >
// <

package controllers

import (
	"encoding/json"
	"io"
	"log"
	"net/http"
	"regexp"

	"models"

	"appengine/datastore"
	"appengine/taskqueue"
	"appengine/urlfetch"

	"github.com/PuerkitoBio/goquery"
	"github.com/astaxie/beegae"
)

type IOSAppController struct {
	beegae.Controller
}

func (this *IOSAppController) Get() {
	iosapps := []models.IOSApp{}
	_, err := datastore.NewQuery("IOSApp").Order("-UpdatedAt").GetAll(this.AppEngineCtx, &iosapps)
	if err != nil {
		this.Data["json"] = err
		return
	}
	listDataSet := map[string]interface{}{"items": iosapps}
	this.Data["json"] = listDataSet
}

func (this *IOSAppController) Post() {
	iosapp, err := decodeIOSApp(this.Ctx.Input.Request.Body)
	if err != nil {
		log.Println("decode err")
		this.Data["json"] = err
		return
	}
	i, err := iosapp.Create(this.AppEngineCtx)
	if err != nil {
		this.Data["json"] = err
	} else {
		this.Data["json"] = &i
	}
}

func (this *IOSAppController) GetEntity() {
	keyName := this.Ctx.Input.Param(":key_name")
	key := datastore.NewKey(this.AppEngineCtx, "IOSApp", keyName, 0, nil)
	var iosapp models.IOSApp
	err := datastore.Get(this.AppEngineCtx, key, &iosapp)
	if err != nil {
		this.Data["json"] = err
		return
	} else {
		this.Data["json"] = &iosapp
	}
}
func (this *IOSAppController) UpdateEntity() {
	keyName := this.Ctx.Input.Param(":key_name")
	key := datastore.NewKey(this.AppEngineCtx, "IOSApp", keyName, 0, nil)
	var iosapp models.IOSApp
	err := datastore.Get(this.AppEngineCtx, key, &iosapp)
	if err != nil {
		this.Data["json"] = err
		return
	}
	err = json.NewDecoder(this.Ctx.Input.Request.Body).Decode(&iosapp)
	if err != nil {
		this.Data["json"] = err
		return
	}
	i, err := iosapp.Update(this.AppEngineCtx)
	if err != nil {
		this.Data["json"] = err
	} else {
		this.Data["json"] = &i
	}
}

func (this *IOSAppController) DeleteEntity() {
	keyName := this.Ctx.Input.Param(":key_name")
	key := datastore.NewKey(this.AppEngineCtx, "IOSApp", keyName, 0, nil)
	err := datastore.Delete(this.AppEngineCtx, key)
	if err == nil {
		this.Data["json"] = nil
	} else {
		this.Data["json"] = err
	}
}

func (this *IOSAppController) GetAppReview() {
	keyName := this.Ctx.Input.Param(":key_name")
	key := datastore.NewKey(this.AppEngineCtx, "IOSApp", keyName, 0, nil)
	var iosapp models.IOSApp
	err := datastore.Get(this.AppEngineCtx, key, &iosapp)
	if err != nil {
		this.Data["json"] = err
		return
	}
	client := urlfetch.Client(this.AppEngineCtx)
	req, err := http.NewRequest("GET", "https://itunes.apple.com/WebObjects/MZStore.woa/wa/viewContentsUserReviews?pageNumber=0&sortOrdering=4&onlyLatestVersion=false&type=Purple+Software&id="+keyName, nil)
	req.Header.Add("X-Apple-Store-Front", iosapp.Region)
	req.Header.Add("User-Agent", "iTunes/9.2 (Macintosh; U; Mac OS X 10.6)")
	resp, err := client.Do(req)
	if err != nil {
		this.Data["json"] = err
		return
	}
	defer resp.Body.Close()
	regex_str := "([0-9]{4,}$)"
	re, err := regexp.Compile(regex_str)
	if err != nil {
		panic(err)
	}
	doc, _ := goquery.NewDocumentFromResponse(resp)
	doc.Find("Document View VBoxView View MatrixView VBoxView:nth-child(1) VBoxView VBoxView VBoxView").Each(func(_ int, s *goquery.Selection) {
		title_node := s.Find("HBoxView>TextView>SetFontStyle>b").First()
		title := title_node.Text()
		if title != "" {
			reviewIDURL, idExists := s.Find("HBoxView VBoxView GotoURL").First().Attr("url")
			if idExists {

				reviewID := re.FindString(reviewIDURL)
				var content string
				if len(reviewID) > 4 {
					num := 0
					log.Println(title)
					log.Println(reviewID)
					s.Find("TextView SetFontStyle").Each(func(_ int, sc *goquery.Selection) {
						num = num + 1
						if num == 4 {
							content = sc.Text()
						}
					})
					var appreview models.AppReview
					appreview.AppID = keyName
					appreview.ReviewID = reviewID
					appreview.Title = title
					appreview.Content = content
					_, err = appreview.Create(this.AppEngineCtx)
					if err != nil {
						this.Data["json"] = err
						return
					}
				}

			}
		}

	})
}

func (this *IOSAppController) GetReviews() {
	iosapps := []models.IOSApp{}
	_, err := datastore.NewQuery("IOSApp").Order("-UpdatedAt").GetAll(this.AppEngineCtx, &iosapps)
	if err != nil {
		this.Data["json"] = err
		return
	}
	for i := 0; i < len(iosapps); i++ {
		log.Println(iosapps[i].AppID)
		t := taskqueue.NewPOSTTask("/admin/task/iosapp/getappreview/"+iosapps[i].AppID, nil)
		if _, err := taskqueue.Add(this.AppEngineCtx, t, ""); err != nil {
			this.Data["json"] = err
			return
		}
	}
	listDataSet := map[string]interface{}{"items": iosapps}
	this.Data["json"] = listDataSet
	// TODO set taskque to get app reviews
}

func (this *IOSAppController) Render() error {
	if _, ok := this.Data["json"].(error); ok {
		this.AppEngineCtx.Errorf("iosapp error: %v", this.Data["json"])
	}
	this.ServeJson()
	return nil
}

func decodeIOSApp(r io.ReadCloser) (*models.IOSApp, error) {
	defer r.Close()
	var iosapp models.IOSApp
	err := json.NewDecoder(r).Decode(&iosapp)
	return &iosapp, err
}
<|endoftext|>


<commit_before>package runtimehandlerhooks

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/cri-o/cri-o/internal/log"

	"github.com/cri-o/cri-o/internal/lib/sandbox"
	"github.com/cri-o/cri-o/internal/oci"

	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
)

const (
	// HighPerformance contains the high-performance runtime handler name
	HighPerformance = "high-performance"
)

const (
	annotationCPULoadBalancing = "cpu-load-balancing.crio.io"
	schedDomainDir             = "/proc/sys/kernel/sched_domain"
)

// HighPerformanceHooks used to run additional hooks that will configure a system for the latency sensitive workloads
type HighPerformanceHooks struct{}

func (h *HighPerformanceHooks) PreStart(ctx context.Context, c *oci.Container, s *sandbox.Sandbox) error {
	log.Infof(ctx, "Run %q runtime handler pre-start hook for the container %q", HighPerformance, c.ID())
	// disable the CPU load balancing for the container CPUs
	if shouldCPULoadBalancingBeDisabled(s.Annotations()) {
		if err := setCPUSLoadBalancing(c, false, schedDomainDir); err != nil {
			return err
		}
	}

	return nil
}

func (h *HighPerformanceHooks) PreStop(ctx context.Context, c *oci.Container, s *sandbox.Sandbox) error {
	log.Infof(ctx, "Run %q runtime handler pre-stop hook for the container %q", HighPerformance, c.ID())
	// enable the CPU load balancing for the container CPUs
	if shouldCPULoadBalancingBeDisabled(s.Annotations()) {
		if err := setCPUSLoadBalancing(c, true, schedDomainDir); err != nil {
			return err
		}
	}

	return nil
}

func shouldCPULoadBalancingBeDisabled(annotations fields.Set) bool {
	return annotations[annotationCPULoadBalancing] == "true"
}

func setCPUSLoadBalancing(c *oci.Container, enable bool, schedDomainDir string) error {
	if c.Spec().Linux == nil ||
		c.Spec().Linux.Resources == nil ||
		c.Spec().Linux.Resources.CPU == nil ||
		c.Spec().Linux.Resources.CPU.Cpus == "" {
		return fmt.Errorf("failed to find the container %q CPUs", c.ID())
	}

	cpus, err := cpuset.Parse(c.Spec().Linux.Resources.CPU.Cpus)
	if err != nil {
		return err
	}

	for _, cpu := range cpus.ToSlice() {
		cpuSchedDomainDir := fmt.Sprintf("%s/cpu%d", schedDomainDir, cpu)
		err := filepath.Walk(cpuSchedDomainDir, func(path string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}

			if path == cpuSchedDomainDir {
				return nil
			}

			if !strings.Contains(path, "flags") {
				return nil
			}

			content, err := ioutil.ReadFile(path)
			if err != nil {
				return err
			}

			flags, err := strconv.Atoi(strings.Trim(string(content), "\n"))
			if err != nil {
				return err
			}

			var newContent string
			if enable {
				newContent = strconv.Itoa(flags | 1)
			} else {
				// we should set the LSB to 0 to disable the load balancing for the specified CPU
				// in case of sched domain all flags can be represented by the binary number 111111111111111 that equals
				// to 32767 in the decimal form
				// see https://github.com/torvalds/linux/blob/0fe5f9ca223573167c4c4156903d751d2c8e160e/include/linux/sched/topology.h#L14
				// for more information regarding the sched domain flags
				newContent = strconv.Itoa(flags & 32766)
			}

			err = ioutil.WriteFile(path, []byte(newContent), 0o644)
			return err
		})
		if err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>setCPUSLoadBalancing: rm repeated call to c.Spec()<commit_after>package runtimehandlerhooks

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/cri-o/cri-o/internal/log"

	"github.com/cri-o/cri-o/internal/lib/sandbox"
	"github.com/cri-o/cri-o/internal/oci"

	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
)

const (
	// HighPerformance contains the high-performance runtime handler name
	HighPerformance = "high-performance"
)

const (
	annotationCPULoadBalancing = "cpu-load-balancing.crio.io"
	schedDomainDir             = "/proc/sys/kernel/sched_domain"
)

// HighPerformanceHooks used to run additional hooks that will configure a system for the latency sensitive workloads
type HighPerformanceHooks struct{}

func (h *HighPerformanceHooks) PreStart(ctx context.Context, c *oci.Container, s *sandbox.Sandbox) error {
	log.Infof(ctx, "Run %q runtime handler pre-start hook for the container %q", HighPerformance, c.ID())
	// disable the CPU load balancing for the container CPUs
	if shouldCPULoadBalancingBeDisabled(s.Annotations()) {
		if err := setCPUSLoadBalancing(c, false, schedDomainDir); err != nil {
			return err
		}
	}

	return nil
}

func (h *HighPerformanceHooks) PreStop(ctx context.Context, c *oci.Container, s *sandbox.Sandbox) error {
	log.Infof(ctx, "Run %q runtime handler pre-stop hook for the container %q", HighPerformance, c.ID())
	// enable the CPU load balancing for the container CPUs
	if shouldCPULoadBalancingBeDisabled(s.Annotations()) {
		if err := setCPUSLoadBalancing(c, true, schedDomainDir); err != nil {
			return err
		}
	}

	return nil
}

func shouldCPULoadBalancingBeDisabled(annotations fields.Set) bool {
	return annotations[annotationCPULoadBalancing] == "true"
}

func setCPUSLoadBalancing(c *oci.Container, enable bool, schedDomainDir string) error {
	lspec := c.Spec().Linux
	if lspec == nil ||
		lspec.Resources == nil ||
		lspec.Resources.CPU == nil ||
		lspec.Resources.CPU.Cpus == "" {
		return fmt.Errorf("failed to find the container %q CPUs", c.ID())
	}

	cpus, err := cpuset.Parse(lspec.Resources.CPU.Cpus)
	if err != nil {
		return err
	}

	for _, cpu := range cpus.ToSlice() {
		cpuSchedDomainDir := fmt.Sprintf("%s/cpu%d", schedDomainDir, cpu)
		err := filepath.Walk(cpuSchedDomainDir, func(path string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}

			if path == cpuSchedDomainDir {
				return nil
			}

			if !strings.Contains(path, "flags") {
				return nil
			}

			content, err := ioutil.ReadFile(path)
			if err != nil {
				return err
			}

			flags, err := strconv.Atoi(strings.Trim(string(content), "\n"))
			if err != nil {
				return err
			}

			var newContent string
			if enable {
				newContent = strconv.Itoa(flags | 1)
			} else {
				// we should set the LSB to 0 to disable the load balancing for the specified CPU
				// in case of sched domain all flags can be represented by the binary number 111111111111111 that equals
				// to 32767 in the decimal form
				// see https://github.com/torvalds/linux/blob/0fe5f9ca223573167c4c4156903d751d2c8e160e/include/linux/sched/topology.h#L14
				// for more information regarding the sched domain flags
				newContent = strconv.Itoa(flags & 32766)
			}

			err = ioutil.WriteFile(path, []byte(newContent), 0o644)
			return err
		})
		if err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>