<commit_before>/*
Copyright 2020 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cluster

import (
	"encoding/json"
	"fmt"
	"os/exec"
	"path"
	"path/filepath"
	"time"

	"github.com/docker/machine/libmachine"
	"github.com/docker/machine/libmachine/drivers"
	"github.com/docker/machine/libmachine/engine"
	"github.com/docker/machine/libmachine/host"
	"github.com/golang/glog"
	"github.com/juju/mutex"
	"github.com/pkg/errors"
	"github.com/spf13/viper"
	"k8s.io/minikube/pkg/minikube/command"
	"k8s.io/minikube/pkg/minikube/config"
	"k8s.io/minikube/pkg/minikube/constants"
	"k8s.io/minikube/pkg/minikube/driver"
	"k8s.io/minikube/pkg/minikube/localpath"
	"k8s.io/minikube/pkg/minikube/out"
	"k8s.io/minikube/pkg/minikube/registry"
	"k8s.io/minikube/pkg/minikube/sshutil"
	"k8s.io/minikube/pkg/minikube/vmpath"
	"k8s.io/minikube/pkg/util/lock"
)

var (
	// requiredDirectories are directories to create on the host during setup
	requiredDirectories = []string{
		vmpath.GuestAddonsDir,
		vmpath.GuestManifestsDir,
		vmpath.GuestEphemeralDir,
		vmpath.GuestPersistentDir,
		vmpath.GuestKubernetesCertsDir,
		path.Join(vmpath.GuestPersistentDir, "images"),
		path.Join(vmpath.GuestPersistentDir, "binaries"),
		vmpath.GuestGvisorDir,
		vmpath.GuestCertAuthDir,
		vmpath.GuestCertStoreDir,
	}
)

// StartHost starts a host VM.
func StartHost(api libmachine.API, cfg config.MachineConfig) (*host.Host, error) {
	// Prevent machine-driver boot races, as well as our own certificate race
	releaser, err := acquireMachinesLock(cfg.Name)
	if err != nil {
		return nil, errors.Wrap(err, "boot lock")
	}
	start := time.Now()
	defer func() {
		glog.Infof("releasing machines lock for %q, held for %s", cfg.Name, time.Since(start))
		releaser.Release()
	}()

	exists, err := api.Exists(cfg.Name)
	if err != nil {
		return nil, errors.Wrapf(err, "exists: %s", cfg.Name)
	}
	if !exists {
		glog.Infof("Provisioning new machine with config: %+v", cfg)
		return createHost(api, cfg)
	}
	glog.Infoln("Skipping create...Using existing machine configuration")
	return fixHost(api, cfg)
}

func engineOptions(cfg config.MachineConfig) *engine.Options {
	o := engine.Options{
		Env:              cfg.DockerEnv,
		InsecureRegistry: append([]string{constants.DefaultServiceCIDR}, cfg.InsecureRegistry...),
		RegistryMirror:   cfg.RegistryMirror,
		ArbitraryFlags:   cfg.DockerOpt,
		InstallURL:       drivers.DefaultEngineInstallURL,
	}
	return &o
}

func createHost(api libmachine.API, cfg config.MachineConfig) (*host.Host, error) {
	glog.Infof("createHost starting for %q (driver=%q)", cfg.Name, cfg.VMDriver)
	start := time.Now()
	defer func() {
		glog.Infof("createHost completed in %s", time.Since(start))
	}()

	if cfg.VMDriver == driver.VMwareFusion && viper.GetBool(config.ShowDriverDeprecationNotification) {
		out.WarningT(`The vmwarefusion driver is deprecated and support for it will be removed in a future release.
			Please consider switching to the new vmware unified driver, which is intended to replace the vmwarefusion driver.
			See https://minikube.sigs.k8s.io/docs/reference/drivers/vmware/ for more information.
			To disable this message, run [minikube config set ShowDriverDeprecationNotification false]`)
	}
	showHostInfo(cfg)
	def := registry.Driver(cfg.VMDriver)
	if def.Empty() {
		return nil, fmt.Errorf("unsupported/missing driver: %s", cfg.VMDriver)
	}
	dd, err := def.Config(cfg)
	if err != nil {
		return nil, errors.Wrap(err, "config")
	}
	data, err := json.Marshal(dd)
	if err != nil {
		return nil, errors.Wrap(err, "marshal")
	}

	h, err := api.NewHost(cfg.VMDriver, data)
	if err != nil {
		return nil, errors.Wrap(err, "new host")
	}

	h.HostOptions.AuthOptions.CertDir = localpath.MiniPath()
	h.HostOptions.AuthOptions.StorePath = localpath.MiniPath()
	h.HostOptions.EngineOptions = engineOptions(cfg)

	cstart := time.Now()
	glog.Infof("libmachine.API.Create for %q (driver=%q)", cfg.Name, cfg.VMDriver)
	if err := api.Create(h); err != nil {
		// Wait for all the logs to reach the client
		time.Sleep(2 * time.Second)
		return nil, errors.Wrap(err, "create")
	}
	glog.Infof("libmachine.API.Create for %q took %s", cfg.Name, time.Since(cstart))

	if err := postStartSetup(h, cfg); err != nil {
		return h, errors.Wrap(err, "post-start")
	}

	if err := api.Save(h); err != nil {
		return nil, errors.Wrap(err, "save")
	}
	return h, nil
}

// postStart are functions shared between startHost and fixHost
func postStartSetup(h *host.Host, mc config.MachineConfig) error {
	glog.Infof("post-start starting for %q (driver=%q)", h.Name, h.DriverName)
	start := time.Now()
	defer func() {
		glog.Infof("post-start completed in %s", time.Since(start))
	}()

	if driver.IsMock(h.DriverName) {
		return nil
	}

	glog.Infof("creating required directories: %v", requiredDirectories)
	r, err := commandRunner(h)
	if err != nil {
		return errors.Wrap(err, "command runner")
	}

	args := append([]string{"mkdir", "-p"}, requiredDirectories...)
	if _, err := r.RunCmd(exec.Command("sudo", args...)); err != nil {
		return errors.Wrapf(err, "sudo mkdir (%s)", h.DriverName)
	}

	if driver.BareMetal(mc.VMDriver) {
		showLocalOsRelease()
	}
	if driver.IsVM(mc.VMDriver) {
		logRemoteOsRelease(h.Driver)
	}
	return syncLocalAssets(r)
}

// commandRunner returns best available command runner for this host
func commandRunner(h *host.Host) (command.Runner, error) {
	d := h.Driver.DriverName()
	glog.V(1).Infof("determining appropriate runner for %q", d)
	if driver.IsMock(d) {
		glog.Infof("returning FakeCommandRunner for %q driver", d)
		return &command.FakeCommandRunner{}, nil
	}

	if driver.BareMetal(h.Driver.DriverName()) {
		glog.Infof("returning ExecRunner for %q driver", d)
		return command.NewExecRunner(), nil
	}
	if driver.IsKIC(d) {
		glog.Infof("Returning KICRunner for %q driver", d)
		return command.NewKICRunner(h.Name, d), nil
	}

	glog.Infof("Creating SSH client and returning SSHRunner for %q driver", d)
	client, err := sshutil.NewSSHClient(h.Driver)
	if err != nil {
		return nil, errors.Wrap(err, "ssh client")
	}
	return command.NewSSHRunner(client), nil
}

// acquireMachinesLock protects against code that is not parallel-safe (libmachine, cert setup)
func acquireMachinesLock(name string) (mutex.Releaser, error) {
	spec := lock.PathMutexSpec(filepath.Join(localpath.MiniPath(), "machines"))
	// NOTE: Provisioning generally completes within 60 seconds
	spec.Timeout = 15 * time.Minute

	glog.Infof("acquiring machines lock for %s: %+v", name, spec)
	start := time.Now()
	r, err := mutex.Acquire(spec)
	if err == nil {
		glog.Infof("acquired machines lock for %q in %s", name, time.Since(start))
	}
	return r, err
}
<commit_msg>Fail fast if waiting for SSH to be available<commit_after>/*
Copyright 2020 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cluster

import (
	"encoding/json"
	"fmt"
	"os/exec"
	"path"
	"path/filepath"
	"time"

	"github.com/docker/machine/libmachine"
	"github.com/docker/machine/libmachine/drivers"
	"github.com/docker/machine/libmachine/engine"
	"github.com/docker/machine/libmachine/host"
	"github.com/golang/glog"
	"github.com/juju/mutex"
	"github.com/pkg/errors"
	"github.com/spf13/viper"
	"k8s.io/minikube/pkg/minikube/command"
	"k8s.io/minikube/pkg/minikube/config"
	"k8s.io/minikube/pkg/minikube/constants"
	"k8s.io/minikube/pkg/minikube/driver"
	"k8s.io/minikube/pkg/minikube/localpath"
	"k8s.io/minikube/pkg/minikube/out"
	"k8s.io/minikube/pkg/minikube/registry"
	"k8s.io/minikube/pkg/minikube/sshutil"
	"k8s.io/minikube/pkg/minikube/vmpath"
	"k8s.io/minikube/pkg/util/lock"
)

var (
	// requiredDirectories are directories to create on the host during setup
	requiredDirectories = []string{
		vmpath.GuestAddonsDir,
		vmpath.GuestManifestsDir,
		vmpath.GuestEphemeralDir,
		vmpath.GuestPersistentDir,
		vmpath.GuestKubernetesCertsDir,
		path.Join(vmpath.GuestPersistentDir, "images"),
		path.Join(vmpath.GuestPersistentDir, "binaries"),
		vmpath.GuestGvisorDir,
		vmpath.GuestCertAuthDir,
		vmpath.GuestCertStoreDir,
	}
)

// StartHost starts a host VM.
func StartHost(api libmachine.API, cfg config.MachineConfig) (*host.Host, error) {
	// Prevent machine-driver boot races, as well as our own certificate race
	releaser, err := acquireMachinesLock(cfg.Name)
	if err != nil {
		return nil, errors.Wrap(err, "boot lock")
	}
	start := time.Now()
	defer func() {
		glog.Infof("releasing machines lock for %q, held for %s", cfg.Name, time.Since(start))
		releaser.Release()
	}()

	exists, err := api.Exists(cfg.Name)
	if err != nil {
		return nil, errors.Wrapf(err, "exists: %s", cfg.Name)
	}
	if !exists {
		glog.Infof("Provisioning new machine with config: %+v", cfg)
		return createHost(api, cfg)
	}
	glog.Infoln("Skipping create...Using existing machine configuration")
	return fixHost(api, cfg)
}

func engineOptions(cfg config.MachineConfig) *engine.Options {
	o := engine.Options{
		Env:              cfg.DockerEnv,
		InsecureRegistry: append([]string{constants.DefaultServiceCIDR}, cfg.InsecureRegistry...),
		RegistryMirror:   cfg.RegistryMirror,
		ArbitraryFlags:   cfg.DockerOpt,
		InstallURL:       drivers.DefaultEngineInstallURL,
	}
	return &o
}

func createHost(api libmachine.API, cfg config.MachineConfig) (*host.Host, error) {
	glog.Infof("createHost starting for %q (driver=%q)", cfg.Name, cfg.VMDriver)
	start := time.Now()
	defer func() {
		glog.Infof("createHost completed in %s", time.Since(start))
	}()

	if cfg.VMDriver == driver.VMwareFusion && viper.GetBool(config.ShowDriverDeprecationNotification) {
		out.WarningT(`The vmwarefusion driver is deprecated and support for it will be removed in a future release.
			Please consider switching to the new vmware unified driver, which is intended to replace the vmwarefusion driver.
			See https://minikube.sigs.k8s.io/docs/reference/drivers/vmware/ for more information.
			To disable this message, run [minikube config set ShowDriverDeprecationNotification false]`)
	}
	showHostInfo(cfg)
	def := registry.Driver(cfg.VMDriver)
	if def.Empty() {
		return nil, fmt.Errorf("unsupported/missing driver: %s", cfg.VMDriver)
	}
	dd, err := def.Config(cfg)
	if err != nil {
		return nil, errors.Wrap(err, "config")
	}
	data, err := json.Marshal(dd)
	if err != nil {
		return nil, errors.Wrap(err, "marshal")
	}

	h, err := api.NewHost(cfg.VMDriver, data)
	if err != nil {
		return nil, errors.Wrap(err, "new host")
	}

	h.HostOptions.AuthOptions.CertDir = localpath.MiniPath()
	h.HostOptions.AuthOptions.StorePath = localpath.MiniPath()
	h.HostOptions.EngineOptions = engineOptions(cfg)

	cstart := time.Now()
	glog.Infof("libmachine.API.Create for %q (driver=%q)", cfg.Name, cfg.VMDriver)
	// Allow two minutes to create host before failing fast
	if err := timedCreateHost(h, api, 2*time.Minute); err != nil {
		return nil, errors.Wrap(err, "creating host")
	}
	glog.Infof("libmachine.API.Create for %q took %s", cfg.Name, time.Since(cstart))

	if err := postStartSetup(h, cfg); err != nil {
		return h, errors.Wrap(err, "post-start")
	}

	if err := api.Save(h); err != nil {
		return nil, errors.Wrap(err, "save")
	}
	return h, nil
}

func timedCreateHost(h *host.Host, api libmachine.API, t time.Duration) error {
	timeout := make(chan bool, 1)
	go func() {
		time.Sleep(t)
		timeout <- true
	}()

	createFinished := make(chan bool, 1)
	var err error
	go func() {
		err = api.Create(h)
		createFinished <- true
	}()

	select {
	case <-createFinished:
		if err != nil {
			time.Sleep(2 * time.Second)
			return errors.Wrap(err, "create")
		}
		return nil
	case <-timeout:
		return fmt.Errorf("create host timed out in %f seconds", t.Seconds())
	}
}

// postStart are functions shared between startHost and fixHost
func postStartSetup(h *host.Host, mc config.MachineConfig) error {
	glog.Infof("post-start starting for %q (driver=%q)", h.Name, h.DriverName)
	start := time.Now()
	defer func() {
		glog.Infof("post-start completed in %s", time.Since(start))
	}()

	if driver.IsMock(h.DriverName) {
		return nil
	}

	glog.Infof("creating required directories: %v", requiredDirectories)
	r, err := commandRunner(h)
	if err != nil {
		return errors.Wrap(err, "command runner")
	}

	args := append([]string{"mkdir", "-p"}, requiredDirectories...)
	if _, err := r.RunCmd(exec.Command("sudo", args...)); err != nil {
		return errors.Wrapf(err, "sudo mkdir (%s)", h.DriverName)
	}

	if driver.BareMetal(mc.VMDriver) {
		showLocalOsRelease()
	}
	if driver.IsVM(mc.VMDriver) {
		logRemoteOsRelease(h.Driver)
	}
	return syncLocalAssets(r)
}

// commandRunner returns best available command runner for this host
func commandRunner(h *host.Host) (command.Runner, error) {
	d := h.Driver.DriverName()
	glog.V(1).Infof("determining appropriate runner for %q", d)
	if driver.IsMock(d) {
		glog.Infof("returning FakeCommandRunner for %q driver", d)
		return &command.FakeCommandRunner{}, nil
	}

	if driver.BareMetal(h.Driver.DriverName()) {
		glog.Infof("returning ExecRunner for %q driver", d)
		return command.NewExecRunner(), nil
	}
	if driver.IsKIC(d) {
		glog.Infof("Returning KICRunner for %q driver", d)
		return command.NewKICRunner(h.Name, d), nil
	}

	glog.Infof("Creating SSH client and returning SSHRunner for %q driver", d)
	client, err := sshutil.NewSSHClient(h.Driver)
	if err != nil {
		return nil, errors.Wrap(err, "ssh client")
	}
	return command.NewSSHRunner(client), nil
}

// acquireMachinesLock protects against code that is not parallel-safe (libmachine, cert setup)
func acquireMachinesLock(name string) (mutex.Releaser, error) {
	spec := lock.PathMutexSpec(filepath.Join(localpath.MiniPath(), "machines"))
	// NOTE: Provisioning generally completes within 60 seconds
	spec.Timeout = 15 * time.Minute

	glog.Infof("acquiring machines lock for %s: %+v", name, spec)
	start := time.Now()
	r, err := mutex.Acquire(spec)
	if err == nil {
		glog.Infof("acquired machines lock for %q in %s", name, time.Since(start))
	}
	return r, err
}
<|endoftext|>


<commit_before>package output

import (
	"errors"
	"fmt"
	"strings"
)

var requiredFields = [2]string{"field", "metric"}
var varTemplate = "${%s}"

type Template struct {
	template string
}

// Create string from template with input parameters
func (template *Template) Process(field string, metric string, templateVars map[string]string) (error, string) {

	finalString := template.template

	if templateVars == nil {
		templateVars = make(map[string]string)
	}

	templateVars["field"] = field
	templateVars["metric"] = metric

	for k, v:= range templateVars {
		replaceString := fmt.Sprintf(varTemplate, k)

		if !strings.Contains(finalString, replaceString) {
			return errors.New("field \"" + k + "\" not found in template " + template.template), ""
		}

		finalString = strings.ReplaceAll(finalString, replaceString, v)
	}

	return nil, finalString
}

func NewTempate(template string) (error, *Template) {
	var err error

	t := new(Template)

	if err = validateTemplate(template); err != nil {
		return err, nil
	}

	t.template = template

	return nil, t
}

func validateTemplate(inputString string) error {

	for _, field := range requiredFields {
		if !strings.Contains(inputString, "${"+field+"}") {
			return errors.New(fmt.Sprintf("\"${%s}\" not found in [%s]", field, inputString))
		}
	}

	return nil
}
<commit_msg>copy map in pkg/output/output_template.go:17<commit_after>package output

import (
	"errors"
	"fmt"
	"strings"
)

var requiredFields = [2]string{"field", "metric"}
var varTemplate = "${%s}"

type Template struct {
	template string
}

// Create string from template with input parameters
func (template *Template) Process(field string, metric string, templateVars map[string]string) (error, string) {

	tempVars := make(map[string]string)

	finalString := template.template

	if templateVars != nil {

		for k, v := range templateVars {
			tempVars[k] = v
		}
	}

	tempVars["field"] = field
	tempVars["metric"] = metric

	for k, v:= range tempVars {
		replaceString := fmt.Sprintf(varTemplate, k)

		if !strings.Contains(finalString, replaceString) {
			return errors.New("field \"" + k + "\" not found in template " + template.template), ""
		}

		finalString = strings.ReplaceAll(finalString, replaceString, v)
	}

	return nil, finalString
}

func NewTempate(template string) (error, *Template) {
	var err error

	t := new(Template)

	if err = validateTemplate(template); err != nil {
		return err, nil
	}

	t.template = template

	return nil, t
}

func validateTemplate(inputString string) error {

	for _, field := range requiredFields {
		if !strings.Contains(inputString, "${"+field+"}") {
			return errors.New(fmt.Sprintf("\"${%s}\" not found in [%s]", field, inputString))
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2016 The prometheus-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package prometheus

import (
	"fmt"
	"net/url"
	"path"

	"k8s.io/client-go/pkg/api/resource"
	"k8s.io/client-go/pkg/api/v1"
	"k8s.io/client-go/pkg/apis/apps/v1beta1"
	"k8s.io/client-go/pkg/util/intstr"

	"github.com/coreos/prometheus-operator/pkg/client/monitoring/v1alpha1"
)

func makeStatefulSet(p v1alpha1.Prometheus, old *v1beta1.StatefulSet) *v1beta1.StatefulSet {
	// TODO(fabxc): is this the right point to inject defaults?
	// Ideally we would do it before storing but that's currently not possible.
	// Potentially an update handler on first insertion.

	if p.Spec.BaseImage == "" {
		p.Spec.BaseImage = "quay.io/prometheus/prometheus"
	}
	if p.Spec.Version == "" {
		p.Spec.Version = "v1.4.0"
	}
	if p.Spec.Replicas < 1 {
		p.Spec.Replicas = 1
	}
	if p.Spec.Retention == "" {
		p.Spec.Retention = "24h"
	}

	if p.Spec.Resources.Requests == nil {
		p.Spec.Resources.Requests = v1.ResourceList{}
	}
	if _, ok := p.Spec.Resources.Requests[v1.ResourceMemory]; !ok {
		p.Spec.Resources.Requests[v1.ResourceMemory] = resource.MustParse("2Gi")
	}

	statefulset := &v1beta1.StatefulSet{
		ObjectMeta: v1.ObjectMeta{
			Name:        fmt.Sprintf("prometheus-%s", p.Name),
			Labels:      p.ObjectMeta.Labels,
			Annotations: p.ObjectMeta.Annotations,
		},
		Spec: makeStatefulSetSpec(p),
	}
	if vc := p.Spec.Storage; vc == nil {
		statefulset.Spec.Template.Spec.Volumes = append(statefulset.Spec.Template.Spec.Volumes, v1.Volume{
			Name: fmt.Sprintf("%s-db", p.Name),
			VolumeSource: v1.VolumeSource{
				EmptyDir: &v1.EmptyDirVolumeSource{},
			},
		})
	} else {
		pvc := v1.PersistentVolumeClaim{
			ObjectMeta: v1.ObjectMeta{
				Name: fmt.Sprintf("%s-db", p.Name),
			},
			Spec: v1.PersistentVolumeClaimSpec{
				AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
				Resources:   vc.Resources,
				Selector:    vc.Selector,
			},
		}
		if len(vc.Class) > 0 {
			pvc.ObjectMeta.Annotations = map[string]string{
				"volume.beta.kubernetes.io/storage-class": vc.Class,
			}
		}
		statefulset.Spec.VolumeClaimTemplates = append(statefulset.Spec.VolumeClaimTemplates, pvc)
	}

	if old != nil {
		statefulset.Annotations = old.Annotations
	}
	return statefulset
}

func makeEmptyConfig(name string) *v1.ConfigMap {
	return &v1.ConfigMap{
		ObjectMeta: v1.ObjectMeta{
			Name: fmt.Sprintf("%s", name),
		},
		Data: map[string]string{
			"prometheus.yaml": "",
		},
	}
}

func makeEmptyRules(name string) *v1.ConfigMap {
	return &v1.ConfigMap{
		ObjectMeta: v1.ObjectMeta{
			Name: fmt.Sprintf("%s-rules", name),
		},
	}
}

func makeStatefulSetService(p *v1alpha1.Prometheus) *v1.Service {
	svc := &v1.Service{
		ObjectMeta: v1.ObjectMeta{
			Name: "prometheus-operated",
		},
		Spec: v1.ServiceSpec{
			ClusterIP: "None",
			Ports: []v1.ServicePort{
				{
					Name:       "web",
					Port:       9090,
					TargetPort: intstr.FromString("web"),
				},
			},
			Selector: map[string]string{
				"app": "prometheus",
			},
		},
	}
	return svc
}

func makeStatefulSetSpec(p v1alpha1.Prometheus) v1beta1.StatefulSetSpec {
	// Prometheus may take quite long to shut down to checkpoint existing data.
	// Allow up to 10 minutes for clean termination.
	terminationGracePeriod := int64(600)

	// We attempt to specify decent storage tuning flags based on how much the
	// requested memory can fit. The user has to specify an appropriate buffering
	// in memory limits to catch increased memory usage during query bursts.
	// More info: https://prometheus.io/docs/operating/storage/.
	reqMem := p.Spec.Resources.Requests[v1.ResourceMemory]
	// 1024 byte is the fixed chunk size. With increasing number of chunks actually
	// in memory, overhead owed to their management, higher ingestion buffers, etc.
	// increases.
	// We are conservative for now an assume this to be 80% as the Kubernetes environment
	// generally has a very high time series churn.
	memChunks := reqMem.Value() / 1024 / 5

	promArgs := []string{
		"-storage.local.retention=" + p.Spec.Retention,
		"-storage.local.memory-chunks=" + fmt.Sprintf("%d", memChunks),
		"-storage.local.max-chunks-to-persist=" + fmt.Sprintf("%d", memChunks/2),
		"-storage.local.num-fingerprint-mutexes=4096",
		"-storage.local.path=/var/prometheus/data",
		"-config.file=/etc/prometheus/config/prometheus.yaml",
	}

	webRoutePrefix := ""
	if p.Spec.ExternalURL != "" {
		promArgs = append(promArgs, "-web.external-url="+p.Spec.ExternalURL)
		extUrl, err := url.Parse(p.Spec.ExternalURL)
		if err == nil {
			webRoutePrefix = extUrl.Path
		}
	}

	localReloadURL := &url.URL{
		Scheme: "http",
		Host:   "localhost:9090",
		Path:   path.Clean(webRoutePrefix + "/-/reload"),
	}

	return v1beta1.StatefulSetSpec{
		ServiceName: "prometheus",
		Replicas:    &p.Spec.Replicas,
		Template: v1.PodTemplateSpec{
			ObjectMeta: v1.ObjectMeta{
				Labels: map[string]string{
					"app":        "prometheus",
					"prometheus": p.Name,
				},
			},
			Spec: v1.PodSpec{
				Containers: []v1.Container{
					{
						Name:  "prometheus",
						Image: fmt.Sprintf("%s:%s", p.Spec.BaseImage, p.Spec.Version),
						Ports: []v1.ContainerPort{
							{
								Name:          "web",
								ContainerPort: 9090,
								Protocol:      v1.ProtocolTCP,
							},
						},
						Args: promArgs,
						VolumeMounts: []v1.VolumeMount{
							{
								Name:      "config",
								ReadOnly:  true,
								MountPath: "/etc/prometheus/config",
							},
							{
								Name:      "rules",
								ReadOnly:  true,
								MountPath: "/etc/prometheus/rules",
							},
							{
								Name:      fmt.Sprintf("%s-db", p.Name),
								MountPath: "/var/prometheus/data",
								SubPath:   subPathForStorage(p.Spec.Storage),
							},
						},
						ReadinessProbe: &v1.Probe{
							Handler: v1.Handler{
								HTTPGet: &v1.HTTPGetAction{
									Path: path.Clean(webRoutePrefix + "/status"),
									Port: intstr.FromString("web"),
								},
							},
							InitialDelaySeconds: 1,
							TimeoutSeconds:      3,
							PeriodSeconds:       5,
							// For larger servers, restoring a checkpoint on startup may take quite a bit of time.
							// Wait up to 5 minutes.
							FailureThreshold: 100,
						},
						Resources: p.Spec.Resources,
					}, {
						Name:  "config-reloader",
						Image: "jimmidyson/configmap-reload",
						Args: []string{
							fmt.Sprintf("-webhook-url=%s", localReloadURL),
							"-volume-dir=/etc/prometheus/config",
						},
						VolumeMounts: []v1.VolumeMount{
							{
								Name:      "config",
								ReadOnly:  true,
								MountPath: "/etc/prometheus/config",
							},
						},
						Resources: v1.ResourceRequirements{
							Limits: v1.ResourceList{
								v1.ResourceCPU:    resource.MustParse("5m"),
								v1.ResourceMemory: resource.MustParse("10Mi"),
							},
						},
					}, {
						Name:  "rules-reloader",
						Image: "jimmidyson/configmap-reload",
						Args: []string{
							fmt.Sprintf("-webhook-url=%s", localReloadURL),
							"-volume-dir=/etc/prometheus/rules/",
						},
						VolumeMounts: []v1.VolumeMount{
							{
								Name:      "rules",
								ReadOnly:  true,
								MountPath: "/etc/prometheus/rules",
							},
						},
					},
				},
				TerminationGracePeriodSeconds: &terminationGracePeriod,
				Volumes: []v1.Volume{
					{
						Name: "config",
						VolumeSource: v1.VolumeSource{
							ConfigMap: &v1.ConfigMapVolumeSource{
								LocalObjectReference: v1.LocalObjectReference{
									Name: p.Name,
								},
							},
						},
					},
					{
						Name: "rules",
						VolumeSource: v1.VolumeSource{
							ConfigMap: &v1.ConfigMapVolumeSource{
								LocalObjectReference: v1.LocalObjectReference{
									Name: fmt.Sprintf("%s-rules", p.Name),
								},
							},
						},
					},
				},
			},
		},
	}
}

func subPathForStorage(s *v1alpha1.StorageSpec) string {
	if s == nil {
		return ""
	}

	return "prometheus-db"
}
<commit_msg>prometheus: use single config reloader<commit_after>// Copyright 2016 The prometheus-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package prometheus

import (
	"fmt"
	"net/url"
	"path"

	"k8s.io/client-go/pkg/api/resource"
	"k8s.io/client-go/pkg/api/v1"
	"k8s.io/client-go/pkg/apis/apps/v1beta1"
	"k8s.io/client-go/pkg/util/intstr"

	"github.com/coreos/prometheus-operator/pkg/client/monitoring/v1alpha1"
)

func makeStatefulSet(p v1alpha1.Prometheus, old *v1beta1.StatefulSet) *v1beta1.StatefulSet {
	// TODO(fabxc): is this the right point to inject defaults?
	// Ideally we would do it before storing but that's currently not possible.
	// Potentially an update handler on first insertion.

	if p.Spec.BaseImage == "" {
		p.Spec.BaseImage = "quay.io/prometheus/prometheus"
	}
	if p.Spec.Version == "" {
		p.Spec.Version = "v1.4.0"
	}
	if p.Spec.Replicas < 1 {
		p.Spec.Replicas = 1
	}
	if p.Spec.Retention == "" {
		p.Spec.Retention = "24h"
	}

	if p.Spec.Resources.Requests == nil {
		p.Spec.Resources.Requests = v1.ResourceList{}
	}
	if _, ok := p.Spec.Resources.Requests[v1.ResourceMemory]; !ok {
		p.Spec.Resources.Requests[v1.ResourceMemory] = resource.MustParse("2Gi")
	}

	statefulset := &v1beta1.StatefulSet{
		ObjectMeta: v1.ObjectMeta{
			Name:        fmt.Sprintf("prometheus-%s", p.Name),
			Labels:      p.ObjectMeta.Labels,
			Annotations: p.ObjectMeta.Annotations,
		},
		Spec: makeStatefulSetSpec(p),
	}
	if vc := p.Spec.Storage; vc == nil {
		statefulset.Spec.Template.Spec.Volumes = append(statefulset.Spec.Template.Spec.Volumes, v1.Volume{
			Name: fmt.Sprintf("%s-db", p.Name),
			VolumeSource: v1.VolumeSource{
				EmptyDir: &v1.EmptyDirVolumeSource{},
			},
		})
	} else {
		pvc := v1.PersistentVolumeClaim{
			ObjectMeta: v1.ObjectMeta{
				Name: fmt.Sprintf("%s-db", p.Name),
			},
			Spec: v1.PersistentVolumeClaimSpec{
				AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
				Resources:   vc.Resources,
				Selector:    vc.Selector,
			},
		}
		if len(vc.Class) > 0 {
			pvc.ObjectMeta.Annotations = map[string]string{
				"volume.beta.kubernetes.io/storage-class": vc.Class,
			}
		}
		statefulset.Spec.VolumeClaimTemplates = append(statefulset.Spec.VolumeClaimTemplates, pvc)
	}

	if old != nil {
		statefulset.Annotations = old.Annotations
	}
	return statefulset
}

func makeEmptyConfig(name string) *v1.ConfigMap {
	return &v1.ConfigMap{
		ObjectMeta: v1.ObjectMeta{
			Name: fmt.Sprintf("%s", name),
		},
		Data: map[string]string{
			"prometheus.yaml": "",
		},
	}
}

func makeEmptyRules(name string) *v1.ConfigMap {
	return &v1.ConfigMap{
		ObjectMeta: v1.ObjectMeta{
			Name: fmt.Sprintf("%s-rules", name),
		},
	}
}

func makeStatefulSetService(p *v1alpha1.Prometheus) *v1.Service {
	svc := &v1.Service{
		ObjectMeta: v1.ObjectMeta{
			Name: "prometheus-operated",
		},
		Spec: v1.ServiceSpec{
			ClusterIP: "None",
			Ports: []v1.ServicePort{
				{
					Name:       "web",
					Port:       9090,
					TargetPort: intstr.FromString("web"),
				},
			},
			Selector: map[string]string{
				"app": "prometheus",
			},
		},
	}
	return svc
}

func makeStatefulSetSpec(p v1alpha1.Prometheus) v1beta1.StatefulSetSpec {
	// Prometheus may take quite long to shut down to checkpoint existing data.
	// Allow up to 10 minutes for clean termination.
	terminationGracePeriod := int64(600)

	// We attempt to specify decent storage tuning flags based on how much the
	// requested memory can fit. The user has to specify an appropriate buffering
	// in memory limits to catch increased memory usage during query bursts.
	// More info: https://prometheus.io/docs/operating/storage/.
	reqMem := p.Spec.Resources.Requests[v1.ResourceMemory]
	// 1024 byte is the fixed chunk size. With increasing number of chunks actually
	// in memory, overhead owed to their management, higher ingestion buffers, etc.
	// increases.
	// We are conservative for now an assume this to be 80% as the Kubernetes environment
	// generally has a very high time series churn.
	memChunks := reqMem.Value() / 1024 / 5

	promArgs := []string{
		"-storage.local.retention=" + p.Spec.Retention,
		"-storage.local.memory-chunks=" + fmt.Sprintf("%d", memChunks),
		"-storage.local.max-chunks-to-persist=" + fmt.Sprintf("%d", memChunks/2),
		"-storage.local.num-fingerprint-mutexes=4096",
		"-storage.local.path=/var/prometheus/data",
		"-config.file=/etc/prometheus/config/prometheus.yaml",
	}

	webRoutePrefix := ""
	if p.Spec.ExternalURL != "" {
		promArgs = append(promArgs, "-web.external-url="+p.Spec.ExternalURL)
		extUrl, err := url.Parse(p.Spec.ExternalURL)
		if err == nil {
			webRoutePrefix = extUrl.Path
		}
	}

	localReloadURL := &url.URL{
		Scheme: "http",
		Host:   "localhost:9090",
		Path:   path.Clean(webRoutePrefix + "/-/reload"),
	}

	return v1beta1.StatefulSetSpec{
		ServiceName: "prometheus",
		Replicas:    &p.Spec.Replicas,
		Template: v1.PodTemplateSpec{
			ObjectMeta: v1.ObjectMeta{
				Labels: map[string]string{
					"app":        "prometheus",
					"prometheus": p.Name,
				},
			},
			Spec: v1.PodSpec{
				Containers: []v1.Container{
					{
						Name:  "prometheus",
						Image: fmt.Sprintf("%s:%s", p.Spec.BaseImage, p.Spec.Version),
						Ports: []v1.ContainerPort{
							{
								Name:          "web",
								ContainerPort: 9090,
								Protocol:      v1.ProtocolTCP,
							},
						},
						Args: promArgs,
						VolumeMounts: []v1.VolumeMount{
							{
								Name:      "config",
								ReadOnly:  true,
								MountPath: "/etc/prometheus/config",
							},
							{
								Name:      "rules",
								ReadOnly:  true,
								MountPath: "/etc/prometheus/rules",
							},
							{
								Name:      fmt.Sprintf("%s-db", p.Name),
								MountPath: "/var/prometheus/data",
								SubPath:   subPathForStorage(p.Spec.Storage),
							},
						},
						ReadinessProbe: &v1.Probe{
							Handler: v1.Handler{
								HTTPGet: &v1.HTTPGetAction{
									Path: path.Clean(webRoutePrefix + "/status"),
									Port: intstr.FromString("web"),
								},
							},
							InitialDelaySeconds: 1,
							TimeoutSeconds:      3,
							PeriodSeconds:       5,
							// For larger servers, restoring a checkpoint on startup may take quite a bit of time.
							// Wait up to 5 minutes.
							FailureThreshold: 100,
						},
						Resources: p.Spec.Resources,
					}, {
						Name:  "config-reloader",
						Image: "jimmidyson/configmap-reload",
						Args: []string{
							fmt.Sprintf("-webhook-url=%s", localReloadURL),
							"-volume-dir=/etc/prometheus/config",
							"-volume-dir=/etc/prometheus/rules/",
						},
						VolumeMounts: []v1.VolumeMount{
							{
								Name:      "config",
								ReadOnly:  true,
								MountPath: "/etc/prometheus/config",
							},
							{
								Name:      "rules",
								ReadOnly:  true,
								MountPath: "/etc/prometheus/rules",
							},
						},
						Resources: v1.ResourceRequirements{
							Limits: v1.ResourceList{
								v1.ResourceCPU:    resource.MustParse("5m"),
								v1.ResourceMemory: resource.MustParse("10Mi"),
							},
						},
					},
				},
				TerminationGracePeriodSeconds: &terminationGracePeriod,
				Volumes: []v1.Volume{
					{
						Name: "config",
						VolumeSource: v1.VolumeSource{
							ConfigMap: &v1.ConfigMapVolumeSource{
								LocalObjectReference: v1.LocalObjectReference{
									Name: p.Name,
								},
							},
						},
					},
					{
						Name: "rules",
						VolumeSource: v1.VolumeSource{
							ConfigMap: &v1.ConfigMapVolumeSource{
								LocalObjectReference: v1.LocalObjectReference{
									Name: fmt.Sprintf("%s-rules", p.Name),
								},
							},
						},
					},
				},
			},
		},
	}
}

func subPathForStorage(s *v1alpha1.StorageSpec) string {
	if s == nil {
		return ""
	}

	return "prometheus-db"
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package runner

import (
	"context"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/bazel"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build/gcb"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build/kaniko"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build/local"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build/tag"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/color"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/config"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/deploy"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes"
	kubectx "github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes/context"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/v1alpha2"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/watch"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

const PollInterval = 1000 * time.Millisecond

// ErrorConfigurationChanged is a special error that's returned when the skaffold configuration was changed.
var ErrorConfigurationChanged = errors.New("configuration changed")

// SkaffoldRunner is responsible for running the skaffold build and deploy pipeline.
type SkaffoldRunner struct {
	build.Builder
	deploy.Deployer
	tag.Tagger

	opts         *config.SkaffoldOptions
	watchFactory watch.Factory
	builds       []build.Artifact
}

// NewForConfig returns a new SkaffoldRunner for a SkaffoldConfig
func NewForConfig(opts *config.SkaffoldOptions, cfg *config.SkaffoldConfig) (*SkaffoldRunner, error) {
	kubeContext, err := kubectx.CurrentContext()
	if err != nil {
		return nil, errors.Wrap(err, "getting current cluster context")
	}
	logrus.Infof("Using kubectl context: %s", kubeContext)

	tagger, err := getTagger(cfg.Build.TagPolicy, opts.CustomTag)
	if err != nil {
		return nil, errors.Wrap(err, "parsing skaffold tag config")
	}

	builder, err := getBuilder(&cfg.Build, kubeContext)
	if err != nil {
		return nil, errors.Wrap(err, "parsing skaffold build config")
	}

	deployer, err := getDeployer(&cfg.Deploy, kubeContext, opts.Namespace)
	if err != nil {
		return nil, errors.Wrap(err, "parsing skaffold deploy config")
	}

	deployer = deploy.WithLabels(deployer, opts, builder, deployer, tagger)
	builder, deployer = WithTimings(builder, deployer)
	if opts.Notification {
		deployer = WithNotification(deployer)
	}

	return &SkaffoldRunner{
		Builder:      builder,
		Deployer:     deployer,
		Tagger:       tagger,
		opts:         opts,
		watchFactory: watch.NewWatcher,
	}, nil
}

func getBuilder(cfg *v1alpha2.BuildConfig, kubeContext string) (build.Builder, error) {
	switch {
	case cfg.LocalBuild != nil:
		logrus.Debugf("Using builder: local")
		return local.NewBuilder(cfg.LocalBuild, kubeContext)

	case cfg.GoogleCloudBuild != nil:
		logrus.Debugf("Using builder: google cloud")
		return gcb.NewBuilder(cfg.GoogleCloudBuild), nil

	case cfg.KanikoBuild != nil:
		logrus.Debugf("Using builder: kaniko")
		return kaniko.NewBuilder(cfg.KanikoBuild), nil

	default:
		return nil, fmt.Errorf("Unknown builder for config %+v", cfg)
	}
}

func getDeployer(cfg *v1alpha2.DeployConfig, kubeContext string, namespace string) (deploy.Deployer, error) {
	switch {
	case cfg.KubectlDeploy != nil:
		// TODO(dgageot): this should be the folder containing skaffold.yaml. Should also be moved elsewhere.
		cwd, err := os.Getwd()
		if err != nil {
			return nil, errors.Wrap(err, "finding current directory")
		}
		return deploy.NewKubectlDeployer(cwd, cfg.KubectlDeploy, kubeContext, namespace), nil

	case cfg.HelmDeploy != nil:
		return deploy.NewHelmDeployer(cfg.HelmDeploy, kubeContext, namespace), nil

	case cfg.KustomizeDeploy != nil:
		return deploy.NewKustomizeDeployer(cfg.KustomizeDeploy, kubeContext, namespace), nil

	default:
		return nil, fmt.Errorf("Unknown deployer for config %+v", cfg)
	}
}

func getTagger(t v1alpha2.TagPolicy, customTag string) (tag.Tagger, error) {
	switch {
	case customTag != "":
		return &tag.CustomTag{
			Tag: customTag,
		}, nil

	case t.EnvTemplateTagger != nil:
		return tag.NewEnvTemplateTagger(t.EnvTemplateTagger.Template)

	case t.ShaTagger != nil:
		return &tag.ChecksumTagger{}, nil

	case t.GitTagger != nil:
		return &tag.GitCommit{}, nil

	case t.DateTimeTagger != nil:
		return tag.NewDateTimeTagger(t.DateTimeTagger.Format, t.DateTimeTagger.TimeZone), nil

	default:
		return nil, fmt.Errorf("Unknown tagger for strategy %+v", t)
	}
}

// Run builds artifacts and then deploys them.
func (r *SkaffoldRunner) Run(ctx context.Context, out io.Writer, artifacts []*v1alpha2.Artifact) error {
	bRes, err := r.Build(ctx, out, r.Tagger, artifacts)
	if err != nil {
		return errors.Wrap(err, "build step")
	}

	_, err = r.Deploy(ctx, out, bRes)
	if err != nil {
		return errors.Wrap(err, "deploy step")
	}

	return r.TailLogs(ctx, out, artifacts, bRes)
}

// TailLogs prints the logs for deployed artifacts.
func (r *SkaffoldRunner) TailLogs(ctx context.Context, out io.Writer, artifacts []*v1alpha2.Artifact, bRes []build.Artifact) error {
	if !r.opts.Tail {
		return nil
	}

	imageList := kubernetes.NewImageList()
	for _, b := range bRes {
		imageList.Add(b.Tag)
	}

	colorPicker := kubernetes.NewColorPicker(artifacts)
	logger := kubernetes.NewLogAggregator(out, imageList, colorPicker)
	if err := logger.Start(ctx); err != nil {
		return errors.Wrap(err, "starting logger")
	}

	<-ctx.Done()
	return nil
}

// Dev watches for changes and runs the skaffold build and deploy
// pipeline until interrrupted by the user.
func (r *SkaffoldRunner) Dev(ctx context.Context, out io.Writer, artifacts []*v1alpha2.Artifact) ([]build.Artifact, error) {
	imageList := kubernetes.NewImageList()
	colorPicker := kubernetes.NewColorPicker(artifacts)
	logger := kubernetes.NewLogAggregator(out, imageList, colorPicker)

	// Create watcher and register artifacts to build current state of files.
	changed := changes{}
	onChange := func() error {
		logger.Mute()

		var err error

		switch {
		case changed.needsReload:
			err = ErrorConfigurationChanged
		case len(changed.dirtyArtifacts) > 0:
			err = r.buildAndDeploy(ctx, out, changed.dirtyArtifacts, imageList)
		case changed.needsRedeploy:
			if _, err := r.Deploy(ctx, out, r.builds); err != nil {
				logrus.Warnln("Skipping Deploy due to error:", err)
			}
		}

		color.Default.Fprintln(out, "Watching for changes...")
		changed.reset()
		logger.Unmute()

		return err
	}

	watcher := r.watchFactory()

	// Watch artifacts
	for i := range artifacts {
		artifact := artifacts[i]

		if !r.shouldWatch(artifact) {
			continue
		}

		if err := watcher.Register(
			func() ([]string, error) { return dependenciesForArtifact(artifact) },
			func() { changed.Add(artifact) },
		); err != nil {
			return nil, errors.Wrapf(err, "watching files for artifact %s", artifact.ImageName)
		}
	}

	// Watch deployment configuration
	if err := watcher.Register(
		func() ([]string, error) { return r.Dependencies() },
		func() { changed.needsRedeploy = true },
	); err != nil {
		return nil, errors.Wrap(err, "watching files for deployer")
	}

	// Watch Skaffold configuration
	if err := watcher.Register(
		func() ([]string, error) { return []string{r.opts.ConfigurationFile}, nil },
		func() { changed.needsReload = true },
	); err != nil {
		return nil, errors.Wrapf(err, "watching skaffold configuration %s", r.opts.ConfigurationFile)
	}

	// First run
	if err := r.buildAndDeploy(ctx, out, artifacts, imageList); err != nil {
		return nil, errors.Wrap(err, "first run")
	}

	// Start logs
	if err := logger.Start(ctx); err != nil {
		return nil, errors.Wrap(err, "starting logger")
	}

	return nil, watcher.Run(ctx, PollInterval, onChange)
}

func (r *SkaffoldRunner) shouldWatch(artifact *v1alpha2.Artifact) bool {
	if len(r.opts.Watch) == 0 {
		return true
	}

	for _, watchExpression := range r.opts.Watch {
		if strings.Contains(artifact.ImageName, watchExpression) {
			return true
		}
	}

	return false
}

// buildAndDeploy builds a subset of the artifacts and deploys everything.
func (r *SkaffoldRunner) buildAndDeploy(ctx context.Context, out io.Writer, artifacts []*v1alpha2.Artifact, images *kubernetes.ImageList) error {
	firstRun := r.builds == nil

	bRes, err := r.Build(ctx, out, r.Tagger, artifacts)
	if err != nil {
		if firstRun {
			return errors.Wrap(err, "exiting dev mode because the first build failed")
		}

		logrus.Warnln("Skipping Deploy due to build error:", err)
		return nil
	}

	// Update which images are logged.
	for _, build := range bRes {
		images.Add(build.Tag)
	}

	// Make sure all artifacts are redeployed. Not only those that were just rebuilt.
	r.builds = mergeWithPreviousBuilds(bRes, r.builds)

	_, err = r.Deploy(ctx, out, r.builds)
	if err != nil {
		if firstRun {
			return errors.Wrap(err, "exiting dev mode because the first deploy failed")
		}

		logrus.Warnln("Skipping Deploy due to error:", err)
		return nil
	}

	return nil
}

func mergeWithPreviousBuilds(builds, previous []build.Artifact) []build.Artifact {
	updatedBuilds := map[string]bool{}
	for _, build := range builds {
		updatedBuilds[build.ImageName] = true
	}

	var merged []build.Artifact
	merged = append(merged, builds...)

	for _, b := range previous {
		if !updatedBuilds[b.ImageName] {
			merged = append(merged, b)
		}
	}

	return merged
}

func dependenciesForArtifact(a *v1alpha2.Artifact) ([]string, error) {
	var (
		paths []string
		err   error
	)

	switch {
	case a.DockerArtifact != nil:
		paths, err = docker.GetDependencies(a.Workspace, a.DockerArtifact)

	case a.BazelArtifact != nil:
		paths, err = bazel.GetDependencies(a.Workspace, a.BazelArtifact)

	default:
		return nil, fmt.Errorf("undefined artifact type: %+v", a.ArtifactType)
	}

	if err != nil {
		return nil, err
	}

	var p []string
	for _, path := range paths {
		p = append(p, filepath.Join(a.Workspace, path))
	}
	return p, nil
}
<commit_msg>Don't unmute logs if an error happened<commit_after>/*
Copyright 2018 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package runner

import (
	"context"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/bazel"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build/gcb"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build/kaniko"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build/local"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build/tag"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/color"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/config"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/deploy"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes"
	kubectx "github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes/context"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/v1alpha2"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/watch"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

const PollInterval = 1000 * time.Millisecond

// ErrorConfigurationChanged is a special error that's returned when the skaffold configuration was changed.
var ErrorConfigurationChanged = errors.New("configuration changed")

// SkaffoldRunner is responsible for running the skaffold build and deploy pipeline.
type SkaffoldRunner struct {
	build.Builder
	deploy.Deployer
	tag.Tagger

	opts         *config.SkaffoldOptions
	watchFactory watch.Factory
	builds       []build.Artifact
}

// NewForConfig returns a new SkaffoldRunner for a SkaffoldConfig
func NewForConfig(opts *config.SkaffoldOptions, cfg *config.SkaffoldConfig) (*SkaffoldRunner, error) {
	kubeContext, err := kubectx.CurrentContext()
	if err != nil {
		return nil, errors.Wrap(err, "getting current cluster context")
	}
	logrus.Infof("Using kubectl context: %s", kubeContext)

	tagger, err := getTagger(cfg.Build.TagPolicy, opts.CustomTag)
	if err != nil {
		return nil, errors.Wrap(err, "parsing skaffold tag config")
	}

	builder, err := getBuilder(&cfg.Build, kubeContext)
	if err != nil {
		return nil, errors.Wrap(err, "parsing skaffold build config")
	}

	deployer, err := getDeployer(&cfg.Deploy, kubeContext, opts.Namespace)
	if err != nil {
		return nil, errors.Wrap(err, "parsing skaffold deploy config")
	}

	deployer = deploy.WithLabels(deployer, opts, builder, deployer, tagger)
	builder, deployer = WithTimings(builder, deployer)
	if opts.Notification {
		deployer = WithNotification(deployer)
	}

	return &SkaffoldRunner{
		Builder:      builder,
		Deployer:     deployer,
		Tagger:       tagger,
		opts:         opts,
		watchFactory: watch.NewWatcher,
	}, nil
}

func getBuilder(cfg *v1alpha2.BuildConfig, kubeContext string) (build.Builder, error) {
	switch {
	case cfg.LocalBuild != nil:
		logrus.Debugf("Using builder: local")
		return local.NewBuilder(cfg.LocalBuild, kubeContext)

	case cfg.GoogleCloudBuild != nil:
		logrus.Debugf("Using builder: google cloud")
		return gcb.NewBuilder(cfg.GoogleCloudBuild), nil

	case cfg.KanikoBuild != nil:
		logrus.Debugf("Using builder: kaniko")
		return kaniko.NewBuilder(cfg.KanikoBuild), nil

	default:
		return nil, fmt.Errorf("Unknown builder for config %+v", cfg)
	}
}

func getDeployer(cfg *v1alpha2.DeployConfig, kubeContext string, namespace string) (deploy.Deployer, error) {
	switch {
	case cfg.KubectlDeploy != nil:
		// TODO(dgageot): this should be the folder containing skaffold.yaml. Should also be moved elsewhere.
		cwd, err := os.Getwd()
		if err != nil {
			return nil, errors.Wrap(err, "finding current directory")
		}
		return deploy.NewKubectlDeployer(cwd, cfg.KubectlDeploy, kubeContext, namespace), nil

	case cfg.HelmDeploy != nil:
		return deploy.NewHelmDeployer(cfg.HelmDeploy, kubeContext, namespace), nil

	case cfg.KustomizeDeploy != nil:
		return deploy.NewKustomizeDeployer(cfg.KustomizeDeploy, kubeContext, namespace), nil

	default:
		return nil, fmt.Errorf("Unknown deployer for config %+v", cfg)
	}
}

func getTagger(t v1alpha2.TagPolicy, customTag string) (tag.Tagger, error) {
	switch {
	case customTag != "":
		return &tag.CustomTag{
			Tag: customTag,
		}, nil

	case t.EnvTemplateTagger != nil:
		return tag.NewEnvTemplateTagger(t.EnvTemplateTagger.Template)

	case t.ShaTagger != nil:
		return &tag.ChecksumTagger{}, nil

	case t.GitTagger != nil:
		return &tag.GitCommit{}, nil

	case t.DateTimeTagger != nil:
		return tag.NewDateTimeTagger(t.DateTimeTagger.Format, t.DateTimeTagger.TimeZone), nil

	default:
		return nil, fmt.Errorf("Unknown tagger for strategy %+v", t)
	}
}

// Run builds artifacts and then deploys them.
func (r *SkaffoldRunner) Run(ctx context.Context, out io.Writer, artifacts []*v1alpha2.Artifact) error {
	bRes, err := r.Build(ctx, out, r.Tagger, artifacts)
	if err != nil {
		return errors.Wrap(err, "build step")
	}

	_, err = r.Deploy(ctx, out, bRes)
	if err != nil {
		return errors.Wrap(err, "deploy step")
	}

	return r.TailLogs(ctx, out, artifacts, bRes)
}

// TailLogs prints the logs for deployed artifacts.
func (r *SkaffoldRunner) TailLogs(ctx context.Context, out io.Writer, artifacts []*v1alpha2.Artifact, bRes []build.Artifact) error {
	if !r.opts.Tail {
		return nil
	}

	imageList := kubernetes.NewImageList()
	for _, b := range bRes {
		imageList.Add(b.Tag)
	}

	colorPicker := kubernetes.NewColorPicker(artifacts)
	logger := kubernetes.NewLogAggregator(out, imageList, colorPicker)
	if err := logger.Start(ctx); err != nil {
		return errors.Wrap(err, "starting logger")
	}

	<-ctx.Done()
	return nil
}

// Dev watches for changes and runs the skaffold build and deploy
// pipeline until interrrupted by the user.
func (r *SkaffoldRunner) Dev(ctx context.Context, out io.Writer, artifacts []*v1alpha2.Artifact) ([]build.Artifact, error) {
	imageList := kubernetes.NewImageList()
	colorPicker := kubernetes.NewColorPicker(artifacts)
	logger := kubernetes.NewLogAggregator(out, imageList, colorPicker)

	// Create watcher and register artifacts to build current state of files.
	changed := changes{}
	onChange := func() error {
		hasError := true

		logger.Mute()
		defer func() {
			changed.reset()
			color.Default.Fprintln(out, "Watching for changes...")
			if !hasError {
				logger.Unmute()
			}
		}()

		switch {
		case changed.needsReload:
			return ErrorConfigurationChanged
		case len(changed.dirtyArtifacts) > 0:
			bRes, err := r.Build(ctx, out, r.Tagger, changed.dirtyArtifacts)
			if err != nil {
				logrus.Warnln("Skipping Deploy due to build error:", err)
				return nil
			}

			r.updateBuiltImages(imageList, bRes)

			if _, err = r.Deploy(ctx, out, r.builds); err != nil {
				logrus.Warnln("Skipping Deploy due to error:", err)
				return nil
			}
		case changed.needsRedeploy:
			if _, err := r.Deploy(ctx, out, r.builds); err != nil {
				logrus.Warnln("Skipping Deploy due to error:", err)
				return nil
			}
		}

		hasError = false
		return nil
	}

	watcher := r.watchFactory()

	// Watch artifacts
	for i := range artifacts {
		artifact := artifacts[i]

		if !r.shouldWatch(artifact) {
			continue
		}

		if err := watcher.Register(
			func() ([]string, error) { return dependenciesForArtifact(artifact) },
			func() { changed.Add(artifact) },
		); err != nil {
			return nil, errors.Wrapf(err, "watching files for artifact %s", artifact.ImageName)
		}
	}

	// Watch deployment configuration
	if err := watcher.Register(
		func() ([]string, error) { return r.Dependencies() },
		func() { changed.needsRedeploy = true },
	); err != nil {
		return nil, errors.Wrap(err, "watching files for deployer")
	}

	// Watch Skaffold configuration
	if err := watcher.Register(
		func() ([]string, error) { return []string{r.opts.ConfigurationFile}, nil },
		func() { changed.needsReload = true },
	); err != nil {
		return nil, errors.Wrapf(err, "watching skaffold configuration %s", r.opts.ConfigurationFile)
	}

	// First run
	bRes, err := r.Build(ctx, out, r.Tagger, artifacts)
	if err != nil {
		return nil, errors.Wrap(err, "exiting dev mode because the first build failed")
	}

	r.updateBuiltImages(imageList, bRes)

	_, err = r.Deploy(ctx, out, r.builds)
	if err != nil {
		return nil, errors.Wrap(err, "exiting dev mode because the first deploy failed")
	}

	// Start logs
	if err := logger.Start(ctx); err != nil {
		return nil, errors.Wrap(err, "starting logger")
	}

	return nil, watcher.Run(ctx, PollInterval, onChange)
}

func (r *SkaffoldRunner) shouldWatch(artifact *v1alpha2.Artifact) bool {
	if len(r.opts.Watch) == 0 {
		return true
	}

	for _, watchExpression := range r.opts.Watch {
		if strings.Contains(artifact.ImageName, watchExpression) {
			return true
		}
	}

	return false
}

func (r *SkaffoldRunner) updateBuiltImages(images *kubernetes.ImageList, bRes []build.Artifact) {
	// Update which images are logged.
	for _, build := range bRes {
		images.Add(build.Tag)
	}

	// Make sure all artifacts are redeployed. Not only those that were just rebuilt.
	r.builds = mergeWithPreviousBuilds(bRes, r.builds)
}

func mergeWithPreviousBuilds(builds, previous []build.Artifact) []build.Artifact {
	updatedBuilds := map[string]bool{}
	for _, build := range builds {
		updatedBuilds[build.ImageName] = true
	}

	var merged []build.Artifact
	merged = append(merged, builds...)

	for _, b := range previous {
		if !updatedBuilds[b.ImageName] {
			merged = append(merged, b)
		}
	}

	return merged
}

func dependenciesForArtifact(a *v1alpha2.Artifact) ([]string, error) {
	var (
		paths []string
		err   error
	)

	switch {
	case a.DockerArtifact != nil:
		paths, err = docker.GetDependencies(a.Workspace, a.DockerArtifact)

	case a.BazelArtifact != nil:
		paths, err = bazel.GetDependencies(a.Workspace, a.BazelArtifact)

	default:
		return nil, fmt.Errorf("undefined artifact type: %+v", a.ArtifactType)
	}

	if err != nil {
		return nil, err
	}

	var p []string
	for _, path := range paths {
		p = append(p, filepath.Join(a.Workspace, path))
	}
	return p, nil
}
<|endoftext|>


<commit_before>package air

import (
	"bytes"
	"errors"
	"fmt"
	"net/http"
	"path"
	"reflect"
	"runtime"
	"sync"
)

type (
	// Air is the top-level framework struct.
	Air struct {
		pool     *pool
		pregases []GasFunc
		gases    []GasFunc
		router   *router
		binder   *binder
		renderer *renderer

		Config           *Config
		HTTPErrorHandler HTTPErrorHandler
		Logger           *Logger
	}

	// pool represents the pools of `Air`.
	pool struct {
		context        *sync.Pool
		request        *sync.Pool
		response       *sync.Pool
		requestHeader  *sync.Pool
		responseHeader *sync.Pool
		uri            *sync.Pool
	}

	// HandlerFunc defines a function to server HTTP requests.
	HandlerFunc func(*Context) error

	// GasFunc defines a function to process gas.
	GasFunc func(HandlerFunc) HandlerFunc

	// HTTPError represents an error that occurred while handling a request.
	HTTPError struct {
		Code    int
		Message string
	}

	// HTTPErrorHandler is a centralized HTTP error handler.
	HTTPErrorHandler func(error, *Context)
)

// HTTP methods (which follows the REST principle)
const (
	GET    = "GET"
	POST   = "POST"
	PUT    = "PUT" // The Air advises you to forget the PATCH.
	DELETE = "DELETE"
)

// For easy for-range
var methods = [4]string{GET, POST, PUT, DELETE}

// MIME types
const (
	MIMEApplicationJSON       = "application/json; charset=utf-8"
	MIMEApplicationJavaScript = "application/javascript; charset=utf-8"
	MIMEApplicationXML        = "application/xml; charset=utf-8"
	MIMEApplicationForm       = "application/x-www-form-urlencoded"
	MIMEApplicationProtobuf   = "application/protobuf"
	MIMEApplicationMsgpack    = "application/msgpack"
	MIMETextHTML              = "text/html; charset=utf-8"
	MIMETextPlain             = "text/plain; charset=utf-8"
	MIMEMultipartForm         = "multipart/form-data"
	MIMEOctetStream           = "application/octet-stream"
)

// Headers
const (
	HeaderAcceptEncoding                = "Accept-Encoding"
	HeaderAllow                         = "Allow"
	HeaderAuthorization                 = "Authorization"
	HeaderContentDisposition            = "Content-Disposition"
	HeaderContentEncoding               = "Content-Encoding"
	HeaderContentLength                 = "Content-Length"
	HeaderContentType                   = "Content-Type"
	HeaderCookie                        = "Cookie"
	HeaderSetCookie                     = "Set-Cookie"
	HeaderIfModifiedSince               = "If-Modified-Since"
	HeaderLastModified                  = "Last-Modified"
	HeaderLocation                      = "Location"
	HeaderUpgrade                       = "Upgrade"
	HeaderVary                          = "Vary"
	HeaderWWWAuthenticate               = "WWW-Authenticate"
	HeaderXForwardedProto               = "X-Forwarded-Proto"
	HeaderXHTTPMethodOverride           = "X-HTTP-Method-Override"
	HeaderXForwardedFor                 = "X-Forwarded-For"
	HeaderXRealIP                       = "X-Real-IP"
	HeaderServer                        = "Server"
	HeaderOrigin                        = "Origin"
	HeaderAccessControlRequestMethod    = "Access-Control-Request-Method"
	HeaderAccessControlRequestHeaders   = "Access-Control-Request-Headers"
	HeaderAccessControlAllowOrigin      = "Access-Control-Allow-Origin"
	HeaderAccessControlAllowMethods     = "Access-Control-Allow-Methods"
	HeaderAccessControlAllowHeaders     = "Access-Control-Allow-Headers"
	HeaderAccessControlAllowCredentials = "Access-Control-Allow-Credentials"
	HeaderAccessControlExposeHeaders    = "Access-Control-Expose-Headers"
	HeaderAccessControlMaxAge           = "Access-Control-Max-Age"

	HeaderStrictTransportSecurity = "Strict-Transport-Security"
	HeaderXContentTypeOptions     = "X-Content-Type-Options"
	HeaderXXSSProtection          = "X-XSS-Protection"
	HeaderXFrameOptions           = "X-Frame-Options"
	HeaderContentSecurityPolicy   = "Content-Security-Policy"
	HeaderXCSRFToken              = "X-CSRF-Token"
)

// Errors
var (
	ErrUnauthorized                = NewHTTPError(http.StatusUnauthorized)          // 401
	ErrNotFound                    = NewHTTPError(http.StatusNotFound)              // 404
	ErrMethodNotAllowed            = NewHTTPError(http.StatusMethodNotAllowed)      // 405
	ErrStatusRequestEntityTooLarge = NewHTTPError(http.StatusRequestEntityTooLarge) // 413
	ErrUnsupportedMediaType        = NewHTTPError(http.StatusUnsupportedMediaType)  // 415

	ErrInternalServerError = NewHTTPError(http.StatusInternalServerError) // 500
	ErrBadGateway          = NewHTTPError(http.StatusBadGateway)          // 502
	ErrServiceUnavailable  = NewHTTPError(http.StatusServiceUnavailable)  // 503
	ErrGatewayTimeout      = NewHTTPError(http.StatusGatewayTimeout)      // 504

	ErrInvalidRedirectCode = errors.New("Invalid Redirect Status Code")
	ErrCookieNotFound      = errors.New("Cookie Not Found")

	ErrDataTemplateNotSetted = errors.New("c.Data[\"template\"] Not Setted")
	ErrDataHTMLNotSetted     = errors.New("c.Data[\"html\"] Not Setted")
	ErrDataStringNotSetted   = errors.New("c.Data[\"string\"] Not Setted")
	ErrDataJSONNotSetted     = errors.New("c.Data[\"json\"] Not Setted")
	ErrDataJSONPNotSetted    = errors.New("c.Data[\"jsonp\"] Not Setted")
	ErrDataCallbackNotSetted = errors.New("c.Data[\"callback\"] Not Setted")
	ErrDataXMLNotSetted      = errors.New("c.Data[\"xml\"] Not Setted")
)

// HTTP error handlers
var (
	NotFoundHandler = func(c *Context) error {
		return ErrNotFound
	}

	MethodNotAllowedHandler = func(c *Context) error {
		return ErrMethodNotAllowed
	}
)

// New returns a new instance of `Air`.
func New() *Air {
	a := &Air{}
	a.pool = newPool(a)
	a.router = newRouter(a)
	a.binder = newBinder(a)
	a.renderer = newRenderer(a)
	a.Config = NewConfig("air")
	a.HTTPErrorHandler = a.defaultHTTPErrorHandler
	a.Logger = NewLogger(a)
	a.Logger.Level = ERROR
	return a
}

// defaultHTTPErrorHandler invokes the default HTTP error handler.
func (a *Air) defaultHTTPErrorHandler(err error, c *Context) {
	code := http.StatusInternalServerError
	msg := http.StatusText(code)
	if he, ok := err.(*HTTPError); ok {
		code = he.Code
		msg = he.Message
	}
	if a.Config.DebugMode {
		msg = err.Error()
	}
	if !c.Response.Committed {
		c.Data["string"] = msg
		c.Status = code
		c.String()
	}
	a.Logger.Error(err)
}

// Precontain adds gases to the chain which is run before router.
func (a *Air) Precontain(gases ...GasFunc) {
	a.pregases = append(a.pregases, gases...)
}

// Contain adds gases to the chain which is run after router.
func (a *Air) Contain(gases ...GasFunc) {
	a.gases = append(a.gases, gases...)
}

// Get registers a new GET route for a path with matching handler in the router
// with optional route-level gases.
func (a *Air) Get(path string, handler HandlerFunc, gases ...GasFunc) {
	a.add(GET, path, handler, gases...)
}

// Post registers a new POST route for a path with matching handler in the
// router with optional route-level gases.
func (a *Air) Post(path string, handler HandlerFunc, gases ...GasFunc) {
	a.add(POST, path, handler, gases...)
}

// Put registers a new PUT route for a path with matching handler in the
// router with optional route-level gases.
func (a *Air) Put(path string, handler HandlerFunc, gases ...GasFunc) {
	a.add(PUT, path, handler, gases...)
}

// Delete registers a new DELETE route for a path with matching handler in
// the router with optional route-level gases.
func (a *Air) Delete(path string, handler HandlerFunc, gases ...GasFunc) {
	a.add(DELETE, path, handler, gases...)
}

// Static registers a new route with path prefix to serve static files from
// the provided root directory.
func (a *Air) Static(prefix, root string) {
	a.Get(prefix+"*", func(c *Context) error {
		return c.File(path.Join(root, c.Params[c.ParamNames[0]]))
	})
}

// File registers a new route with path to serve a static file.
func (a *Air) File(path, file string) {
	a.Get(path, func(c *Context) error {
		return c.File(file)
	})
}

// add registers a new route for a path with a HTTP method and matching handler
// in the router with optional route-level gases.
func (a *Air) add(method, path string, handler HandlerFunc, gases ...GasFunc) {
	name := handlerName(handler)
	a.router.add(method, path, func(c *Context) error {
		h := handler
		// Chain gases
		for i := len(gases) - 1; i >= 0; i-- {
			h = gases[i](h)
		}
		return h(c)
	}, a)
	r := route{
		method:  method,
		path:    path,
		handler: name,
	}
	a.router.routes[method+path] = r
}

// SetTemplateFunc sets the f into template func map with a name.
func (a *Air) SetTemplateFunc(name string, f interface{}) {
	a.renderer.templateFuncMap[name] = f
}

// URI returns a URI generated from handler with optional params.
func (a *Air) URI(handler HandlerFunc, params ...interface{}) string {
	uri := new(bytes.Buffer)
	ln := len(params)
	n := 0
	name := handlerName(handler)
	for _, r := range a.router.routes {
		if r.handler == name {
			for i, l := 0, len(r.path); i < l; i++ {
				if r.path[i] == ':' && n < ln {
					for ; i < l && r.path[i] != '/'; i++ {
					}
					uri.WriteString(fmt.Sprintf("%v", params[n]))
					n++
				}
				if i < l {
					uri.WriteByte(r.path[i])
				}
			}
			break
		}
	}
	return uri.String()
}

// Run starts the HTTP server.
func (a *Air) Run() {
	a.renderer.parseTemplates()
	if a.Config.DebugMode {
		a.Logger.Level = DEBUG
		a.Logger.Debug("Running In Debug Mode")
	}
	s := newServer(a)
	a.Logger.Error(s.start())
}

// serveHTTP implements `serverHandler#serveHTTP()`.
func (a *Air) serveHTTP(req *Request, res *Response) {
	c := a.pool.context.Get().(*Context)
	c.Request = req
	c.Response = res

	// Gases
	h := func(*Context) error {
		method := req.Method()
		path := req.URI.Path()
		a.router.find(method, path, c)
		h := c.Handler
		for i := len(a.gases) - 1; i >= 0; i-- {
			h = a.gases[i](h)
		}
		return h(c)
	}

	// Pregases
	for i := len(a.pregases) - 1; i >= 0; i-- {
		h = a.pregases[i](h)
	}

	// Execute chain
	if err := h(c); err != nil {
		a.HTTPErrorHandler(err, c)
	}

	c.reset()
	a.pool.context.Put(c)
}

// newPool returnes a new instance of `pool`.
func newPool(a *Air) *pool {
	return &pool{
		context: &sync.Pool{
			New: func() interface{} {
				return newContext(a)
			},
		},
		request: &sync.Pool{
			New: func() interface{} {
				return newRequest(a)
			},
		},
		response: &sync.Pool{
			New: func() interface{} {
				return newResponse(a)
			},
		},
		requestHeader: &sync.Pool{
			New: func() interface{} {
				return &RequestHeader{}
			},
		},
		responseHeader: &sync.Pool{
			New: func() interface{} {
				return &ResponseHeader{}
			},
		},
		uri: &sync.Pool{
			New: func() interface{} {
				return &URI{}
			},
		},
	}
}

// handlerName returns the handler's func name.
func handlerName(handler HandlerFunc) string {
	t := reflect.ValueOf(handler).Type()
	if t.Kind() == reflect.Func {
		return runtime.FuncForPC(reflect.ValueOf(handler).Pointer()).Name()
	}
	return t.String()
}

// NewHTTPError returns a new instance of `HTTPError`.
func NewHTTPError(code int, msg ...string) *HTTPError {
	he := &HTTPError{Code: code, Message: http.StatusText(code)}
	if len(msg) > 0 {
		he.Message = msg[0]
	}
	return he
}

// Error implements `error#Error()`.
func (he *HTTPError) Error() string {
	return he.Message
}
<commit_msg>style: beautify air.go<commit_after>package air

import (
	"bytes"
	"errors"
	"fmt"
	"net/http"
	"path"
	"reflect"
	"runtime"
	"sync"
)

type (
	// Air is the top-level framework struct.
	Air struct {
		pool     *pool
		pregases []GasFunc
		gases    []GasFunc
		router   *router
		binder   *binder
		renderer *renderer

		Config           *Config
		HTTPErrorHandler HTTPErrorHandler
		Logger           *Logger
	}

	// pool represents the pools of `Air`.
	pool struct {
		context        *sync.Pool
		request        *sync.Pool
		response       *sync.Pool
		requestHeader  *sync.Pool
		responseHeader *sync.Pool
		uri            *sync.Pool
	}

	// HandlerFunc defines a function to server HTTP requests.
	HandlerFunc func(*Context) error

	// GasFunc defines a function to process gas.
	GasFunc func(HandlerFunc) HandlerFunc

	// HTTPError represents an error that occurred while handling a request.
	HTTPError struct {
		Code    int
		Message string
	}

	// HTTPErrorHandler is a centralized HTTP error handler.
	HTTPErrorHandler func(error, *Context)
)

// HTTP methods (which follows the REST principle)
const (
	GET    = "GET"
	POST   = "POST"
	PUT    = "PUT" // The Air advises you to forget the PATCH.
	DELETE = "DELETE"
)

// For easy for-range
var methods = [4]string{GET, POST, PUT, DELETE}

// MIME types
const (
	MIMEApplicationJSON       = "application/json; charset=utf-8"
	MIMEApplicationJavaScript = "application/javascript; charset=utf-8"
	MIMEApplicationXML        = "application/xml; charset=utf-8"
	MIMEApplicationForm       = "application/x-www-form-urlencoded"
	MIMEApplicationProtobuf   = "application/protobuf"
	MIMEApplicationMsgpack    = "application/msgpack"
	MIMETextHTML              = "text/html; charset=utf-8"
	MIMETextPlain             = "text/plain; charset=utf-8"
	MIMEMultipartForm         = "multipart/form-data"
	MIMEOctetStream           = "application/octet-stream"
)

// Headers
const (
	HeaderAcceptEncoding                = "Accept-Encoding"
	HeaderAllow                         = "Allow"
	HeaderAuthorization                 = "Authorization"
	HeaderContentDisposition            = "Content-Disposition"
	HeaderContentEncoding               = "Content-Encoding"
	HeaderContentLength                 = "Content-Length"
	HeaderContentType                   = "Content-Type"
	HeaderCookie                        = "Cookie"
	HeaderSetCookie                     = "Set-Cookie"
	HeaderIfModifiedSince               = "If-Modified-Since"
	HeaderLastModified                  = "Last-Modified"
	HeaderLocation                      = "Location"
	HeaderUpgrade                       = "Upgrade"
	HeaderVary                          = "Vary"
	HeaderWWWAuthenticate               = "WWW-Authenticate"
	HeaderXForwardedProto               = "X-Forwarded-Proto"
	HeaderXHTTPMethodOverride           = "X-HTTP-Method-Override"
	HeaderXForwardedFor                 = "X-Forwarded-For"
	HeaderXRealIP                       = "X-Real-IP"
	HeaderServer                        = "Server"
	HeaderOrigin                        = "Origin"
	HeaderAccessControlRequestMethod    = "Access-Control-Request-Method"
	HeaderAccessControlRequestHeaders   = "Access-Control-Request-Headers"
	HeaderAccessControlAllowOrigin      = "Access-Control-Allow-Origin"
	HeaderAccessControlAllowMethods     = "Access-Control-Allow-Methods"
	HeaderAccessControlAllowHeaders     = "Access-Control-Allow-Headers"
	HeaderAccessControlAllowCredentials = "Access-Control-Allow-Credentials"
	HeaderAccessControlExposeHeaders    = "Access-Control-Expose-Headers"
	HeaderAccessControlMaxAge           = "Access-Control-Max-Age"

	HeaderStrictTransportSecurity = "Strict-Transport-Security"
	HeaderXContentTypeOptions     = "X-Content-Type-Options"
	HeaderXXSSProtection          = "X-XSS-Protection"
	HeaderXFrameOptions           = "X-Frame-Options"
	HeaderContentSecurityPolicy   = "Content-Security-Policy"
	HeaderXCSRFToken              = "X-CSRF-Token"
)

// Errors
var (
	ErrUnauthorized                = NewHTTPError(http.StatusUnauthorized)          // 401
	ErrNotFound                    = NewHTTPError(http.StatusNotFound)              // 404
	ErrMethodNotAllowed            = NewHTTPError(http.StatusMethodNotAllowed)      // 405
	ErrStatusRequestEntityTooLarge = NewHTTPError(http.StatusRequestEntityTooLarge) // 413
	ErrUnsupportedMediaType        = NewHTTPError(http.StatusUnsupportedMediaType)  // 415

	ErrInternalServerError = NewHTTPError(http.StatusInternalServerError) // 500
	ErrBadGateway          = NewHTTPError(http.StatusBadGateway)          // 502
	ErrServiceUnavailable  = NewHTTPError(http.StatusServiceUnavailable)  // 503
	ErrGatewayTimeout      = NewHTTPError(http.StatusGatewayTimeout)      // 504

	ErrInvalidRedirectCode = errors.New("Invalid Redirect Status Code")
	ErrCookieNotFound      = errors.New("Cookie Not Found")

	ErrDataTemplateNotSetted = errors.New("c.Data[\"template\"] Not Setted")
	ErrDataHTMLNotSetted     = errors.New("c.Data[\"html\"] Not Setted")
	ErrDataStringNotSetted   = errors.New("c.Data[\"string\"] Not Setted")
	ErrDataJSONNotSetted     = errors.New("c.Data[\"json\"] Not Setted")
	ErrDataJSONPNotSetted    = errors.New("c.Data[\"jsonp\"] Not Setted")
	ErrDataCallbackNotSetted = errors.New("c.Data[\"callback\"] Not Setted")
	ErrDataXMLNotSetted      = errors.New("c.Data[\"xml\"] Not Setted")
)

// HTTP error handlers
var (
	NotFoundHandler = func(c *Context) error {
		return ErrNotFound
	}

	MethodNotAllowedHandler = func(c *Context) error {
		return ErrMethodNotAllowed
	}
)

// New returns a new instance of `Air`.
func New() *Air {
	a := &Air{}
	a.pool = newPool(a)
	a.router = newRouter(a)
	a.binder = newBinder(a)
	a.renderer = newRenderer(a)
	a.Config = NewConfig("air")
	a.HTTPErrorHandler = a.defaultHTTPErrorHandler
	a.Logger = NewLogger(a)
	a.Logger.Level = ERROR
	return a
}

// defaultHTTPErrorHandler invokes the default HTTP error handler.
func (a *Air) defaultHTTPErrorHandler(err error, c *Context) {
	code := http.StatusInternalServerError
	msg := http.StatusText(code)
	if he, ok := err.(*HTTPError); ok {
		code = he.Code
		msg = he.Message
	}
	if a.Config.DebugMode {
		msg = err.Error()
	}
	if !c.Response.Committed {
		c.Data["string"] = msg
		c.Status = code
		c.String()
	}
	a.Logger.Error(err)
}

// Precontain adds gases to the chain which is run before router.
func (a *Air) Precontain(gases ...GasFunc) {
	a.pregases = append(a.pregases, gases...)
}

// Contain adds gases to the chain which is run after router.
func (a *Air) Contain(gases ...GasFunc) {
	a.gases = append(a.gases, gases...)
}

// Get registers a new GET route for a path with matching handler in the router
// with optional route-level gases.
func (a *Air) Get(path string, handler HandlerFunc, gases ...GasFunc) {
	a.add(GET, path, handler, gases...)
}

// Post registers a new POST route for a path with matching handler in the
// router with optional route-level gases.
func (a *Air) Post(path string, handler HandlerFunc, gases ...GasFunc) {
	a.add(POST, path, handler, gases...)
}

// Put registers a new PUT route for a path with matching handler in the
// router with optional route-level gases.
func (a *Air) Put(path string, handler HandlerFunc, gases ...GasFunc) {
	a.add(PUT, path, handler, gases...)
}

// Delete registers a new DELETE route for a path with matching handler in
// the router with optional route-level gases.
func (a *Air) Delete(path string, handler HandlerFunc, gases ...GasFunc) {
	a.add(DELETE, path, handler, gases...)
}

// Static registers a new route with path prefix to serve static files from
// the provided root directory.
func (a *Air) Static(prefix, root string) {
	a.Get(prefix+"*", func(c *Context) error {
		return c.File(path.Join(root, c.Params[c.ParamNames[0]]))
	})
}

// File registers a new route with path to serve a static file.
func (a *Air) File(path, file string) {
	a.Get(path, func(c *Context) error {
		return c.File(file)
	})
}

// add registers a new route for a path with a HTTP method and matching handler
// in the router with optional route-level gases.
func (a *Air) add(method, path string, handler HandlerFunc, gases ...GasFunc) {
	name := handlerName(handler)
	a.router.add(method, path, func(c *Context) error {
		h := handler
		// Chain gases
		for i := len(gases) - 1; i >= 0; i-- {
			h = gases[i](h)
		}
		return h(c)
	}, a)
	r := route{
		method:  method,
		path:    path,
		handler: name,
	}
	a.router.routes[method+path] = r
}

// SetTemplateFunc sets the f into template func map with a name.
func (a *Air) SetTemplateFunc(name string, f interface{}) {
	a.renderer.templateFuncMap[name] = f
}

// URI returns a URI generated from handler with optional params.
func (a *Air) URI(handler HandlerFunc, params ...interface{}) string {
	uri := new(bytes.Buffer)
	ln := len(params)
	n := 0
	name := handlerName(handler)
	for _, r := range a.router.routes {
		if r.handler == name {
			for i, l := 0, len(r.path); i < l; i++ {
				if r.path[i] == ':' && n < ln {
					for ; i < l && r.path[i] != '/'; i++ {
					}
					uri.WriteString(fmt.Sprintf("%v", params[n]))
					n++
				}
				if i < l {
					uri.WriteByte(r.path[i])
				}
			}
			break
		}
	}
	return uri.String()
}

// Run starts the HTTP server.
func (a *Air) Run() {
	a.renderer.parseTemplates()
	if a.Config.DebugMode {
		a.Logger.Level = DEBUG
		a.Logger.Debug("Running In Debug Mode")
	}
	s := newServer(a)
	a.Logger.Error(s.start())
}

// serveHTTP implements `serverHandler#serveHTTP()`.
func (a *Air) serveHTTP(req *Request, res *Response) {
	c := a.pool.context.Get().(*Context)
	c.Request = req
	c.Response = res

	// Gases
	h := func(*Context) error {
		method := req.Method()
		path := req.URI.Path()
		a.router.find(method, path, c)
		h := c.Handler
		for i := len(a.gases) - 1; i >= 0; i-- {
			h = a.gases[i](h)
		}
		return h(c)
	}

	// Pregases
	for i := len(a.pregases) - 1; i >= 0; i-- {
		h = a.pregases[i](h)
	}

	// Execute chain
	if err := h(c); err != nil {
		a.HTTPErrorHandler(err, c)
	}

	c.reset()
	a.pool.context.Put(c)
}

// newPool returnes a new instance of `pool`.
func newPool(a *Air) *pool {
	return &pool{
		context: &sync.Pool{
			New: func() interface{} {
				return newContext(a)
			},
		},
		request: &sync.Pool{
			New: func() interface{} {
				return newRequest(a)
			},
		},
		response: &sync.Pool{
			New: func() interface{} {
				return newResponse(a)
			},
		},
		requestHeader: &sync.Pool{
			New: func() interface{} {
				return &RequestHeader{}
			},
		},
		responseHeader: &sync.Pool{
			New: func() interface{} {
				return &ResponseHeader{}
			},
		},
		uri: &sync.Pool{
			New: func() interface{} {
				return &URI{}
			},
		},
	}
}

// NewHTTPError returns a new instance of `HTTPError`.
func NewHTTPError(code int, msg ...string) *HTTPError {
	he := &HTTPError{Code: code, Message: http.StatusText(code)}
	if len(msg) > 0 {
		he.Message = msg[0]
	}
	return he
}

// Error implements `error#Error()`.
func (he *HTTPError) Error() string {
	return he.Message
}

// handlerName returns the handler's func name.
func handlerName(handler HandlerFunc) string {
	t := reflect.ValueOf(handler).Type()
	if t.Kind() == reflect.Func {
		return runtime.FuncForPC(reflect.ValueOf(handler).Pointer()).Name()
	}
	return t.String()
}
<|endoftext|>


<commit_before>package main

import "fmt"
import "math/big"
import "os"
import "runtime"

// Returns whether (X + a)^n = X^n + a mod (n, X^r - 1).
func isAKSWitness(n, r, a *big.Int) bool {
	reduceAKS := func(p *IntPoly) {
		p.PowMod(p, r).Mod(p, n)
	}

	zero := big.NewInt(0)
	one := big.NewInt(1)
	lhs := NewIntPoly([][2]*big.Int{{a, zero}, {one, one}})
	lhs.GenPow(lhs, n, reduceAKS)

	rhs := NewIntPoly([][2]*big.Int{{a, zero}, {one, n}})
	reduceAKS(rhs)

	isWitness := !lhs.Eq(rhs)
	return isWitness
}

// Returns the first AKS witness of n with the parameters r and M, or
// nil if there isn't one.
func getFirstAKSWitness(n, r, M *big.Int) *big.Int {
	for a := big.NewInt(1); a.Cmp(M) < 0; a.Add(a, big.NewInt(1)) {
		fmt.Printf("Testing %v (M = %v)...\n", a, M)
		if isWitness := isAKSWitness(n, r, a); isWitness {
			return a
		}
	}
	return nil
}

// Holds the result of an AKS witness test.
type witnessResult struct {
	a         *big.Int
	isWitness bool
}

// Tests all numbers received on numberCh if they are witnesses of n
// with parameter r. Sends the results to resultCh.
func testAKSWitnesses(
	n, r *big.Int,
	numberCh chan *big.Int,
	resultCh chan witnessResult) {
	for a := range numberCh {
		fmt.Printf("Testing %v...\n", a)
		isWitness := isAKSWitness(n, r, a)
		fmt.Printf("Finished testing %v (isWitness=%t)\n",
			a, isWitness)
		resultCh <- witnessResult{a, isWitness}
	}
}

// Returns an AKS witness of n with the parameters r and M, or nil if
// there isn't one. Tests up to maxOutstanding numbers at once.
func getAKSWitness(n, r, M *big.Int, maxOutstanding int) *big.Int {
	numberCh := make(chan *big.Int, maxOutstanding)
	defer close(numberCh)
	resultCh := make(chan witnessResult, maxOutstanding)
	for i := 0; i < maxOutstanding; i++ {
		go testAKSWitnesses(n, r, numberCh, resultCh)
	}

	// Send off all numbers for testing, draining any results that
	// come in while we're doing so.
	tested := big.NewInt(1)
	for i := big.NewInt(1); i.Cmp(M) < 0; {
		select {
		case result := <-resultCh:
			tested.Add(tested, big.NewInt(1))
			fmt.Printf("%v isWitness=%t\n",
				result.a, result.isWitness)
			if result.isWitness {
				return result.a
			}
		default:
			var a big.Int
			a.Set(i)
			numberCh <- &a
			i.Add(i, big.NewInt(1))
		}
	}

	// Drain any remaining results.
	for tested.Cmp(M) < 0 {
		result := <-resultCh
		tested.Add(tested, big.NewInt(1))
		fmt.Printf("%v isWitness=%t\n", result.a, result.isWitness)
		if result.isWitness {
			return result.a
		}
	}

	return nil
}

// Returns an upper bound for r such that o_r(n) > ceil(lg(n))^2 that
// is polylog in n.
func calculateAKSModulusUpperBound(n *big.Int) *big.Int {
	two := big.NewInt(2)
	three := big.NewInt(3)
	five := big.NewInt(5)
	eight := big.NewInt(8)

	// Calculate max(ceil(lg(n))^5, 3).
	ceilLgN := big.NewInt(int64(n.BitLen()))
	rUpperBound := &big.Int{}
	rUpperBound.Exp(ceilLgN, five, nil)
	rUpperBound = Max(rUpperBound, three)

	var nMod8 big.Int
	nMod8.Mod(n, eight)
	if (nMod8.Cmp(three) == 0) || (nMod8.Cmp(five) == 0) {
		// Calculate 8*ceil(lg(n))^2.
		var rUpperBound2 big.Int
		rUpperBound2.Exp(ceilLgN, two, nil)
		rUpperBound2.Mul(&rUpperBound2, eight)
		rUpperBound = Min(rUpperBound, &rUpperBound2)
	}
	return rUpperBound
}

// Returns the least r such that o_r(n) > ceil(lg(n))^2 >= ceil(lg(n)^2).
func calculateAKSModulus(n *big.Int) *big.Int {
	one := big.NewInt(1)
	two := big.NewInt(2)

	ceilLgNSq := big.NewInt(int64(n.BitLen()))
	ceilLgNSq.Mul(ceilLgNSq, ceilLgNSq)
	var r big.Int
	r.Add(ceilLgNSq, two)
	rUpperBound := calculateAKSModulusUpperBound(n)
	for ; r.Cmp(rUpperBound) < 0; r.Add(&r, one) {
		var gcd big.Int
		gcd.GCD(nil, nil, n, &r)
		if gcd.Cmp(one) != 0 {
			continue
		}
		o := CalculateMultiplicativeOrder(n, &r)
		if o.Cmp(ceilLgNSq) > 0 {
			return &r
		}
	}

	panic("Could not find AKS modulus")
}

// Returns floor(sqrt(Phi(r))) * ceil(lg(n)) + 1 > floor(sqrt(Phi(r))) * lg(n).
func calculateAKSUpperBound(n, r *big.Int) *big.Int {
	one := big.NewInt(1)
	two := big.NewInt(2)

	M := CalculateEulerPhi(r)
	M = FloorRoot(M, two)
	M.Mul(M, big.NewInt(int64(n.BitLen())))
	M.Add(M, one)
	return M
}

func main() {
	numCPU := runtime.NumCPU()
	runtime.GOMAXPROCS(numCPU)

	if len(os.Args) < 2 {
		fmt.Fprintf(os.Stderr, "%s [number]\n", os.Args[0])
		os.Exit(-1)
	}

	var n big.Int
	_, parsed := n.SetString(os.Args[1], 10)
	if !parsed {
		fmt.Fprintf(os.Stderr, "could not parse %s\n", os.Args[1])
		os.Exit(-1)
	}
	if n.Cmp(big.NewInt(2)) < 0 {
		fmt.Fprintf(os.Stderr, "n must be >= 2\n")
		os.Exit(-1)
	}

	r := calculateAKSModulus(&n)
	// TODO(akalin): Check for factors less than M.
	M := calculateAKSUpperBound(&n, r)
	fmt.Printf("n = %v, r = %v, M = %v\n", &n, r, M)
	a := getAKSWitness(&n, r, M, numCPU)
	if a != nil {
		fmt.Printf("n is composite with AKS witness %v\n", a)
	} else {
		fmt.Printf("n is prime\n")
	}
}
<commit_msg>Trial-divide up to M<commit_after>package main

import "fmt"
import "math/big"
import "os"
import "runtime"

// Returns whether (X + a)^n = X^n + a mod (n, X^r - 1).
func isAKSWitness(n, r, a *big.Int) bool {
	reduceAKS := func(p *IntPoly) {
		p.PowMod(p, r).Mod(p, n)
	}

	zero := big.NewInt(0)
	one := big.NewInt(1)
	lhs := NewIntPoly([][2]*big.Int{{a, zero}, {one, one}})
	lhs.GenPow(lhs, n, reduceAKS)

	rhs := NewIntPoly([][2]*big.Int{{a, zero}, {one, n}})
	reduceAKS(rhs)

	isWitness := !lhs.Eq(rhs)
	return isWitness
}

// Returns the first AKS witness of n with the parameters r and M, or
// nil if there isn't one.
func getFirstAKSWitness(n, r, M *big.Int) *big.Int {
	for a := big.NewInt(1); a.Cmp(M) < 0; a.Add(a, big.NewInt(1)) {
		fmt.Printf("Testing %v (M = %v)...\n", a, M)
		if isWitness := isAKSWitness(n, r, a); isWitness {
			return a
		}
	}
	return nil
}

// Holds the result of an AKS witness test.
type witnessResult struct {
	a         *big.Int
	isWitness bool
}

// Tests all numbers received on numberCh if they are witnesses of n
// with parameter r. Sends the results to resultCh.
func testAKSWitnesses(
	n, r *big.Int,
	numberCh chan *big.Int,
	resultCh chan witnessResult) {
	for a := range numberCh {
		fmt.Printf("Testing %v...\n", a)
		isWitness := isAKSWitness(n, r, a)
		fmt.Printf("Finished testing %v (isWitness=%t)\n",
			a, isWitness)
		resultCh <- witnessResult{a, isWitness}
	}
}

// Returns an AKS witness of n with the parameters r and M, or nil if
// there isn't one. Tests up to maxOutstanding numbers at once.
func getAKSWitness(n, r, M *big.Int, maxOutstanding int) *big.Int {
	numberCh := make(chan *big.Int, maxOutstanding)
	defer close(numberCh)
	resultCh := make(chan witnessResult, maxOutstanding)
	for i := 0; i < maxOutstanding; i++ {
		go testAKSWitnesses(n, r, numberCh, resultCh)
	}

	// Send off all numbers for testing, draining any results that
	// come in while we're doing so.
	tested := big.NewInt(1)
	for i := big.NewInt(1); i.Cmp(M) < 0; {
		select {
		case result := <-resultCh:
			tested.Add(tested, big.NewInt(1))
			fmt.Printf("%v isWitness=%t\n",
				result.a, result.isWitness)
			if result.isWitness {
				return result.a
			}
		default:
			var a big.Int
			a.Set(i)
			numberCh <- &a
			i.Add(i, big.NewInt(1))
		}
	}

	// Drain any remaining results.
	for tested.Cmp(M) < 0 {
		result := <-resultCh
		tested.Add(tested, big.NewInt(1))
		fmt.Printf("%v isWitness=%t\n", result.a, result.isWitness)
		if result.isWitness {
			return result.a
		}
	}

	return nil
}

// Returns an upper bound for r such that o_r(n) > ceil(lg(n))^2 that
// is polylog in n.
func calculateAKSModulusUpperBound(n *big.Int) *big.Int {
	two := big.NewInt(2)
	three := big.NewInt(3)
	five := big.NewInt(5)
	eight := big.NewInt(8)

	// Calculate max(ceil(lg(n))^5, 3).
	ceilLgN := big.NewInt(int64(n.BitLen()))
	rUpperBound := &big.Int{}
	rUpperBound.Exp(ceilLgN, five, nil)
	rUpperBound = Max(rUpperBound, three)

	var nMod8 big.Int
	nMod8.Mod(n, eight)
	if (nMod8.Cmp(three) == 0) || (nMod8.Cmp(five) == 0) {
		// Calculate 8*ceil(lg(n))^2.
		var rUpperBound2 big.Int
		rUpperBound2.Exp(ceilLgN, two, nil)
		rUpperBound2.Mul(&rUpperBound2, eight)
		rUpperBound = Min(rUpperBound, &rUpperBound2)
	}
	return rUpperBound
}

// Returns the least r such that o_r(n) > ceil(lg(n))^2 >= ceil(lg(n)^2).
func calculateAKSModulus(n *big.Int) *big.Int {
	one := big.NewInt(1)
	two := big.NewInt(2)

	ceilLgNSq := big.NewInt(int64(n.BitLen()))
	ceilLgNSq.Mul(ceilLgNSq, ceilLgNSq)
	var r big.Int
	r.Add(ceilLgNSq, two)
	rUpperBound := calculateAKSModulusUpperBound(n)
	for ; r.Cmp(rUpperBound) < 0; r.Add(&r, one) {
		var gcd big.Int
		gcd.GCD(nil, nil, n, &r)
		if gcd.Cmp(one) != 0 {
			continue
		}
		o := CalculateMultiplicativeOrder(n, &r)
		if o.Cmp(ceilLgNSq) > 0 {
			return &r
		}
	}

	panic("Could not find AKS modulus")
}

// Returns floor(sqrt(Phi(r))) * ceil(lg(n)) + 1 > floor(sqrt(Phi(r))) * lg(n).
func calculateAKSUpperBound(n, r *big.Int) *big.Int {
	one := big.NewInt(1)
	two := big.NewInt(2)

	M := CalculateEulerPhi(r)
	M = FloorRoot(M, two)
	M.Mul(M, big.NewInt(int64(n.BitLen())))
	M.Add(M, one)
	return M
}

// Returns the first factor of n less than M.
func getFirstFactorBelow(n, M *big.Int) *big.Int {
	var factor *big.Int
	var mMinusOne big.Int
	mMinusOne.Sub(M, big.NewInt(1))
	TrialDivide(n, func(q, e *big.Int) bool {
		if q.Cmp(M) < 0 && q.Cmp(n) < 0 {
			factor = q
		}
		return false
	}, &mMinusOne)
	return factor
}

func main() {
	numCPU := runtime.NumCPU()
	runtime.GOMAXPROCS(numCPU)

	if len(os.Args) < 2 {
		fmt.Fprintf(os.Stderr, "%s [number]\n", os.Args[0])
		os.Exit(-1)
	}

	var n big.Int
	_, parsed := n.SetString(os.Args[1], 10)
	if !parsed {
		fmt.Fprintf(os.Stderr, "could not parse %s\n", os.Args[1])
		os.Exit(-1)
	}

	two := big.NewInt(2)

	if n.Cmp(two) < 0 {
		fmt.Fprintf(os.Stderr, "n must be >= 2\n")
		os.Exit(-1)
	}

	r := calculateAKSModulus(&n)
	M := calculateAKSUpperBound(&n, r)
	fmt.Printf("n = %v, r = %v, M = %v\n", &n, r, M)
	factor := getFirstFactorBelow(&n, M)
	if factor != nil {
		fmt.Printf("n has factor %v\n", factor)
		return
	}

	fmt.Printf("n has no factor less than %v\n", M)
	sqrtN := FloorRoot(&n, two)
	if M.Cmp(sqrtN) > 0 {
		fmt.Printf("%v is greater than sqrt(%v), so %v is prime\n",
			M, &n, &n)
		return
	}

	a := getAKSWitness(&n, r, M, numCPU)
	if a != nil {
		fmt.Printf("n is composite with AKS witness %v\n", a)
	} else {
		fmt.Printf("n is prime\n")
	}
}
<|endoftext|>


<commit_before>package sloth

import (
  "encoding/json"
  "fmt"
  "net/http"
  "net/url"
  "time"
)

const (
  GET    = "GET"
  POST   = "POST"
  PUT    = "PUT"
  DELETE = "DELETE"
)

// Ensures type checking during json marshalling
var _ json.Marshaler = (*RawMessage)(nil)

// type RestRequest struct {
//   *http.Request
// }

// type RestResponse struct {
//   *http.Response
// }

type RestResponse (int, interface{})

type RestError interface {
  Error() string
}

type Service interface {
  uri, contentType, string

  // FIXME - move to resource level for greater flexibility
  MarshalContent(data)
  RequestHandler(resource RestResource) http.HandlerFunc
}

func (service *Service) MarshalContent(data) {
  return data
}

type JsonService interface {
  contentType := 'application/json'

  Service
}

func (service *JsonService) MarshalContent(data) {
  return json.Marshal(data)
}

func (service *Service) RequestHandler(resource RestResource) http.HandlerFunc {
  return func(rw http.ResponseWriter, request *http.Request) {
    var data interface{}
    var code int

    request.ParseForm()
    method := request.Method
    values := request.Form

    switch method {
    case GET:
      code, data = resource.Get(values)
    case POST:
      code, data = resource.Post(values)
    case PUT:
      code, data = resource.Put(values)
    case DELETE:
      code, data = resource.Delete(values)
    default:
      service.Abort(rw, 405)
      return
    }

    content, err := service.MarshalContent(data)

    if err != nil {
      // log - failed to marshal content
      service.Abort(rw, 500)
    }

    rw.WriteHeader(code)
    rw.Write(content)
  }
}

func (service *Service) AddResource(resource RestResource, path string) {
  http.HandleFunc(path, service.requestHandler(resource))
}

func (service *Service) Start(port int) {
  portStr := fmt.Sprintf(":%d", port)

  http.ListenAndServe(portStr, nil)
}

func (service *Service) Abort(rw http.ResponseWriter, statusCode int) {
  rw.WriteHeader(statusCode)
}

type RestResource interface {
  baseUrl string

  Get(values url.Values)    RestResponse
  Post(values url.Values)   RestResponse
  Put(values url.Values)    RestResponse
  Delete(values url.Values) RestResponse

  all()    RestResource
  byId(id) RestResource
}

// default GET (remove eventually)
func (resource *RestResource) Get(values url.Values) RestResponse {
  data := map[string]string{"hello": "world"}
  return 200, data
}

type RestAPI interface {
  host, base string

  resources []RestResource
}
<commit_msg>notes<commit_after>// http://golang.org/pkg/net/http/

package sloth

import (
  "encoding/json"
  "fmt"
  "net/http"
  "net/url"
  "time"
)

const (
  GET    = "GET"
  POST   = "POST"
  PUT    = "PUT"
  DELETE = "DELETE"
)

// Ensures type checking during json marshalling
var _ json.Marshaler = (*RawMessage)(nil)

type RestResponse (int, interface{})

type RestError interface {
  Error() string
}

type Service interface {
  uri, contentType, string

  // FIXME - move to resource level for greater flexibility
  MarshalContent(data)
  RequestHandler(resource RestResource) http.HandlerFunc
}

func (service *Service) MarshalContent(data) {
  return data
}

type JsonService interface {
  contentType := 'application/json'

  Service
}

func (service *JsonService) MarshalContent(data) {
  return json.Marshal(data)
}

func (service *Service) RequestHandler(resource RestResource) http.HandlerFunc {
  return func(rw http.ResponseWriter, request *http.Request) {
    var data interface{}
    var code int

    request.ParseForm()
    method := request.Method
    values := request.Form

    switch method {
    case GET:
      code, data = resource.Get(values)
    case POST:
      code, data = resource.Post(values)
    case PUT:
      code, data = resource.Put(values)
    case DELETE:
      code, data = resource.Delete(values)
    default:
      service.Abort(rw, 405)
      return
    }

    content, err := service.MarshalContent(data)

    if err != nil {
      // log - failed to marshal content
      service.Abort(rw, 500)
    }

    rw.WriteHeader(code)
    rw.Write(content)
  }
}

func (service *Service) AddResource(resource RestResource, path string) {
  http.HandleFunc(path, service.requestHandler(resource))
}

func (service *Service) Start(port int) {
  portStr := fmt.Sprintf(":%d", port)

  http.ListenAndServe(portStr, nil)
}

func (service *Service) Abort(rw http.ResponseWriter, statusCode int) {
  rw.WriteHeader(statusCode)
}

type RestResource interface {
  baseUrl string

  Get(values url.Values)    RestResponse
  Post(values url.Values)   RestResponse
  Put(values url.Values)    RestResponse
  Delete(values url.Values) RestResponse

  all()    RestResource
  byId(id) RestResource
}

// default GET (remove eventually)
func (resource *RestResource) Get(values url.Values) RestResponse {
  data := map[string]string{"hello": "world"}
  return StatusOK, data
}

type RestAPI interface {
  host, base string

  resources []RestResource
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
)

const (
	API_URL = "https://api.github.com"
	GH_URL  = "https://github.com"
)

/* create a new request that sends the auth token */
func NewAuthRequest(method, url, bodyType, token string, headers map[string]string, body io.Reader) (*http.Request, error) {
	vprintln("creating request:", method, url, bodyType, token)

	req, err := http.NewRequest(method, url, body)
	if err != nil {
		return nil, err
	}

	if body != nil {
		switch v := body.(type) {
		case *os.File:
			/* when uploading a file, you need to either excplicitly set the
			 * Content-Length header or send a chunked request. Since the
			 * github upload server doesn't accept chunked encoding, we have
			 * to set the size of the file manually. If the file is actually
			 * stdin, this won't work, and we'll just quit for now.
			 *
			 * TODO: if stdin, read everything into a byte buffer, take the
			 * length and send it. */
			off, err := GetFileSize(v)
			if err != nil {
				return nil, err
			}

			req.ContentLength = off
			vprintln("setting content-length to", off)
		}
	}

	if bodyType != "" {
		req.Header.Set("Content-Type", bodyType)
	}
	req.Header.Set("Authorization", fmt.Sprintf("token %s", token))

	for k, v := range headers {
		req.Header.Set(k, v)
	}

	return req, nil
}

func DoAuthRequest(method, url, bodyType, token string, headers map[string]string, body io.Reader) (*http.Response, error) {
	req, err := NewAuthRequest(method, url, bodyType, token, headers, body)
	if err != nil {
		return nil, err
	}

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}

	return resp, nil
}

func GithubGet(uri string, v interface{}) error {
	resp, err := http.Get(ApiURL() + uri)
	if err != nil {
		return fmt.Errorf("could not fetch releases, %v", err)
	}
	defer resp.Body.Close()

	vprintln("GET", ApiURL()+uri, "->", resp)

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("github did not response with 200 OK but with %v", resp.Status)
	}

	if VERBOSITY == 0 {
		if err = json.NewDecoder(resp.Body).Decode(v); err != nil {
			return fmt.Errorf("could not unmarshall JSON into Release struct, %v", err)
		}
	} else {
		body, err := ioutil.ReadAll(resp.Body)
		vprintln("BODY", string(body))

		if err = json.Unmarshal(body, v); err != nil {
			return fmt.Errorf("could not unmarshall JSON into Release struct, %v", err)
		}
	}

	return nil
}

func ApiURL() string {
	if "" == EnvApiEndpoint {
		return API_URL
	} else {
		return EnvApiEndpoint
	}
}
<commit_msg>api: support uploading from stdin/pipes<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
)

const (
	API_URL = "https://api.github.com"
	GH_URL  = "https://github.com"
)

// materializeFile takes a physical file or stream (named pipe, user input,
// ...) and returns an io.Reader and the number of bytes that can be read
// from it.
func materializeFile(f *os.File) (io.Reader, int64, error) {
	fi, err := f.Stat()
	if err != nil {
		return nil, 0, err
	}

	// If the file is actually a char device (like user typed input)
	// or a named pipe (like a streamed in file), buffer it up.
	//
	// When uploading a file, you need to either explicitly set the
	// Content-Length header or send a chunked request. Since the
	// github upload server doesn't accept chunked encoding, we have
	// to set the size of the file manually. Since a stream doesn't have a
	// predefined length, it's read entirely into a byte buffer.
	if fi.Mode()&(os.ModeCharDevice|os.ModeNamedPipe) == 1 {
		vprintln("input was a stream, buffering up")

		var buf bytes.Buffer
		n, err := buf.ReadFrom(f)
		if err != nil {
			return nil, 0, errors.New("req: could not buffer up input stream: " + err.Error())
		}
		return &buf, n, err
	}

	// We know the os.File is most likely an actual file now.
	n, err := GetFileSize(f)
	return f, n, err
}

/* create a new request that sends the auth token */
func NewAuthRequest(method, url, bodyType, token string, headers map[string]string, body io.Reader) (*http.Request, error) {
	vprintln("creating request:", method, url, bodyType, token)

	var n int64 // content length
	var err error
	if f, ok := body.(*os.File); ok {
		// Retrieve the content-length and buffer up if necessary.
		body, n, err = materializeFile(f)
		if err != nil {
			return nil, err
		}
	}

	req, err := http.NewRequest(method, url, body)
	if err != nil {
		return nil, err
	}

	if n != 0 {
		vprintln("setting content-length to", n)
		req.ContentLength = n
	}

	if bodyType != "" {
		req.Header.Set("Content-Type", bodyType)
	}
	req.Header.Set("Authorization", fmt.Sprintf("token %s", token))

	for k, v := range headers {
		req.Header.Set(k, v)
	}

	return req, nil
}

func DoAuthRequest(method, url, bodyType, token string, headers map[string]string, body io.Reader) (*http.Response, error) {
	req, err := NewAuthRequest(method, url, bodyType, token, headers, body)
	if err != nil {
		return nil, err
	}

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}

	return resp, nil
}

func GithubGet(uri string, v interface{}) error {
	resp, err := http.Get(ApiURL() + uri)
	if err != nil {
		return fmt.Errorf("could not fetch releases, %v", err)
	}
	defer resp.Body.Close()

	vprintln("GET", ApiURL()+uri, "->", resp)

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("github did not response with 200 OK but with %v", resp.Status)
	}

	if VERBOSITY == 0 {
		if err = json.NewDecoder(resp.Body).Decode(v); err != nil {
			return fmt.Errorf("could not unmarshall JSON into Release struct, %v", err)
		}
	} else {
		body, err := ioutil.ReadAll(resp.Body)
		vprintln("BODY", string(body))

		if err = json.Unmarshal(body, v); err != nil {
			return fmt.Errorf("could not unmarshall JSON into Release struct, %v", err)
		}
	}

	return nil
}

func ApiURL() string {
	if "" == EnvApiEndpoint {
		return API_URL
	} else {
		return EnvApiEndpoint
	}
}
<|endoftext|>