<commit_before>// Copyright 2019 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bootstrap

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"

	"istio.io/pkg/log"

	"istio.io/istio/pilot/pkg/features"
	"istio.io/istio/security/pkg/k8s/chiron"
)

const (
	// defaultCertGracePeriodRatio is the default length of certificate rotation grace period,
	// configured as the ratio of the certificate TTL.
	defaultCertGracePeriodRatio = 0.5

	// defaultMinCertGracePeriod is the default minimum grace period for workload cert rotation.
	defaultMinCertGracePeriod = 10 * time.Minute

	// Default CA certificate path
	// Currently, custom CA path is not supported; no API to get custom CA cert yet.
	defaultCACertPath = "./var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
)

var (
	// dnsCertDir is the location to save generated DNS certificates.
	// TODO: we can probably avoid saving, but will require deeper changes.
	dnsCertDir  = "./var/run/secrets/istio-dns"
	dnsKeyFile  = "./" + filepath.Join(dnsCertDir, "key.pem")
	dnsCertFile = "./" + filepath.Join(dnsCertDir, "cert-chain.pem")

	KubernetesCAProvider = "kubernetes"
	IstiodCAProvider     = "istiod"
)

// CertController can create certificates signed by K8S server.
func (s *Server) initCertController(args *PilotArgs) error {
	var err error
	var secretNames, dnsNames, namespaces []string

	meshConfig := s.environment.Mesh()
	if meshConfig.GetCertificates() == nil || len(meshConfig.GetCertificates()) == 0 {
		// TODO: if the provider is set to Citadel, use that instead of k8s so the API is still preserved.
		log.Info("No certificates specified, skipping K8S DNS certificate controller")
		return nil
	}

	k8sClient := s.kubeClient
	for _, c := range meshConfig.GetCertificates() {
		name := strings.Join(c.GetDnsNames(), ",")
		if len(name) == 0 { // must have a DNS name
			continue
		}
		if len(c.GetSecretName()) > 0 {
			// Chiron will generate the key and certificate and save them in a secret
			secretNames = append(secretNames, c.GetSecretName())
			dnsNames = append(dnsNames, name)
			namespaces = append(namespaces, args.Namespace)
		}
	}

	// Provision and manage the certificates for non-Pilot services.
	// If services are empty, the certificate controller will do nothing.
	s.certController, err = chiron.NewWebhookController(defaultCertGracePeriodRatio, defaultMinCertGracePeriod,
		k8sClient.CoreV1(), k8sClient.AdmissionregistrationV1beta1(), k8sClient.CertificatesV1beta1(),
		defaultCACertPath, secretNames, dnsNames, namespaces)
	if err != nil {
		return fmt.Errorf("failed to create certificate controller: %v", err)
	}
	s.addStartFunc(func(stop <-chan struct{}) error {
		go func() {
			// Run Chiron to manage the lifecycles of certificates
			s.certController.Run(stop)
		}()

		return nil
	})

	return nil
}

// initDNSCerts will create the certificates to be used by Istiod GRPC server and webhooks.
// If the certificate creation fails - for example no support in K8S - returns an error.
// Will use the mesh.yaml DiscoveryAddress to find the default expected address of the control plane,
// with an environment variable allowing override.
//
// Controlled by features.IstiodService env variable, which defines the name of the service to use in the DNS
// cert, or empty for disabling this feature.
//
// TODO: If the discovery address in mesh.yaml is set to port 15012 (XDS-with-DNS-certs) and the name
// matches the k8s namespace, failure to start DNS server is a fatal error.
func (s *Server) initDNSCerts(hostname string) error {
	parts := strings.Split(hostname, ".")
	if len(parts) < 2 {
		return fmt.Errorf("invalid hostname %s, should contain at least service name and namespace", hostname)
	}
	// Names in the Istiod cert - support the old service names as well.
	// The first is the recommended one, also used by Apiserver for webhooks.
	names := []string{hostname, "istiod.istio-system.svc", "istio-pilot.istio-system.svc"}

	var certChain, keyPEM []byte
	var err error
	if features.PilotCertProvider.Get() == KubernetesCAProvider {
		log.Infof("Generating K8S-signed cert for %v", names)
		certChain, keyPEM, _, err = chiron.GenKeyCertK8sCA(s.kubeClient.CertificatesV1beta1().CertificateSigningRequests(),
			strings.Join(names, ","), parts[0]+".csr.secret", parts[1], defaultCACertPath)
	} else if features.PilotCertProvider.Get() == IstiodCAProvider {
		signingKeyFile := path.Join(localCertDir.Get(), "ca-key.pem")
		// check if signing key file exists the cert dir
		if _, err := os.Stat(signingKeyFile); err != nil {
			log.Infof("No plugged-in cert at %v; self-signed cert is used", signingKeyFile)
			log.Infof("Generating istiod-signed cert for %v", names)
			certChain, keyPEM, err = s.ca.GenKeyCert(names, SelfSignedCACertTTL.Get())

			// When Citadel is configured to use self-signed certs, keep a local copy so other
			// components can load it via file (e.g. webhook config controller).
			if err := os.MkdirAll(dnsCertDir, 0700); err != nil {
				return err
			}
			// We have direct access to the self-signed
			internalSelfSignedRootPath := path.Join(dnsCertDir, "self-signed-root.pem")

			rootCert := s.ca.GetCAKeyCertBundle().GetRootCertPem()
			if err = ioutil.WriteFile(internalSelfSignedRootPath, rootCert, 0600); err != nil {
				return err
			}

			s.addStartFunc(func(stop <-chan struct{}) error {
				go func() {
					for {
						select {
						case <-stop:
							return
						case <-time.After(namespaceResyncPeriod):
							newRootCert := s.ca.GetCAKeyCertBundle().GetRootCertPem()
							if !bytes.Equal(rootCert, newRootCert) {
								rootCert = newRootCert
								if err = ioutil.WriteFile(internalSelfSignedRootPath, rootCert, 0600); err != nil {
									log.Errorf("Failed to update local copy of self-signed root: %v", err)
								} else {
									log.Info("Updated local copy of self-signed root")
								}
							}
						}
					}
				}()
				return nil
			})
			s.caBundlePath = internalSelfSignedRootPath
		} else {
			log.Infof("Use plugged-in cert at %v", signingKeyFile)
			s.caBundlePath = path.Join(localCertDir.Get(), "root-cert.pem")
		}

	} else {
		log.Infof("User specified cert provider: %v", features.PilotCertProvider.Get())
		return nil
	}
	if err != nil {
		return err
	}

	// Save the certificates to ./var/run/secrets/istio-dns - this is needed since most of the code we currently
	// use to start grpc and webhooks is based on files. This is a memory-mounted dir.
	if err := os.MkdirAll(dnsCertDir, 0700); err != nil {
		return err
	}
	err = ioutil.WriteFile(dnsKeyFile, keyPEM, 0600)
	if err != nil {
		return err
	}
	err = ioutil.WriteFile(dnsCertFile, certChain, 0600)
	if err != nil {
		return err
	}
	log.Infoa("DNS certificates created in ", dnsCertDir)
	return nil
}
<commit_msg>revert portions of  clarify logs for init DNS certs #21699 (#21724)<commit_after>// Copyright 2019 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bootstrap

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"

	"istio.io/pkg/log"

	"istio.io/istio/pilot/pkg/features"
	"istio.io/istio/security/pkg/k8s/chiron"
)

const (
	// defaultCertGracePeriodRatio is the default length of certificate rotation grace period,
	// configured as the ratio of the certificate TTL.
	defaultCertGracePeriodRatio = 0.5

	// defaultMinCertGracePeriod is the default minimum grace period for workload cert rotation.
	defaultMinCertGracePeriod = 10 * time.Minute

	// Default CA certificate path
	// Currently, custom CA path is not supported; no API to get custom CA cert yet.
	defaultCACertPath = "./var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
)

var (
	// dnsCertDir is the location to save generated DNS certificates.
	// TODO: we can probably avoid saving, but will require deeper changes.
	dnsCertDir  = "./var/run/secrets/istio-dns"
	dnsKeyFile  = "./" + filepath.Join(dnsCertDir, "key.pem")
	dnsCertFile = "./" + filepath.Join(dnsCertDir, "cert-chain.pem")

	KubernetesCAProvider = "kubernetes"
	IstiodCAProvider     = "istiod"
)

// CertController can create certificates signed by K8S server.
func (s *Server) initCertController(args *PilotArgs) error {
	var err error
	var secretNames, dnsNames, namespaces []string

	meshConfig := s.environment.Mesh()
	if meshConfig.GetCertificates() == nil || len(meshConfig.GetCertificates()) == 0 {
		// TODO: if the provider is set to Citadel, use that instead of k8s so the API is still preserved.
		log.Info("No certificates specified, skipping K8S DNS certificate controller")
		return nil
	}

	k8sClient := s.kubeClient
	for _, c := range meshConfig.GetCertificates() {
		name := strings.Join(c.GetDnsNames(), ",")
		if len(name) == 0 { // must have a DNS name
			continue
		}
		if len(c.GetSecretName()) > 0 {
			// Chiron will generate the key and certificate and save them in a secret
			secretNames = append(secretNames, c.GetSecretName())
			dnsNames = append(dnsNames, name)
			namespaces = append(namespaces, args.Namespace)
		}
	}

	// Provision and manage the certificates for non-Pilot services.
	// If services are empty, the certificate controller will do nothing.
	s.certController, err = chiron.NewWebhookController(defaultCertGracePeriodRatio, defaultMinCertGracePeriod,
		k8sClient.CoreV1(), k8sClient.AdmissionregistrationV1beta1(), k8sClient.CertificatesV1beta1(),
		defaultCACertPath, secretNames, dnsNames, namespaces)
	if err != nil {
		return fmt.Errorf("failed to create certificate controller: %v", err)
	}
	s.addStartFunc(func(stop <-chan struct{}) error {
		go func() {
			// Run Chiron to manage the lifecycles of certificates
			s.certController.Run(stop)
		}()

		return nil
	})

	return nil
}

// initDNSCerts will create the certificates to be used by Istiod GRPC server and webhooks.
// If the certificate creation fails - for example no support in K8S - returns an error.
// Will use the mesh.yaml DiscoveryAddress to find the default expected address of the control plane,
// with an environment variable allowing override.
//
// Controlled by features.IstiodService env variable, which defines the name of the service to use in the DNS
// cert, or empty for disabling this feature.
//
// TODO: If the discovery address in mesh.yaml is set to port 15012 (XDS-with-DNS-certs) and the name
// matches the k8s namespace, failure to start DNS server is a fatal error.
func (s *Server) initDNSCerts(hostname string) error {
	parts := strings.Split(hostname, ".")
	if len(parts) < 2 {
		return fmt.Errorf("invalid hostname %s, should contain at least service name and namespace", hostname)
	}
	// Names in the Istiod cert - support the old service names as well.
	// The first is the recommended one, also used by Apiserver for webhooks.
	names := []string{hostname, "istiod.istio-system.svc", "istio-pilot.istio-system.svc"}

	var certChain, keyPEM []byte
	var err error
	if features.PilotCertProvider.Get() == KubernetesCAProvider {
		log.Infof("Generating K8S-signed cert for %v", names)
		certChain, keyPEM, _, err = chiron.GenKeyCertK8sCA(s.kubeClient.CertificatesV1beta1().CertificateSigningRequests(),
			strings.Join(names, ","), parts[0]+".csr.secret", parts[1], defaultCACertPath)
	} else if features.PilotCertProvider.Get() == IstiodCAProvider {
		log.Infof("Generating istiod-signed cert for %v", names)
		certChain, keyPEM, err = s.ca.GenKeyCert(names, SelfSignedCACertTTL.Get())

		signingKeyFile := path.Join(localCertDir.Get(), "ca-key.pem")
		// check if signing key file exists the cert dir
		if _, err := os.Stat(signingKeyFile); err != nil {
			log.Infof("No plugged-in cert at %v; self-signed cert is used", signingKeyFile)

			// When Citadel is configured to use self-signed certs, keep a local copy so other
			// components can load it via file (e.g. webhook config controller).
			if err := os.MkdirAll(dnsCertDir, 0700); err != nil {
				return err
			}
			// We have direct access to the self-signed
			internalSelfSignedRootPath := path.Join(dnsCertDir, "self-signed-root.pem")

			rootCert := s.ca.GetCAKeyCertBundle().GetRootCertPem()
			if err = ioutil.WriteFile(internalSelfSignedRootPath, rootCert, 0600); err != nil {
				return err
			}

			s.addStartFunc(func(stop <-chan struct{}) error {
				go func() {
					for {
						select {
						case <-stop:
							return
						case <-time.After(namespaceResyncPeriod):
							newRootCert := s.ca.GetCAKeyCertBundle().GetRootCertPem()
							if !bytes.Equal(rootCert, newRootCert) {
								rootCert = newRootCert
								if err = ioutil.WriteFile(internalSelfSignedRootPath, rootCert, 0600); err != nil {
									log.Errorf("Failed to update local copy of self-signed root: %v", err)
								} else {
									log.Info("Updated local copy of self-signed root")
								}
							}
						}
					}
				}()
				return nil
			})
			s.caBundlePath = internalSelfSignedRootPath
		} else {
			log.Infof("Use plugged-in cert at %v", signingKeyFile)
			s.caBundlePath = path.Join(localCertDir.Get(), "root-cert.pem")
		}

	} else {
		log.Infof("User specified cert provider: %v", features.PilotCertProvider.Get())
		return nil
	}
	if err != nil {
		return err
	}

	// Save the certificates to ./var/run/secrets/istio-dns - this is needed since most of the code we currently
	// use to start grpc and webhooks is based on files. This is a memory-mounted dir.
	if err := os.MkdirAll(dnsCertDir, 0700); err != nil {
		return err
	}
	err = ioutil.WriteFile(dnsKeyFile, keyPEM, 0600)
	if err != nil {
		return err
	}
	err = ioutil.WriteFile(dnsCertFile, certChain, 0600)
	if err != nil {
		return err
	}
	log.Infoa("DNS certificates created in ", dnsCertDir)
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2019 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !privileged_tests

package alignchecker

import (
	"reflect"
	"testing"

	. "gopkg.in/check.v1"
)

type AlignCheckerSuite struct{}

var _ = Suite(&AlignCheckerSuite{})

func Test(t *testing.T) {
	TestingT(t)
}

const path = "testdata/bpf_foo.o"

type foo struct {
	ipv6 [4]uint32 `align:"$union0"`
	misc uint32    `align:"$union1"`
	f    uint8     `align:"family"`
	pad4 uint8     `align:"pad4"`
	pad5 uint16    `align:"pad5"`
}

type fooInvalidSize struct {
	ipv6 uint32
}

type fooInvalidOffset struct {
	ipv6 [4]uint32 `align:"$union0"`
	misc uint32    `align:"$union1"`
	f    uint16    `align:"family"`
	pad4 uint8     `align:"pad4"`
	pad5 uint8     `align:"pad5"`
}

type toCheck map[string]reflect.Type

func (t *AlignCheckerSuite) TestCheckStructAlignments(c *C) {
	err := CheckStructAlignments(path, toCheck{"foo": reflect.TypeOf(foo{})})
	c.Assert(err, IsNil)

	err = CheckStructAlignments(path, toCheck{"foo": reflect.TypeOf(fooInvalidSize{})})
	c.Assert(err, ErrorMatches,
		`*.fooInvalidSize\(4\) size does not match foo\(24\)`)

	err = CheckStructAlignments(path, toCheck{"foo": reflect.TypeOf(fooInvalidOffset{})})
	c.Assert(err, ErrorMatches,
		`*.fooInvalidOffset.pad4 offset\(22\) does not match foo.pad4\(21\)`)

	err = CheckStructAlignments(path, toCheck{"bar": reflect.TypeOf(foo{})})
	c.Assert(err, ErrorMatches, "C struct bar not found")
}
<commit_msg>alignchecker: Streamline tests<commit_after>// Copyright 2019 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !privileged_tests

package alignchecker

import (
	"reflect"
	"testing"

	. "gopkg.in/check.v1"
)

type AlignCheckerSuite struct{}

var _ = Suite(&AlignCheckerSuite{})

func Test(t *testing.T) {
	TestingT(t)
}

const path = "testdata/bpf_foo.o"

type foo struct {
	ipv6 [4]uint32 `align:"$union0"`
	misc uint32    `align:"$union1"`
	f    uint8     `align:"family"`
	pad4 uint8     `align:"pad4"`
	pad5 uint16    `align:"pad5"`
}

type fooInvalidSize struct {
	ipv6 uint32
}

type fooInvalidOffset struct {
	ipv6 [4]uint32 `align:"$union0"`
	misc uint32    `align:"$union1"`
	f    uint16    `align:"family"`
	pad4 uint8     `align:"pad4"`
	pad5 uint8     `align:"pad5"`
}

type toCheck map[string]reflect.Type

func (t *AlignCheckerSuite) TestCheckStructAlignments(c *C) {
	testCases := []struct {
		cName   string
		goTypes reflect.Type
		err     string
	}{
		{
			"foo",
			reflect.TypeOf(foo{}),
			"",
		},
		{
			"foo",
			reflect.TypeOf(fooInvalidSize{}),
			`*.fooInvalidSize\(4\) size does not match foo\(24\)`,
		},
		{
			"foo",
			reflect.TypeOf(fooInvalidOffset{}),
			`*.fooInvalidOffset.pad4 offset\(22\) does not match foo.pad4\(21\)`,
		},
		{
			"bar",
			reflect.TypeOf(foo{}),
			"C struct bar not found",
		},
	}

	for _, tt := range testCases {
		err := CheckStructAlignments(path, toCheck{tt.cName: tt.goTypes})
		if tt.err == "" {
			c.Assert(err, IsNil)
		} else {
			c.Assert(err, ErrorMatches, tt.err)
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1

const (
	// If you add a new topology domain here, also consider adding it to the set of default values
	// for the scheduler's --failure-domain command-line argument.
	LabelHostname          = "kubernetes.io/hostname"
	LabelZoneFailureDomain = "failure-domain.beta.kubernetes.io/zone"
	LabelZoneRegion        = "failure-domain.beta.kubernetes.io/region"

	LabelInstanceType = "beta.kubernetes.io/instance-type"

	LabelOS   = "beta.kubernetes.io/os"
	LabelArch = "beta.kubernetes.io/arch"

	// When feature-gate for TaintBasedEvictions=true flag is enabled,
	// TaintNodeNotReady would be automatically added by node controller
	// when node is not ready, and removed when node becomes ready.
	TaintNodeNotReady = "node.alpha.kubernetes.io/notReady"

	// When feature-gate for TaintBasedEvictions=true flag is enabled,
	// TaintNodeUnreachable would be automatically added by node controller
	// when node becomes unreachable (corresponding to NodeReady status ConditionUnknown)
	// and removed when node becomes reachable (NodeReady status ConditionTrue).
	TaintNodeUnreachable = "node.alpha.kubernetes.io/unreachable"
)

// Role labels are applied to Nodes to mark their purpose.  In particular, we
// usually want to distinguish the master, so that we can isolate privileged
// pods and operations.
//
// Originally we relied on not registering the master, on the fact that the
// master was Unschedulable, and on static manifests for master components.
// But we now do register masters in many environments, are generally moving
// away from static manifests (for better manageability), and working towards
// deprecating the unschedulable field (replacing it with taints & tolerations
// instead).
//
// Even with tainting, a label remains the easiest way of making a positive
// selection, so that pods can schedule only to master nodes for example, and
// thus installations will likely define a label for their master nodes.
//
// So that we can recognize master nodes in consequent places though (such as
// kubectl get nodes), we encourage installations to use the well-known labels.
// We define NodeLabelRole, which is the preferred form, but we will also recognize
// other forms that are known to be in widespread use (NodeLabelKubeadmAlphaRole).

const (
	// NodeLabelRole is the preferred label applied to a Node as a hint that it has a particular purpose (defined by the value).
	NodeLabelRole = "kubernetes.io/role"

	// NodeLabelKubeadmAlphaRole is a label that kubeadm applies to a Node as a hint that it has a particular purpose.
	// Use of NodeLabelRole is preferred.
	NodeLabelKubeadmAlphaRole = "kubeadm.alpha.kubernetes.io/role"

	// NodeLabelRoleMaster is the value of a NodeLabelRole or NodeLabelKubeadmAlphaRole label, indicating a master node.
	// A master node typically runs kubernetes system components and will not typically run user workloads.
	NodeLabelRoleMaster = "master"

	// NodeLabelRoleNode is the value of a NodeLabelRole or NodeLabelKubeadmAlphaRole label, indicating a "normal" node,
	// as opposed to a RoleMaster node.
	NodeLabelRoleNode = "node"
)
<commit_msg>`failure-domain` is no longer used in command line arg.<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1

const (
	LabelHostname          = "kubernetes.io/hostname"
	LabelZoneFailureDomain = "failure-domain.beta.kubernetes.io/zone"
	LabelZoneRegion        = "failure-domain.beta.kubernetes.io/region"

	LabelInstanceType = "beta.kubernetes.io/instance-type"

	LabelOS   = "beta.kubernetes.io/os"
	LabelArch = "beta.kubernetes.io/arch"

	// When feature-gate for TaintBasedEvictions=true flag is enabled,
	// TaintNodeNotReady would be automatically added by node controller
	// when node is not ready, and removed when node becomes ready.
	TaintNodeNotReady = "node.alpha.kubernetes.io/notReady"

	// When feature-gate for TaintBasedEvictions=true flag is enabled,
	// TaintNodeUnreachable would be automatically added by node controller
	// when node becomes unreachable (corresponding to NodeReady status ConditionUnknown)
	// and removed when node becomes reachable (NodeReady status ConditionTrue).
	TaintNodeUnreachable = "node.alpha.kubernetes.io/unreachable"
)

// Role labels are applied to Nodes to mark their purpose.  In particular, we
// usually want to distinguish the master, so that we can isolate privileged
// pods and operations.
//
// Originally we relied on not registering the master, on the fact that the
// master was Unschedulable, and on static manifests for master components.
// But we now do register masters in many environments, are generally moving
// away from static manifests (for better manageability), and working towards
// deprecating the unschedulable field (replacing it with taints & tolerations
// instead).
//
// Even with tainting, a label remains the easiest way of making a positive
// selection, so that pods can schedule only to master nodes for example, and
// thus installations will likely define a label for their master nodes.
//
// So that we can recognize master nodes in consequent places though (such as
// kubectl get nodes), we encourage installations to use the well-known labels.
// We define NodeLabelRole, which is the preferred form, but we will also recognize
// other forms that are known to be in widespread use (NodeLabelKubeadmAlphaRole).

const (
	// NodeLabelRole is the preferred label applied to a Node as a hint that it has a particular purpose (defined by the value).
	NodeLabelRole = "kubernetes.io/role"

	// NodeLabelKubeadmAlphaRole is a label that kubeadm applies to a Node as a hint that it has a particular purpose.
	// Use of NodeLabelRole is preferred.
	NodeLabelKubeadmAlphaRole = "kubeadm.alpha.kubernetes.io/role"

	// NodeLabelRoleMaster is the value of a NodeLabelRole or NodeLabelKubeadmAlphaRole label, indicating a master node.
	// A master node typically runs kubernetes system components and will not typically run user workloads.
	NodeLabelRoleMaster = "master"

	// NodeLabelRoleNode is the value of a NodeLabelRole or NodeLabelKubeadmAlphaRole label, indicating a "normal" node,
	// as opposed to a RoleMaster node.
	NodeLabelRoleNode = "node"
)
<|endoftext|>


<commit_before>package generator

import (
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/openshift/origin/pkg/generate/app"
	"github.com/openshift/origin/pkg/generate/dockerfile"
	"github.com/openshift/origin/pkg/generate/errors"
	"github.com/openshift/origin/pkg/generate/git"
	"github.com/openshift/origin/pkg/generate/source"
)

// Flows for BuildStrategyRef
// SourceRef -> BuildStrategyRef
// SourceRef + Docker Context -> BuildStrategyRef
// Docker Context + Parent Image -> BuildStrategyRef
// STI Builder Image -> BuildStrategyRef

// BuildStrategyRefGenerator generates BuildStrategyRef
type BuildStrategyRefGenerator struct {
	gitRepository     git.Repository
	dockerfileFinder  dockerfile.Finder
	dockerfileParser  dockerfile.Parser
	sourceDetectors   source.Detectors
	imageRefGenerator ImageRefGenerator
	resolver          app.Resolver
}

// NewBuildStrategyRefGenerator creates a BuildStrategyRefGenerator
func NewBuildStrategyRefGenerator(sourceDetectors source.Detectors, resolver app.Resolver) *BuildStrategyRefGenerator {
	return &BuildStrategyRefGenerator{
		gitRepository:     git.NewRepository(),
		dockerfileFinder:  dockerfile.NewFinder(),
		dockerfileParser:  dockerfile.NewParser(),
		sourceDetectors:   sourceDetectors,
		imageRefGenerator: NewImageRefGenerator(),
		resolver:          resolver,
	}
}

// FromSourceRef creates a build strategy from a source reference
func (g *BuildStrategyRefGenerator) FromSourceRef(srcRef app.SourceRef) (*app.BuildStrategyRef, error) {

	// Download source locally first if not available
	if len(srcRef.Dir) == 0 {
		if err := g.getSource(&srcRef); err != nil {
			return nil, err
		}
	}

	// Detect a Dockerfile
	context, found, err := g.detectDockerFile(srcRef.Dir)
	if err != nil {
		return nil, err
	}
	if found {
		return g.FromSourceRefAndDockerContext(srcRef, context)
	}

	// Detect a STI repository
	sourceInfo, ok := g.sourceDetectors.DetectSource(srcRef.Dir)
	if !ok {
		return nil, errors.CouldNotDetect
	}
	builderImage, err := g.imageForSourceInfo(sourceInfo)
	if err != nil {
		return nil, err
	}
	return g.FromSTIBuilderImage(builderImage)
}

// FromSourceRefAndDockerContext generates a BuildStrategyRef from a source ref and context path
func (g *BuildStrategyRefGenerator) FromSourceRefAndDockerContext(srcRef app.SourceRef, context string) (*app.BuildStrategyRef, error) {
	// Download source locally first if not available
	if len(srcRef.Dir) == 0 {
		if err := g.getSource(&srcRef); err != nil {
			return nil, err
		}
	}

	if len(context) > 0 {
		srcRef.ContextDir = context
	}

	// Look for Dockerfile in repository
	file, err := os.Open(filepath.Join(srcRef.Dir, context, "Dockerfile"))
	if err != nil {
		return nil, err
	}

	dockerFile, err := g.dockerfileParser.Parse(file)
	if err != nil {
		return nil, err
	}

	parentImageName, ok := dockerFile.GetDirective("FROM")
	if !ok {
		return nil, errors.InvalidDockerfile
	}
	ports, ok := dockerFile.GetDirective("EXPOSE")

	parentRef, err := g.imageRefGenerator.FromNameAndPorts(parentImageName[0], ports)
	if err != nil {
		return nil, err
	}

	return g.FromDockerContextAndParent(parentRef)

}

// FromContextAndParent generates a build strategy ref from a context path and parent image name
func (g *BuildStrategyRefGenerator) FromDockerContextAndParent(parentRef *app.ImageRef) (*app.BuildStrategyRef, error) {
	return &app.BuildStrategyRef{
		IsDockerBuild: true,
		Base:          parentRef,
	}, nil
}

// FromSTIBuilderImage generates a build strategy from a builder image ref
func (g *BuildStrategyRefGenerator) FromSTIBuilderImage(image *app.ImageRef) (*app.BuildStrategyRef, error) {
	return &app.BuildStrategyRef{
		IsDockerBuild: false,
		Base:          image,
	}, nil
}

func (g *BuildStrategyRefGenerator) imageForSourceInfo(s *source.Info) (*app.ImageRef, error) {
	var imageName string
	// TODO: More sophisticated matching
	switch s.Platform {
	case "Ruby":
		imageName = "openshift/ruby-20-centos"
	case "JEE":
		imageName = "openshift/wildfly-8-centos"
	case "NodeJS":
		imageName = "openshift/nodejs-010-centos7"
	default:
		return nil, errors.NoBuilderFound
	}
	if g.resolver != nil {
		return g.imageRefGenerator.FromNameAndResolver(imageName, g.resolver)
	}
	return g.imageRefGenerator.FromName(imageName)
}

func (g *BuildStrategyRefGenerator) detectDockerFile(dir string) (contextDir string, found bool, err error) {
	dockerFiles, err := g.dockerfileFinder.Find(dir)
	if err != nil {
		return "", false, err
	}
	if len(dockerFiles) > 1 {
		return "", true, errors.NewMultipleDockerfilesErr(dockerFiles)
	}
	if len(dockerFiles) == 1 {
		return filepath.Dir(dockerFiles[0]), true, nil
	}
	return "", false, nil
}

func (g *BuildStrategyRefGenerator) getSource(srcRef *app.SourceRef) error {
	var err error
	// Clone git repository into a local directory
	if srcRef.Dir, err = ioutil.TempDir("", "gen"); err != nil {
		return err
	}
	if err = g.gitRepository.Clone(srcRef.Dir, srcRef.URL.String()); err != nil {
		return err
	}
	if len(srcRef.Ref) != 0 {
		if err = g.gitRepository.Checkout(srcRef.Dir, srcRef.Ref); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>Bug 1190576: Improve error message when trying to use non-existent reference<commit_after>package generator

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/openshift/origin/pkg/generate/app"
	"github.com/openshift/origin/pkg/generate/dockerfile"
	"github.com/openshift/origin/pkg/generate/errors"
	"github.com/openshift/origin/pkg/generate/git"
	"github.com/openshift/origin/pkg/generate/source"
)

// Flows for BuildStrategyRef
// SourceRef -> BuildStrategyRef
// SourceRef + Docker Context -> BuildStrategyRef
// Docker Context + Parent Image -> BuildStrategyRef
// STI Builder Image -> BuildStrategyRef

// BuildStrategyRefGenerator generates BuildStrategyRef
type BuildStrategyRefGenerator struct {
	gitRepository     git.Repository
	dockerfileFinder  dockerfile.Finder
	dockerfileParser  dockerfile.Parser
	sourceDetectors   source.Detectors
	imageRefGenerator ImageRefGenerator
	resolver          app.Resolver
}

// NewBuildStrategyRefGenerator creates a BuildStrategyRefGenerator
func NewBuildStrategyRefGenerator(sourceDetectors source.Detectors, resolver app.Resolver) *BuildStrategyRefGenerator {
	return &BuildStrategyRefGenerator{
		gitRepository:     git.NewRepository(),
		dockerfileFinder:  dockerfile.NewFinder(),
		dockerfileParser:  dockerfile.NewParser(),
		sourceDetectors:   sourceDetectors,
		imageRefGenerator: NewImageRefGenerator(),
		resolver:          resolver,
	}
}

// FromSourceRef creates a build strategy from a source reference
func (g *BuildStrategyRefGenerator) FromSourceRef(srcRef app.SourceRef) (*app.BuildStrategyRef, error) {

	// Download source locally first if not available
	if len(srcRef.Dir) == 0 {
		if err := g.getSource(&srcRef); err != nil {
			return nil, err
		}
	}

	// Detect a Dockerfile
	context, found, err := g.detectDockerFile(srcRef.Dir)
	if err != nil {
		return nil, err
	}
	if found {
		return g.FromSourceRefAndDockerContext(srcRef, context)
	}

	// Detect a STI repository
	sourceInfo, ok := g.sourceDetectors.DetectSource(srcRef.Dir)
	if !ok {
		return nil, errors.CouldNotDetect
	}
	builderImage, err := g.imageForSourceInfo(sourceInfo)
	if err != nil {
		return nil, err
	}
	return g.FromSTIBuilderImage(builderImage)
}

// FromSourceRefAndDockerContext generates a BuildStrategyRef from a source ref and context path
func (g *BuildStrategyRefGenerator) FromSourceRefAndDockerContext(srcRef app.SourceRef, context string) (*app.BuildStrategyRef, error) {
	// Download source locally first if not available
	if len(srcRef.Dir) == 0 {
		if err := g.getSource(&srcRef); err != nil {
			return nil, err
		}
	}

	if len(context) > 0 {
		srcRef.ContextDir = context
	}

	// Look for Dockerfile in repository
	file, err := os.Open(filepath.Join(srcRef.Dir, context, "Dockerfile"))
	if err != nil {
		return nil, err
	}

	dockerFile, err := g.dockerfileParser.Parse(file)
	if err != nil {
		return nil, err
	}

	parentImageName, ok := dockerFile.GetDirective("FROM")
	if !ok {
		return nil, errors.InvalidDockerfile
	}
	ports, ok := dockerFile.GetDirective("EXPOSE")

	parentRef, err := g.imageRefGenerator.FromNameAndPorts(parentImageName[0], ports)
	if err != nil {
		return nil, err
	}

	return g.FromDockerContextAndParent(parentRef)

}

// FromContextAndParent generates a build strategy ref from a context path and parent image name
func (g *BuildStrategyRefGenerator) FromDockerContextAndParent(parentRef *app.ImageRef) (*app.BuildStrategyRef, error) {
	return &app.BuildStrategyRef{
		IsDockerBuild: true,
		Base:          parentRef,
	}, nil
}

// FromSTIBuilderImage generates a build strategy from a builder image ref
func (g *BuildStrategyRefGenerator) FromSTIBuilderImage(image *app.ImageRef) (*app.BuildStrategyRef, error) {
	return &app.BuildStrategyRef{
		IsDockerBuild: false,
		Base:          image,
	}, nil
}

func (g *BuildStrategyRefGenerator) imageForSourceInfo(s *source.Info) (*app.ImageRef, error) {
	var imageName string
	// TODO: More sophisticated matching
	switch s.Platform {
	case "Ruby":
		imageName = "openshift/ruby-20-centos"
	case "JEE":
		imageName = "openshift/wildfly-8-centos"
	case "NodeJS":
		imageName = "openshift/nodejs-010-centos7"
	default:
		return nil, errors.NoBuilderFound
	}
	if g.resolver != nil {
		return g.imageRefGenerator.FromNameAndResolver(imageName, g.resolver)
	}
	return g.imageRefGenerator.FromName(imageName)
}

func (g *BuildStrategyRefGenerator) detectDockerFile(dir string) (contextDir string, found bool, err error) {
	dockerFiles, err := g.dockerfileFinder.Find(dir)
	if err != nil {
		return "", false, err
	}
	if len(dockerFiles) > 1 {
		return "", true, errors.NewMultipleDockerfilesErr(dockerFiles)
	}
	if len(dockerFiles) == 1 {
		return filepath.Dir(dockerFiles[0]), true, nil
	}
	return "", false, nil
}

func (g *BuildStrategyRefGenerator) getSource(srcRef *app.SourceRef) error {
	var err error
	// Clone git repository into a local directory
	if srcRef.Dir, err = ioutil.TempDir("", "gen"); err != nil {
		return err
	}
	if err = g.gitRepository.Clone(srcRef.Dir, srcRef.URL.String()); err != nil {
		return fmt.Errorf("unable to clone repository at %s", srcRef.URL.String())
	}
	if len(srcRef.Ref) != 0 {
		if err = g.gitRepository.Checkout(srcRef.Dir, srcRef.Ref); err != nil {
			return fmt.Errorf("unable to checkout reference %s from repository at %s", srcRef.Ref, srcRef.URL.String())
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Giulio Iotti. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"crypto/sha1"
	"fmt"
	_ "golang.org/x/image/bmp"
	_ "golang.org/x/image/tiff"
	"hash"
	"image"
	_ "image/gif"
	_ "image/jpeg"
	_ "image/png"
	"io"
	"log"
	"mime"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"
)

const queryFile = `INSERT INTO sys_file (uid, pid, tstamp, last_indexed, missing, storage, type, metadata,
	identifier, identifier_hash, folder_hash, extension, mime_type, name, sha1, size, creation_date, modification_date) VALUES
("UID","0","%d","0","0","1","%d","0","%s","%x","%x","%s","%s","%s","%x","%d","%d","%d");
`
const queryMeta = `INSERT INTO sys_file_metadata (tstamp, crdate, file, width, height) VALUES
("%d","%d","UID","%d","%d");
`

type processor struct {
	nproc  int
	delta  delta
	writer *writer
	wg     sync.WaitGroup
	in     <-chan file
	tools  chan *tools
}

type tools struct {
	hash hash.Hash
	buf  bytes.Buffer
}

func newProcessor(in <-chan file, w *writer, n int, d delta) *processor {
	p := &processor{
		nproc:  n,
		in:     in,
		writer: w,
		delta:  d,
		tools:  make(chan *tools, n),
	}
	for i := 0; i < n; i++ {
		p.tools <- &tools{sha1.New(), bytes.Buffer{}}
	}
	return p
}

func (p *processor) runOne() {
	defer p.wg.Done()
	for f := range p.in {
		var done bool
		// Get one of the available tool structs
		tools := <-p.tools
		// Init basic data for this prop
		name := f.name()
		pr := newProps(tools.hash, f, name)
		// If in delta mode, see if there is a cached delta entry
		if *deltaMode != "" {
			entry := p.delta[pr.ident]
			// If we have an entry and it's modtime is unchanged, use cached entry
			if entry != nil && f.ModTime().Unix() == entry.mtime {
				p.writer.write(fmt.Sprintf("%s\n%s\n", entry.file, entry.meta))
				done = true
			}
		}
		// Do the normal work to create a new prop then write it
		if !done {
			pr.load(tools.hash, name)
			p.writer.write(pr.marshal(&tools.buf))
		}
		// Free up this tool struct for another worker
		p.tools <- tools
	}
}

func (p *processor) run() {
	p.wg.Add(p.nproc)
	for i := 0; i < p.nproc; i++ {
		go p.runOne()
	}
}

func (p *processor) wait() {
	p.wg.Wait()
	p.writer.close()
	p.writer.wait()
}

type digest [sha1.Size]byte

func strhash(data string, h hash.Hash) []byte {
	defer h.Reset()
	if _, err := io.WriteString(h, data); err != nil {
		log.Print("String write to SHA1: ", err)
		return nil
	}
	return h.Sum(nil)
}

func filehash(name string, h hash.Hash, r io.Reader) []byte {
	defer h.Reset()
	if _, err := io.Copy(h, r); err != nil {
		log.Print(name, ": Copy to SHA1: ", err)
		return nil
	}
	return h.Sum(nil)
}

func sniffMIME(name string, r *os.File) string {
	if _, err := r.Seek(0, 0); err != nil {
		log.Print(name, ": Seek: ", err)
		return ""
	}
	buf := make([]byte, 255)
	n, err := r.Read(buf)
	if n == 0 && err != nil {
		log.Print(name, ": Read: ", err)
		return ""
	}
	mimetype := http.DetectContentType(buf)
	n = strings.Index(mimetype, "; ")
	if n >= 0 {
		mimetype = mimetype[:n]
	}
	return mimetype
}

// Metadata to save about a file
type props struct {
	// SHA1 hash of file path + name
	ident digest
	// SHA1 hash of directory
	dident digest
	// SHA1 hash of file contents
	chash digest
	// Full filename
	fname string
	// Basename
	bname string
	// Extension
	ext string
	// Directory name
	dir string
	// MIME type
	mime string
	// If image, width x height
	isize image.Point
	// File size in bytes
	size int64
	// Type of file
	ftype int
	// Modification time
	modtime time.Time
	// Creation time (of this structure)
	ctime time.Time
}

func stripRoot(s string) string {
	n := strings.Index(s, "/")
	if n >= 0 {
		return s[n:]
	}
	return s
}

func mapType(mime string) int {
	n := strings.Index(mime, "/")
	if n >= 0 {
		mime = mime[:n]
	}
	switch mime {
	case "text":
		return 1
	case "image":
		return 2
	case "audio":
		return 3
	case "video":
		return 4
	case "application":
		return 5
	}
	return 0 // unknown
}

// Fast operations to fill props struct
func newProps(h hash.Hash, f file, name string) *props {
	fname := stripRoot(name)
	ext := filepath.Ext(fname)
	dir := filepath.Dir(fname)
	ident := strhash(fname, h)
	p := &props{
		modtime: f.ModTime(),
		ctime:   time.Now(),
		fname:   fname,
		ext:     ext,
		dir:     dir,
		size:    f.Size(),
		bname:   filepath.Base(fname),
	}
	copy(p.ident[:], ident)
	return p
}

// Slower operations to fill props struct
func (p *props) load(h hash.Hash, name string) *props {
	p.mime = mime.TypeByExtension(p.ext)
	r, err := os.Open(name)
	if err != nil {
		log.Print(name, ": Props: ", err)
		return p
	}
	defer r.Close()
	p.ftype = mapType(p.mime)
	// TODO: this is quite unreadable
	copy(p.chash[:], filehash(name, h, r))
	copy(p.dident[:], strhash(p.dir, h))
	// If the extension is empty, we need to detect
	// the MIME type via file contents
	if p.mime == "" {
		p.mime = sniffMIME(name, r)
	}
	// Non-images are completely processed at this point
	if !strings.HasPrefix(p.mime, "image/") {
		return p
	}
	// Image-specific processing
	if _, err := r.Seek(0, 0); err != nil {
		log.Print(name, ": Seek: ", err)
		return p
	}
	imgconf, _, err := image.DecodeConfig(r)
	if err != nil {
		log.Print(name, ": Image decoder: ", err)
		return p
	}
	p.isize = image.Point{imgconf.Width, imgconf.Height}
	return p
}

func escape(s string) string {
	return strings.Replace(s, `"`, `\"`, -1)
}

func (p *props) marshal(w *bytes.Buffer) string {
	defer w.Reset()
	switch true {
	case *singleMode:
		p.writeSingle(w)
	case *sqlMode:
		p.writeSQL(w)
	default:
		p.writeNormal(w)
	}
	return w.String()
}

// Single mode writes a single condensed line.  Used for debugging comparison with tester/tester.
func (p *props) writeSingle(w io.Writer) {
	fmt.Fprintf(w, `"0","0","1","%d","0","%s",`, p.ftype, escape(p.fname))
	fmt.Fprintf(w, `"%x","%x",`, p.ident, p.dident)
	fmt.Fprintf(w, `"%s","%s","%s",`, p.ext, p.mime, escape(p.bname))
	fmt.Fprintf(w, `"%x",`, p.chash)
	fmt.Fprintf(w, "\"%d\",\"%d\",\"%d\"\n", p.size, p.isize.X, p.isize.Y)
}

func (p *props) writeSQL(w io.Writer) {
	fmt.Fprintf(w, queryFile, p.ctime.Unix(), p.ftype, escape(p.fname),
		p.ident, p.dident, p.ext, p.mime, escape(p.bname), p.chash, p.size,
		p.ctime.Unix(), p.modtime.Unix())
	fmt.Fprintf(w, queryMeta, p.modtime.Unix(), p.ctime.Unix(), p.isize.X, p.isize.Y)
}

func (p *props) writeNormal(w io.Writer) {
	// Write file entry
	fmt.Fprintf(w, `file:"UID","0","%d","0","0","1","%d","0","`, p.ctime.Unix(), p.ftype)
	io.WriteString(w, escape(p.fname))
	fmt.Fprintf(w, `","%x","%x",`, p.ident, p.dident)
	fmt.Fprintf(w, `"%s","%s","`, p.ext, p.mime)
	io.WriteString(w, escape(p.bname))
	fmt.Fprintf(w, `","%x","%d",`, p.chash, p.size)
	fmt.Fprintf(w, "\"%d\",\"%d\"\n", p.ctime.Unix(), p.modtime.Unix())
	// Write metadata
	fmt.Fprintf(w, `meta:"UID","0","%d","%d","0","0","0","",`, p.modtime.Unix(), p.ctime.Unix())
	io.WriteString(w, `"0","0","0","","0","0","0","0","0","0",`)
	fmt.Fprintf(w, `"UID","","%d","%d",`, p.isize.X, p.isize.Y)
	io.WriteString(w, "\"\",\"\",\"0\"\n")
}
<commit_msg>Do not include dot in extension field<commit_after>// Copyright 2015 Giulio Iotti. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"crypto/sha1"
	"fmt"
	_ "golang.org/x/image/bmp"
	_ "golang.org/x/image/tiff"
	"hash"
	"image"
	_ "image/gif"
	_ "image/jpeg"
	_ "image/png"
	"io"
	"log"
	"mime"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"
)

const queryFile = `INSERT INTO sys_file (uid, pid, tstamp, last_indexed, missing, storage, type, metadata,
	identifier, identifier_hash, folder_hash, extension, mime_type, name, sha1, size, creation_date, modification_date) VALUES
("UID","0","%d","0","0","1","%d","0","%s","%x","%x","%s","%s","%s","%x","%d","%d","%d");
`
const queryMeta = `INSERT INTO sys_file_metadata (tstamp, crdate, file, width, height) VALUES
("%d","%d","UID","%d","%d");
`

type processor struct {
	nproc  int
	delta  delta
	writer *writer
	wg     sync.WaitGroup
	in     <-chan file
	tools  chan *tools
}

type tools struct {
	hash hash.Hash
	buf  bytes.Buffer
}

func newProcessor(in <-chan file, w *writer, n int, d delta) *processor {
	p := &processor{
		nproc:  n,
		in:     in,
		writer: w,
		delta:  d,
		tools:  make(chan *tools, n),
	}
	for i := 0; i < n; i++ {
		p.tools <- &tools{sha1.New(), bytes.Buffer{}}
	}
	return p
}

func (p *processor) runOne() {
	defer p.wg.Done()
	for f := range p.in {
		var done bool
		// Get one of the available tool structs
		tools := <-p.tools
		// Init basic data for this prop
		name := f.name()
		pr := newProps(tools.hash, f, name)
		// If in delta mode, see if there is a cached delta entry
		if *deltaMode != "" {
			entry := p.delta[pr.ident]
			// If we have an entry and it's modtime is unchanged, use cached entry
			if entry != nil && f.ModTime().Unix() == entry.mtime {
				p.writer.write(fmt.Sprintf("%s\n%s\n", entry.file, entry.meta))
				done = true
			}
		}
		// Do the normal work to create a new prop then write it
		if !done {
			pr.load(tools.hash, name)
			p.writer.write(pr.marshal(&tools.buf))
		}
		// Free up this tool struct for another worker
		p.tools <- tools
	}
}

func (p *processor) run() {
	p.wg.Add(p.nproc)
	for i := 0; i < p.nproc; i++ {
		go p.runOne()
	}
}

func (p *processor) wait() {
	p.wg.Wait()
	p.writer.close()
	p.writer.wait()
}

type digest [sha1.Size]byte

func strhash(data string, h hash.Hash) []byte {
	defer h.Reset()
	if _, err := io.WriteString(h, data); err != nil {
		log.Print("String write to SHA1: ", err)
		return nil
	}
	return h.Sum(nil)
}

func filehash(name string, h hash.Hash, r io.Reader) []byte {
	defer h.Reset()
	if _, err := io.Copy(h, r); err != nil {
		log.Print(name, ": Copy to SHA1: ", err)
		return nil
	}
	return h.Sum(nil)
}

func sniffMIME(name string, r *os.File) string {
	if _, err := r.Seek(0, 0); err != nil {
		log.Print(name, ": Seek: ", err)
		return ""
	}
	buf := make([]byte, 255)
	n, err := r.Read(buf)
	if n == 0 && err != nil {
		log.Print(name, ": Read: ", err)
		return ""
	}
	mimetype := http.DetectContentType(buf)
	n = strings.Index(mimetype, "; ")
	if n >= 0 {
		mimetype = mimetype[:n]
	}
	return mimetype
}

// Metadata to save about a file
type props struct {
	// SHA1 hash of file path + name
	ident digest
	// SHA1 hash of directory
	dident digest
	// SHA1 hash of file contents
	chash digest
	// Full filename
	fname string
	// Basename
	bname string
	// Extension
	ext string
	// Directory name
	dir string
	// MIME type
	mime string
	// If image, width x height
	isize image.Point
	// File size in bytes
	size int64
	// Type of file
	ftype int
	// Modification time
	modtime time.Time
	// Creation time (of this structure)
	ctime time.Time
}

func stripRoot(s string) string {
	n := strings.Index(s, "/")
	if n >= 0 {
		return s[n:]
	}
	return s
}

func mapType(mime string) int {
	n := strings.Index(mime, "/")
	if n >= 0 {
		mime = mime[:n]
	}
	switch mime {
	case "text":
		return 1
	case "image":
		return 2
	case "audio":
		return 3
	case "video":
		return 4
	case "application":
		return 5
	}
	return 0 // unknown
}

func fileExt(fname string) string {
	ext := filepath.Ext(fname)
	if ext != "" {
		ext = ext[1:]
	}
	return ext
}

// Fast operations to fill props struct
func newProps(h hash.Hash, f file, name string) *props {
	fname := stripRoot(name)
	ext := fileExt(fname)
	dir := filepath.Dir(fname)
	ident := strhash(fname, h)
	p := &props{
		modtime: f.ModTime(),
		ctime:   time.Now(),
		fname:   fname,
		ext:     ext,
		dir:     dir,
		size:    f.Size(),
		bname:   filepath.Base(fname),
	}
	copy(p.ident[:], ident)
	return p
}

// Slower operations to fill props struct
func (p *props) load(h hash.Hash, name string) *props {
	p.mime = mime.TypeByExtension(p.ext)
	r, err := os.Open(name)
	if err != nil {
		log.Print(name, ": Props: ", err)
		return p
	}
	defer r.Close()
	p.ftype = mapType(p.mime)
	// TODO: this is quite unreadable
	copy(p.chash[:], filehash(name, h, r))
	copy(p.dident[:], strhash(p.dir, h))
	// If the extension is empty, we need to detect
	// the MIME type via file contents
	if p.mime == "" {
		p.mime = sniffMIME(name, r)
	}
	// Non-images are completely processed at this point
	if !strings.HasPrefix(p.mime, "image/") {
		return p
	}
	// Image-specific processing
	if _, err := r.Seek(0, 0); err != nil {
		log.Print(name, ": Seek: ", err)
		return p
	}
	imgconf, _, err := image.DecodeConfig(r)
	if err != nil {
		log.Print(name, ": Image decoder: ", err)
		return p
	}
	p.isize = image.Point{imgconf.Width, imgconf.Height}
	return p
}

func escape(s string) string {
	return strings.Replace(s, `"`, `\"`, -1)
}

func (p *props) marshal(w *bytes.Buffer) string {
	defer w.Reset()
	switch true {
	case *singleMode:
		p.writeSingle(w)
	case *sqlMode:
		p.writeSQL(w)
	default:
		p.writeNormal(w)
	}
	return w.String()
}

// Single mode writes a single condensed line.  Used for debugging comparison with tester/tester.
func (p *props) writeSingle(w io.Writer) {
	fmt.Fprintf(w, `"0","0","1","%d","0","%s",`, p.ftype, escape(p.fname))
	fmt.Fprintf(w, `"%x","%x",`, p.ident, p.dident)
	fmt.Fprintf(w, `"%s","%s","%s",`, p.ext, p.mime, escape(p.bname))
	fmt.Fprintf(w, `"%x",`, p.chash)
	fmt.Fprintf(w, "\"%d\",\"%d\",\"%d\"\n", p.size, p.isize.X, p.isize.Y)
}

func (p *props) writeSQL(w io.Writer) {
	fmt.Fprintf(w, queryFile, p.ctime.Unix(), p.ftype, escape(p.fname),
		p.ident, p.dident, p.ext, p.mime, escape(p.bname), p.chash, p.size,
		p.ctime.Unix(), p.modtime.Unix())
	fmt.Fprintf(w, queryMeta, p.modtime.Unix(), p.ctime.Unix(), p.isize.X, p.isize.Y)
}

func (p *props) writeNormal(w io.Writer) {
	// Write file entry
	fmt.Fprintf(w, `file:"UID","0","%d","0","0","1","%d","0","`, p.ctime.Unix(), p.ftype)
	io.WriteString(w, escape(p.fname))
	fmt.Fprintf(w, `","%x","%x",`, p.ident, p.dident)
	fmt.Fprintf(w, `"%s","%s","`, p.ext, p.mime)
	io.WriteString(w, escape(p.bname))
	fmt.Fprintf(w, `","%x","%d",`, p.chash, p.size)
	fmt.Fprintf(w, "\"%d\",\"%d\"\n", p.ctime.Unix(), p.modtime.Unix())
	// Write metadata
	fmt.Fprintf(w, `meta:"UID","0","%d","%d","0","0","0","",`, p.modtime.Unix(), p.ctime.Unix())
	io.WriteString(w, `"0","0","0","","0","0","0","0","0","0",`)
	fmt.Fprintf(w, `"UID","","%d","%d",`, p.isize.X, p.isize.Y)
	io.WriteString(w, "\"\",\"\",\"0\"\n")
}
<|endoftext|>


<commit_before>package local

import (
	"context"
	"fmt"
	"io"
	"os"
	"path"
	"strings"
	"time"

	"github.com/cortexproject/cortex/pkg/chunk"
	"github.com/cortexproject/cortex/pkg/chunk/local"
	chunk_util "github.com/cortexproject/cortex/pkg/chunk/util"
	"github.com/cortexproject/cortex/pkg/util"
	"github.com/go-kit/kit/log/level"
)

// checkStorageForUpdates compares files from cache with storage and builds the list of files to be downloaded from storage and to be deleted from cache
func (fc *FilesCollection) checkStorageForUpdates(ctx context.Context) (toDownload []chunk.StorageObject, toDelete []string, err error) {
	// listing tables from store
	var objects []chunk.StorageObject
	objects, _, err = fc.storageClient.List(ctx, fc.period+"/")
	if err != nil {
		return
	}

	listedUploaders := make(map[string]struct{}, len(objects))

	fc.mtx.RLock()
	for _, object := range objects {
		uploader := strings.Split(object.Key, "/")[1]
		listedUploaders[uploader] = struct{}{}

		// Checking whether file was updated in the store after we downloaded it, if not, no need to include it in updates
		downloadedFileDetails, ok := fc.files[uploader]
		if !ok || downloadedFileDetails.mtime != object.ModifiedAt {
			toDownload = append(toDownload, object)
		}
	}
	fc.mtx.RUnlock()

	err = fc.ForEach(func(uploader string, df *downloadedFile) error {
		if _, isOK := listedUploaders[uploader]; !isOK {
			toDelete = append(toDelete, uploader)
		}
		return nil
	})

	return
}

// Sync downloads updated and new files from for given period from all the uploaders and removes deleted ones
func (fc *FilesCollection) Sync(ctx context.Context) error {
	level.Debug(util.Logger).Log("msg", fmt.Sprintf("syncing files for period %s", fc.period))

	toDownload, toDelete, err := fc.checkStorageForUpdates(ctx)
	if err != nil {
		return err
	}

	for _, storageObject := range toDownload {
		err = fc.downloadFile(ctx, storageObject)
		if err != nil {
			return err
		}
	}

	fc.mtx.Lock()
	defer fc.mtx.Unlock()

	for _, uploader := range toDelete {
		err := fc.cleanupFile(uploader)
		if err != nil {
			return err
		}
	}

	return nil
}

// It first downloads file to a temp location so that we close the existing file(if already exists), replace it with new one and then reopen it.
func (fc *FilesCollection) downloadFile(ctx context.Context, storageObject chunk.StorageObject) error {
	uploader := strings.Split(storageObject.Key, "/")[1]
	folderPath, _ := fc.getFolderPathForPeriod(false)
	filePath := path.Join(folderPath, uploader)

	// download the file temporarily with some other name to allow boltdb client to close the existing file first if it exists
	tempFilePath := path.Join(folderPath, fmt.Sprintf("%s.%s", uploader, "temp"))

	err := fc.getFileFromStorage(ctx, storageObject.Key, tempFilePath)
	if err != nil {
		return err
	}

	fc.mtx.Lock()
	defer fc.mtx.Unlock()

	df, ok := fc.files[uploader]
	if ok {
		if err := df.boltdb.Close(); err != nil {
			return err
		}
	} else {
		df = &downloadedFile{}
	}

	// move the file from temp location to actual location
	err = os.Rename(tempFilePath, filePath)
	if err != nil {
		return err
	}

	df.mtime = storageObject.ModifiedAt
	df.boltdb, err = local.OpenBoltdbFile(filePath)
	if err != nil {
		return err
	}

	fc.files[uploader] = df

	return nil
}

// getFileFromStorage downloads a file from storage to given location.
func (fc *FilesCollection) getFileFromStorage(ctx context.Context, objectKey, destination string) error {
	readCloser, err := fc.storageClient.GetObject(ctx, objectKey)
	if err != nil {
		return err
	}

	defer func() {
		if err := readCloser.Close(); err != nil {
			level.Error(util.Logger)
		}
	}()

	f, err := os.Create(destination)
	if err != nil {
		return err
	}

	_, err = io.Copy(f, readCloser)
	if err != nil {
		return err
	}

	level.Info(util.Logger).Log("msg", fmt.Sprintf("downloaded file %s", objectKey))

	return f.Sync()
}

// downloadAllFilesForPeriod should be called when files for a period does not exist i.e they were never downloaded or got cleaned up later on by TTL
// While files are being downloaded it will block all reads/writes on FilesCollection by taking an exclusive lock
func (fc *FilesCollection) downloadAllFilesForPeriod(ctx context.Context) (err error) {
	defer func() {
		status := statusSuccess
		if err != nil {
			status = statusFailure
			fc.setErr(err)

			// cleaning up files due to error to avoid returning invalid results.
			for fileName := range fc.files {
				if err := fc.cleanupFile(fileName); err != nil {
					level.Error(util.Logger).Log("msg", "failed to cleanup partially downloaded file", "filename", fileName, "err", err)
				}
			}
		}
		fc.metrics.filesDownloadOperationTotal.WithLabelValues(status).Inc()
	}()

	startTime := time.Now()
	totalFilesSize := int64(0)

	objects, _, err := fc.storageClient.List(ctx, fc.period+"/")
	if err != nil {
		return
	}

	level.Debug(util.Logger).Log("msg", fmt.Sprintf("list of files to download for period %s: %s", fc.period, objects))

	folderPath, err := fc.getFolderPathForPeriod(true)
	if err != nil {
		return
	}

	for _, object := range objects {
		uploader := getUploaderFromObjectKey(object.Key)

		filePath := path.Join(folderPath, uploader)
		df := downloadedFile{}

		err = fc.getFileFromStorage(ctx, object.Key, filePath)
		if err != nil {
			return
		}

		df.mtime = object.ModifiedAt
		df.boltdb, err = local.OpenBoltdbFile(filePath)
		if err != nil {
			return
		}

		var stat os.FileInfo
		stat, err = os.Stat(filePath)
		if err != nil {
			return
		}

		totalFilesSize += stat.Size()

		fc.files[uploader] = &df
	}

	duration := time.Since(startTime).Seconds()
	fc.metrics.filesDownloadDurationSeconds.add(fc.period, duration)
	fc.metrics.filesDownloadSizeBytes.add(fc.period, totalFilesSize)

	return
}

func (fc *FilesCollection) getFolderPathForPeriod(ensureExists bool) (string, error) {
	folderPath := path.Join(fc.cacheLocation, fc.period)

	if ensureExists {
		err := chunk_util.EnsureDirectory(folderPath)
		if err != nil {
			return "", err
		}
	}

	return folderPath, nil
}

func getUploaderFromObjectKey(objectKey string) string {
	return strings.Split(objectKey, "/")[1]
}
<commit_msg>Improve object key parsing for boltdb shipper. (#2323)<commit_after>package local

import (
	"context"
	"fmt"
	"io"
	"os"
	"path"
	"strings"
	"time"

	"github.com/cortexproject/cortex/pkg/chunk"
	"github.com/cortexproject/cortex/pkg/chunk/local"
	chunk_util "github.com/cortexproject/cortex/pkg/chunk/util"
	"github.com/cortexproject/cortex/pkg/util"
	"github.com/go-kit/kit/log/level"
)

// checkStorageForUpdates compares files from cache with storage and builds the list of files to be downloaded from storage and to be deleted from cache
func (fc *FilesCollection) checkStorageForUpdates(ctx context.Context) (toDownload []chunk.StorageObject, toDelete []string, err error) {
	// listing tables from store
	var objects []chunk.StorageObject
	objects, _, err = fc.storageClient.List(ctx, fc.period+"/")
	if err != nil {
		return
	}

	listedUploaders := make(map[string]struct{}, len(objects))

	fc.mtx.RLock()
	for _, object := range objects {
		uploader, err := getUploaderFromObjectKey(object.Key)
		if err != nil {
			return nil, nil, err
		}
		listedUploaders[uploader] = struct{}{}

		// Checking whether file was updated in the store after we downloaded it, if not, no need to include it in updates
		downloadedFileDetails, ok := fc.files[uploader]
		if !ok || downloadedFileDetails.mtime != object.ModifiedAt {
			toDownload = append(toDownload, object)
		}
	}
	fc.mtx.RUnlock()

	err = fc.ForEach(func(uploader string, df *downloadedFile) error {
		if _, isOK := listedUploaders[uploader]; !isOK {
			toDelete = append(toDelete, uploader)
		}
		return nil
	})

	return
}

// Sync downloads updated and new files from for given period from all the uploaders and removes deleted ones
func (fc *FilesCollection) Sync(ctx context.Context) error {
	level.Debug(util.Logger).Log("msg", fmt.Sprintf("syncing files for period %s", fc.period))

	toDownload, toDelete, err := fc.checkStorageForUpdates(ctx)
	if err != nil {
		return err
	}

	for _, storageObject := range toDownload {
		err = fc.downloadFile(ctx, storageObject)
		if err != nil {
			return err
		}
	}

	fc.mtx.Lock()
	defer fc.mtx.Unlock()

	for _, uploader := range toDelete {
		err := fc.cleanupFile(uploader)
		if err != nil {
			return err
		}
	}

	return nil
}

// It first downloads file to a temp location so that we close the existing file(if already exists), replace it with new one and then reopen it.
func (fc *FilesCollection) downloadFile(ctx context.Context, storageObject chunk.StorageObject) error {
	uploader, err := getUploaderFromObjectKey(storageObject.Key)
	if err != nil {
		return err
	}
	folderPath, _ := fc.getFolderPathForPeriod(false)
	filePath := path.Join(folderPath, uploader)

	// download the file temporarily with some other name to allow boltdb client to close the existing file first if it exists
	tempFilePath := path.Join(folderPath, fmt.Sprintf("%s.%s", uploader, "temp"))

	err = fc.getFileFromStorage(ctx, storageObject.Key, tempFilePath)
	if err != nil {
		return err
	}

	fc.mtx.Lock()
	defer fc.mtx.Unlock()

	df, ok := fc.files[uploader]
	if ok {
		if err := df.boltdb.Close(); err != nil {
			return err
		}
	} else {
		df = &downloadedFile{}
	}

	// move the file from temp location to actual location
	err = os.Rename(tempFilePath, filePath)
	if err != nil {
		return err
	}

	df.mtime = storageObject.ModifiedAt
	df.boltdb, err = local.OpenBoltdbFile(filePath)
	if err != nil {
		return err
	}

	fc.files[uploader] = df

	return nil
}

// getFileFromStorage downloads a file from storage to given location.
func (fc *FilesCollection) getFileFromStorage(ctx context.Context, objectKey, destination string) error {
	readCloser, err := fc.storageClient.GetObject(ctx, objectKey)
	if err != nil {
		return err
	}

	defer func() {
		if err := readCloser.Close(); err != nil {
			level.Error(util.Logger)
		}
	}()

	f, err := os.Create(destination)
	if err != nil {
		return err
	}

	_, err = io.Copy(f, readCloser)
	if err != nil {
		return err
	}

	level.Info(util.Logger).Log("msg", fmt.Sprintf("downloaded file %s", objectKey))

	return f.Sync()
}

// downloadAllFilesForPeriod should be called when files for a period does not exist i.e they were never downloaded or got cleaned up later on by TTL
// While files are being downloaded it will block all reads/writes on FilesCollection by taking an exclusive lock
func (fc *FilesCollection) downloadAllFilesForPeriod(ctx context.Context) (err error) {
	defer func() {
		status := statusSuccess
		if err != nil {
			status = statusFailure
			fc.setErr(err)

			// cleaning up files due to error to avoid returning invalid results.
			for fileName := range fc.files {
				if err := fc.cleanupFile(fileName); err != nil {
					level.Error(util.Logger).Log("msg", "failed to cleanup partially downloaded file", "filename", fileName, "err", err)
				}
			}
		}
		fc.metrics.filesDownloadOperationTotal.WithLabelValues(status).Inc()
	}()

	startTime := time.Now()
	totalFilesSize := int64(0)

	objects, _, err := fc.storageClient.List(ctx, fc.period+"/")
	if err != nil {
		return
	}

	level.Debug(util.Logger).Log("msg", fmt.Sprintf("list of files to download for period %s: %s", fc.period, objects))

	folderPath, err := fc.getFolderPathForPeriod(true)
	if err != nil {
		return
	}

	for _, object := range objects {
		var uploader string
		uploader, err = getUploaderFromObjectKey(object.Key)
		if err != nil {
			return
		}

		filePath := path.Join(folderPath, uploader)
		df := downloadedFile{}

		err = fc.getFileFromStorage(ctx, object.Key, filePath)
		if err != nil {
			return
		}

		df.mtime = object.ModifiedAt
		df.boltdb, err = local.OpenBoltdbFile(filePath)
		if err != nil {
			return
		}

		var stat os.FileInfo
		stat, err = os.Stat(filePath)
		if err != nil {
			return
		}

		totalFilesSize += stat.Size()

		fc.files[uploader] = &df
	}

	duration := time.Since(startTime).Seconds()
	fc.metrics.filesDownloadDurationSeconds.add(fc.period, duration)
	fc.metrics.filesDownloadSizeBytes.add(fc.period, totalFilesSize)

	return
}

func (fc *FilesCollection) getFolderPathForPeriod(ensureExists bool) (string, error) {
	folderPath := path.Join(fc.cacheLocation, fc.period)

	if ensureExists {
		err := chunk_util.EnsureDirectory(folderPath)
		if err != nil {
			return "", err
		}
	}

	return folderPath, nil
}

func getUploaderFromObjectKey(objectKey string) (string, error) {
	uploaders := strings.Split(objectKey, "/")
	if len(uploaders) != 2 {
		return "", fmt.Errorf("invalid object key: %v", objectKey)
	}
	if uploaders[1] == "" {
		return "", fmt.Errorf("empty uploader, object key: %v", objectKey)
	}
	return uploaders[1], nil
}
<|endoftext|>


<commit_before>package pure

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"reflect"
	"strconv"
)

type state int

const tagName = "pure"

type unmarshaler struct {
	Scanner  *scanner
	errors   []*pureError
	tagID    string
	tagValue string
	tagTok   Token
	tagTyp   string
}

// Shamelessly stolen from the Golang JSON decode source. Forgive
func (u *unmarshaler) indirect(v reflect.Value) reflect.Value {
	if v.Kind() != reflect.Ptr && v.Type().Name() != "" && v.CanAddr() {
		v = v.Addr()
	}

	for {
		if v.Kind() == reflect.Interface && !v.IsNil() {
			e := v.Elem()
			if e.Kind() == reflect.Ptr && !e.IsNil() {
				v = e
				continue
			}
		}

		if v.Kind() != reflect.Ptr {
			break
		}

		if v.Elem().Kind() != reflect.Ptr && v.CanSet() {
			break
		}

		if v.IsNil() {
			v.Set(reflect.New(v.Type().Elem()))
		}

		if v.Type().NumMethod() > 0 {
			// TODO
		}

		v = v.Elem()
	}
	return v
}

func (u *unmarshaler) newError(msg string) *pureError {
	s := fmt.Sprintf("Error unmarhsaling Pure property: %s\r\n[%d:%d]-%s", msg, u.Scanner.line, u.Scanner.col, u.Scanner.buf.String())
	err := &pureError{}
	err.error = fmt.Errorf(s)
	return err
}

func (u *unmarshaler) Scan() (tok Token, lit string) {
	return u.Scanner.Scan()
}

func (u *unmarshaler) ScanSkipWhitespace() (tok Token, lit string) {
	for tok, lit = u.Scanner.Scan(); tok == WHITESPACE; {
		tok, lit = u.Scanner.Scan()
	}
	return
}

func (u *unmarshaler) field(v reflect.Value) *pureError {
	var field reflect.Value
	switch v.Kind() {
	case reflect.Ptr:
		iv := u.indirect(v.Elem())
		for i := 0; i < iv.NumField(); i++ {
			tag := iv.Type().Field(i).Tag.Get(tagName)
			if tag != "" && tag != "-" && tag == u.tagID {
				field = iv.Field(i)
			}
		}
	case reflect.Struct:
		iv := u.indirect(v)
		tv := reflect.TypeOf(v.Interface())

		for i := 0; i < iv.NumField(); i++ {
			tag := tv.Field(i).Tag.Get(tagName)
			if tag != "" && tag != "-" && tag == u.tagID {
				field = iv.Field(i)
			}

		}
	}

	if !field.IsValid() {
		field = u.indirect(v)
	}

	switch {
	case field.Kind() == reflect.Int && u.tagTyp == "int":
		_i, err := strconv.Atoi(u.tagValue)
		if err != nil {
			return u.newError(fmt.Sprintf("bad number value '%s'", u.tagValue))
		}
		field.SetInt(int64(_i))
		return nil
	case field.Kind() == reflect.String && (u.tagTyp == "string" || u.tagTyp == "quantity" || u.tagTyp == "path"):
		field.SetString(u.tagValue)
		return nil
	case field.Kind() == reflect.Float64 && u.tagTyp == "double":
		f, err := strconv.ParseFloat(u.tagValue, 64)
		if err != nil {
			return u.newError(fmt.Sprintf("bad floating point value '%s'", u.tagValue))
		}
		field.SetFloat(f)
		return nil
	case field.Kind() == reflect.Bool && u.tagTyp == "bool":
		b, err := strconv.ParseBool(u.tagValue)
		if err != nil {
			return u.newError(fmt.Sprintf("bad bool value '%s'", u.tagValue))
		}
		field.SetBool(b)
		return nil
	case field.Kind() == reflect.Ptr && u.tagTyp == "group":
		return u.group(field.Interface())
	default:
		fi := field.Interface()
		switch fi.(type) {
		case *Quantity:
			if u.tagTyp != "quantity" {
				fmt.Println(u.newError(fmt.Sprintf("mismatched types 'Quantity' & '%s", u.tagTyp)))
				return nil
			}
			fi = NewQuantity(u.tagValue)
			field.Set(reflect.ValueOf(fi))
			return nil
		case Quantity:
			if u.tagTyp != "quantity" {
				fmt.Println(u.newError(fmt.Sprintf("mismatched types 'Quantity' & '%s", u.tagTyp)))
				return nil
			}
			fi = NewQuantity(u.tagValue)
			field.Set(u.indirect(reflect.ValueOf(fi)))
			return nil
		}

	}
	return nil
}

// Peek copies the unmarshalers buffer (to not advance the buffer we're reading form)
// and returns the next n bytes
func (u *unmarshaler) Peek(n int) []byte {
	return bytes.NewBuffer(u.Scanner.buf.Bytes()).Next(n)
}

// This is not pretty, but it works ¯\_(ツ)_/¯
func (u *unmarshaler) PeekLiteral() string {
	buf := bytes.NewBuffer(u.Scanner.buf.Bytes())
	for {
		b, _ := buf.ReadByte()

		if IsAlpha(b) {
			buf.WriteByte(b)
			for {
				b, _ := buf.ReadByte()
				if IsWhitespace(b) {
					break
				}
				buf.WriteByte(b)
			}
			break
		}
	}
	return buf.String()
}

func (u *unmarshaler) group(v interface{}) *pureError {
	iv := u.indirect(reflect.ValueOf(v))
	tv := reflect.TypeOf(v)
	for i := 0; i < iv.NumField(); i++ {
		tag := tv.Elem().Field(i).Tag.Get(tagName)

		if tag == u.tagID {
			f := iv.Field(i)
			for {
				tok, lit := u.Scan()
				if tok == EOF {
					return nil
				}

				if lit == "\r" {
					if b := u.Peek(2); b[0] == '\n' && (IsWhitespace(b[len(b)] - 1)) {
						continue
					}
					return nil
				}

				if lit == " " || lit == "\n" || lit == "\t" {
					continue
				}
				if tok == DOT || lit == "." {
					tok, lit = u.ScanSkipWhitespace()
				}

				if tok == GROUP {
					struc := u.GetStruct(u.tagID, v)
					field := u.GetField(lit, struc)
					u.tagID = u.PeekLiteral()
					err := u.group(field.Interface())
					if err != nil {
						fmt.Println(err.Error())
					}
				}

				u.tagID = lit

				tok, lit = u.ScanSkipWhitespace()

				if tok == EQUALS {
					u.tagTok, u.tagValue = u.ScanSkipWhitespace()
				}

				switch u.tagTok {
				case STRING, QUANTITY, PATH, IDENTIFIER:
					u.tagTyp = "string"
				case INT:
					u.tagTyp = "int"
				case DOUBLE:
					u.tagTyp = "double"
				case BOOL:
					u.tagTyp = "bool"
				}

				err := u.field(f)
				if err != nil {
					fmt.Println(err.Error())
				}
			}
		}
	}
	return nil
}

func (u *unmarshaler) GetStruct(name string, v interface{}) reflect.Value {
	iv := u.indirect(reflect.ValueOf(v))
	for i := 0; i < iv.NumField(); i++ {
		tag := reflect.TypeOf(v).Elem().Field(i).Tag.Get(tagName)
		if tag == name {
			return iv.Field(i)
		}
	}
	return reflect.Zero(nil)
}

func (u *unmarshaler) GetField(name string, v reflect.Value) reflect.Value {
	if v.Kind() == reflect.Ptr {
		iv := u.indirect(v.Elem())

		for i := 0; i < iv.NumField(); i++ {
			tag := iv.Type().Field(i).Tag.Get(tagName)

			if tag == name {
				return iv.Field(i)
			}
		}
	}

	if v.Kind() == reflect.Struct {
		iv := u.indirect(v)
		tv := reflect.TypeOf(v.Interface())
		for i := 0; i < iv.NumField(); i++ {
			tag := tv.Field(i).Tag.Get(tagName)
			if tag == "" {
				tag = iv.Type().Field(i).Tag.Get(tagName)
			}
			if tag == name || tag == u.tagID {
				if iv.Kind() == reflect.Struct || iv.Kind() == reflect.Ptr {
					return u.GetField(u.tagID, reflect.ValueOf(iv.Field(i)))
				}
				return iv.Field(i)
			}
		}
	}

	return v
}

func (u *unmarshaler) unmarshal(v interface{}) {
	pv := u.indirect(reflect.ValueOf(v))
	for {
		tok, lit := u.ScanSkipWhitespace()
		u.tagID = lit
		u.tagTok = tok

		if tok == EOF {
			return
		}

		switch tok {
		case IDENTIFIER:
			if tok, _ := u.ScanSkipWhitespace(); tok == EQUALS {
				u.tagTok, u.tagValue = u.ScanSkipWhitespace()
				switch u.tagTok {
				case INT:
					u.tagTyp = "int"
				case DOUBLE:
					u.tagTyp = "double"
				case PATH, STRING, IDENTIFIER:
					u.tagTyp = "string"
				case BOOL:
					u.tagTyp = "bool"
				case QUANTITY:
					u.tagTyp = "quantity"
				}
			} else if tok == REF {
				var field reflect.Value
				temp := lit
				tok, lit = u.ScanSkipWhitespace()
				if b := u.Peek(1); b[0] == '.' {
					group := lit
					tok, lit = u.ScanSkipWhitespace()
					tok, lit = u.ScanSkipWhitespace()
					u.tagID = lit
					struc := u.GetStruct(group, v)
					u.tagID = temp
					field = u.GetField(lit, struc)
				} else {
					// Assume it's a regular property and not a group property
					tok, lit = u.ScanSkipWhitespace()
					field = u.GetField(u.tagID, u.indirect(reflect.ValueOf(v)))
				}
				switch field.Kind() {
				case reflect.Int:
					u.tagTyp = "int"
					u.tagValue = strconv.Itoa(int(field.Int()))
				case reflect.Float64:
					u.tagTyp = "double"
					u.tagValue = strconv.FormatFloat(field.Float(), 'f', 16, 64)
				case reflect.String:
					u.tagTyp = "string"
					u.tagValue = field.String()
				case reflect.Bool:
					u.tagTyp = "bool"
					u.tagValue = strconv.FormatBool(field.Bool())
				}
			}

			err := u.field(pv)
			if err != nil {
				u.errors = append(u.errors, err)
			}

		case GROUP:
			err := u.group(v)
			if err != nil {
				u.errors = append(u.errors, err)
			}
		case INCLUDE:
			b, err := ioutil.ReadFile(lit)
			if err != nil {
				fmt.Println(u.newError(err.Error()).Error())
			}
			Unmarshal(b, v)
		}
	}
}

func Unmarshal(b []byte, v interface{}) *pureError {
	u := &unmarshaler{
		Scanner: newScanner(b),
	}
	u.unmarshal(v)

	// Should improve error reporting
	// Maybe as soon as they're discovered?
	if len(u.errors) > 0 {
		return u.errors[0]
	}
	return nil
}
<commit_msg>something<commit_after>package pure

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"reflect"
	"strconv"
)

type state int

const tagName = "pure"

type unmarshaler struct {
	Scanner  *scanner
	errors   []*pureError
	tagID    string
	tagValue string
	tagTok   Token
	tagTyp   string
}

// Shamelessly stolen from the Golang JSON decode source. Forgive
func (u *unmarshaler) indirect(v reflect.Value) reflect.Value {
	if v.Kind() != reflect.Ptr && v.Type().Name() != "" && v.CanAddr() {
		v = v.Addr()
	}

	for {
		if v.Kind() == reflect.Interface && !v.IsNil() {
			e := v.Elem()
			if e.Kind() == reflect.Ptr && !e.IsNil() {
				v = e
				continue
			}
		}

		if v.Kind() != reflect.Ptr {
			break
		}

		if v.Elem().Kind() != reflect.Ptr && v.CanSet() {
			break
		}

		if v.IsNil() {
			v.Set(reflect.New(v.Type().Elem()))
		}

		if v.Type().NumMethod() > 0 {
			// TODO
		}

		v = v.Elem()
	}
	return v
}

func (u *unmarshaler) newError(msg string) *pureError {
	s := fmt.Sprintf("Error unmarhsaling Pure property: %s\r\n[%d:%d]-%s", msg, u.Scanner.line, u.Scanner.col, u.Scanner.buf.String())
	err := &pureError{}
	err.error = fmt.Errorf(s)
	return err
}

func (u *unmarshaler) Scan() (tok Token, lit string) {
	return u.Scanner.Scan()
}

func (u *unmarshaler) ScanSkipWhitespace() (tok Token, lit string) {
	for tok, lit = u.Scanner.Scan(); tok == WHITESPACE; {
		tok, lit = u.Scanner.Scan()
	}
	return
}

func (u *unmarshaler) field(v reflect.Value) *pureError {
	var field reflect.Value

	switch v.Kind() {
	case reflect.Ptr:
		iv := u.indirect(v.Elem())
		for i := 0; i < iv.NumField(); i++ {
			tag := iv.Type().Field(i).Tag.Get(tagName)
			if tag != "" && tag != "-" && tag == u.tagID {
				field = iv.Field(i)
			}
		}
	case reflect.Struct:
		iv := u.indirect(v)
		tv := reflect.TypeOf(v.Interface())

		for i := 0; i < iv.NumField(); i++ {
			tag := tv.Field(i).Tag.Get(tagName)
			if tag != "" && tag != "-" && tag == u.tagID {
				field = iv.Field(i)
			}

		}
	}

	if !field.IsValid() {
		field = u.indirect(v)
	}

	switch {
	case field.Kind() == reflect.Int && u.tagTyp == "int":
		_i, err := strconv.Atoi(u.tagValue)
		if err != nil {
			return u.newError(fmt.Sprintf("bad number value '%s'", u.tagValue))
		}
		field.SetInt(int64(_i))
		return nil
	case field.Kind() == reflect.String && (u.tagTyp == "string" || u.tagTyp == "quantity" || u.tagTyp == "path"):
		field.SetString(u.tagValue)
		return nil
	case field.Kind() == reflect.Float64 && u.tagTyp == "double":
		f, err := strconv.ParseFloat(u.tagValue, 64)
		if err != nil {
			return u.newError(fmt.Sprintf("bad floating point value '%s'", u.tagValue))
		}
		field.SetFloat(f)
		return nil
	case field.Kind() == reflect.Bool && u.tagTyp == "bool":
		b, err := strconv.ParseBool(u.tagValue)
		if err != nil {
			return u.newError(fmt.Sprintf("bad bool value '%s'", u.tagValue))
		}
		field.SetBool(b)
		return nil
	case field.Kind() == reflect.Ptr && u.tagTyp == "group":
		return u.group(field.Interface())
	default:
		fi := field.Interface()
		switch fi.(type) {
		case *Quantity:
			if u.tagTyp != "quantity" {
				fmt.Println(u.newError(fmt.Sprintf("mismatched types 'Quantity' & '%s", u.tagTyp)))
				return nil
			}
			fi = NewQuantity(u.tagValue)
			field.Set(reflect.ValueOf(fi))
			return nil
		case Quantity:
			if u.tagTyp != "quantity" {
				fmt.Println(u.newError(fmt.Sprintf("mismatched types 'Quantity' & '%s", u.tagTyp)))
				return nil
			}
			fi = NewQuantity(u.tagValue)
			field.Set(u.indirect(reflect.ValueOf(fi)))
			return nil
		}

	}
	return nil
}

// Peek copies the unmarshalers buffer (to not advance the buffer we're reading form)
// and returns the next n bytes
func (u *unmarshaler) Peek(n int) []byte {
	return bytes.NewBuffer(u.Scanner.buf.Bytes()).Next(n)
}

// This is not pretty, but it works ¯\_(ツ)_/¯
func (u *unmarshaler) PeekLiteral() string {
	buf := bytes.NewBuffer(u.Scanner.buf.Bytes())
	for {
		b, _ := buf.ReadByte()

		if IsAlpha(b) {
			buf.WriteByte(b)
			for {
				b, _ := buf.ReadByte()
				if IsWhitespace(b) {
					break
				}
				buf.WriteByte(b)
			}
			break
		}
	}
	return buf.String()
}

func (u *unmarshaler) group(v interface{}) *pureError {
	iv := u.indirect(reflect.ValueOf(v))
	tv := reflect.TypeOf(v)
	for i := 0; i < iv.NumField(); i++ {
		tag := tv.Elem().Field(i).Tag.Get(tagName)

		if tag == u.tagID {
			f := iv.Field(i)
			for {
				tok, lit := u.Scan()
				if tok == EOF {
					return nil
				}

				if lit == "\r" {
					if b := u.Peek(2); b[0] == '\n' && (IsWhitespace(b[len(b)] - 1)) {
						continue
					}
					return nil
				}

				if lit == " " || lit == "\n" || lit == "\t" {
					continue
				}
				if tok == DOT || lit == "." {
					tok, lit = u.ScanSkipWhitespace()
				}

				if tok == GROUP {
					struc := u.GetStruct(u.tagID, v)
					field := u.GetField(lit, struc)
					u.tagID = u.PeekLiteral()
					err := u.group(field.Interface())
					if err != nil {
						fmt.Println(err.Error())
					}
				}

				u.tagID = lit

				tok, lit = u.ScanSkipWhitespace()

				if tok == EQUALS {
					u.tagTok, u.tagValue = u.ScanSkipWhitespace()
				}

				switch u.tagTok {
				case STRING, QUANTITY, PATH, IDENTIFIER:
					u.tagTyp = "string"
				case INT:
					u.tagTyp = "int"
				case DOUBLE:
					u.tagTyp = "double"
				case BOOL:
					u.tagTyp = "bool"
				}

				err := u.field(f)
				if err != nil {
					fmt.Println(err.Error())
				}
			}
		}
	}
	return nil
}

func (u *unmarshaler) GetStruct(name string, v interface{}) reflect.Value {
	iv := u.indirect(reflect.ValueOf(v))
	for i := 0; i < iv.NumField(); i++ {
		tag := reflect.TypeOf(v).Elem().Field(i).Tag.Get(tagName)
		if tag == name {
			return iv.Field(i)
		}
	}
	return reflect.Zero(nil)
}

func (u *unmarshaler) GetField(name string, v reflect.Value) reflect.Value {
	if v.Kind() == reflect.Ptr {
		iv := u.indirect(v.Elem())

		for i := 0; i < iv.NumField(); i++ {
			tag := iv.Type().Field(i).Tag.Get(tagName)

			if tag == name {
				return iv.Field(i)
			}
		}
	}

	if v.Kind() == reflect.Struct {
		iv := u.indirect(v)
		tv := reflect.TypeOf(v.Interface())
		for i := 0; i < iv.NumField(); i++ {
			tag := tv.Field(i).Tag.Get(tagName)
			if tag == "" {
				tag = iv.Type().Field(i).Tag.Get(tagName)
			}
			if tag == name || tag == u.tagID {
				if iv.Kind() == reflect.Struct || iv.Kind() == reflect.Ptr {
					return u.GetField(u.tagID, reflect.ValueOf(iv.Field(i)))
				}
				return iv.Field(i)
			}
		}
	}

	return v
}

func (u *unmarshaler) unmarshal(v interface{}) {
	pv := u.indirect(reflect.ValueOf(v))
	for {
		tok, lit := u.ScanSkipWhitespace()
		u.tagID = lit
		u.tagTok = tok

		if tok == EOF {
			return
		}

		switch tok {
		case IDENTIFIER:
			if tok, _ := u.ScanSkipWhitespace(); tok == EQUALS {
				u.tagTok, u.tagValue = u.ScanSkipWhitespace()
				switch u.tagTok {
				case INT:
					u.tagTyp = "int"
				case DOUBLE:
					u.tagTyp = "double"
				case PATH, STRING, IDENTIFIER:
					u.tagTyp = "string"
				case BOOL:
					u.tagTyp = "bool"
				case QUANTITY:
					u.tagTyp = "quantity"
				}
			} else if tok == REF {
				var field reflect.Value
				temp := lit
				tok, lit = u.ScanSkipWhitespace()
				if b := u.Peek(1); b[0] == '.' {
					group := lit
					tok, lit = u.ScanSkipWhitespace()
					tok, lit = u.ScanSkipWhitespace()
					u.tagID = lit
					struc := u.GetStruct(group, v)
					u.tagID = temp
					field = u.GetField(lit, struc)
				} else {
					// Assume it's a regular property and not a group property
					tok, lit = u.ScanSkipWhitespace()
					field = u.GetField(u.tagID, u.indirect(reflect.ValueOf(v)))
				}
				switch field.Kind() {
				case reflect.Int:
					u.tagTyp = "int"
					u.tagValue = strconv.Itoa(int(field.Int()))
				case reflect.Float64:
					u.tagTyp = "double"
					u.tagValue = strconv.FormatFloat(field.Float(), 'f', 16, 64)
				case reflect.String:
					u.tagTyp = "string"
					u.tagValue = field.String()
				case reflect.Bool:
					u.tagTyp = "bool"
					u.tagValue = strconv.FormatBool(field.Bool())
				}
			}

			err := u.field(pv)
			if err != nil {
				u.errors = append(u.errors, err)
			}

		case GROUP:
			err := u.group(v)
			if err != nil {
				u.errors = append(u.errors, err)
			}
		case INCLUDE:
			b, err := ioutil.ReadFile(lit)
			if err != nil {
				fmt.Println(u.newError(err.Error()).Error())
			}
			Unmarshal(b, v)
		}
	}
}

func Unmarshal(b []byte, v interface{}) *pureError {
	u := &unmarshaler{
		Scanner: newScanner(b),
	}
	u.unmarshal(v)

	// Should improve error reporting
	// Maybe as soon as they're discovered?
	if len(u.errors) > 0 {
		return u.errors[0]
	}
	return nil
}
<|endoftext|>