<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package parsers

import (
	"github.com/docker/docker/pkg/parsers"
)

const (
	defaultImageTag = "latest"
)

// parseImageName parses a docker image string into two parts: repo and tag.
// If tag is empty, return the defaultImageTag.
func ParseImageName(image string) (string, string) {
	repoToPull, tag := parsers.ParseRepositoryTag(image)
	// If no tag was specified, use the default "latest".
	if len(tag) == 0 {
		tag = defaultImageTag
	}
	return repoToPull, tag
}
<commit_msg>Hackily gets our k8s vendor compiling.<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package parsers

import (
	"github.com/docker/distribution/reference"
)

const (
	defaultImageTag = "latest"
)

// parseImageName parses a docker image string into two parts: repo and tag.
// If tag is empty, return the defaultImageTag.
func ParseImageName(image string) (string, string) {
	ref, _ := reference.Parse(image)
	switch ref := ref.(type) {
	default:
		return ref.String(), defaultImageTag
	case reference.NamedTagged:
		return ref.Name(), ref.Tag()
	case reference.Named:
		return ref.Name(), defaultImageTag
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package lua

import (
	"fmt"
	"strings"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	extensions "k8s.io/api/extensions/v1beta1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"k8s.io/ingress-nginx/test/e2e/framework"
)

var _ = framework.IngressNginxDescribe("Dynamic Certificate", func() {
	f := framework.NewDefaultFramework("dynamic-certificate")
	host := "foo.com"

	BeforeEach(func() {
		f.NewEchoDeploymentWithReplicas(1)
	})

	It("picks up the certificate when we add TLS spec to existing ingress", func() {
		ensureIngress(f, host, "http-svc")

		ing, err := f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.Namespace).Get(host, metav1.GetOptions{})
		Expect(err).ToNot(HaveOccurred())
		ing.Spec.TLS = []extensions.IngressTLS{
			{
				Hosts:      []string{host},
				SecretName: host,
			},
		}
		_, err = framework.CreateIngressTLSSecret(f.KubeClientSet,
			ing.Spec.TLS[0].Hosts,
			ing.Spec.TLS[0].SecretName,
			ing.Namespace)
		Expect(err).ToNot(HaveOccurred())
		_, err = f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.Namespace).Update(ing)
		Expect(err).ToNot(HaveOccurred())
		time.Sleep(waitForLuaSync)

		ensureHTTPSRequest(f.GetURL(framework.HTTPS), host, host)
	})

	It("picks up the previously missing secret for a given ingress without reloading", func() {
		ing := framework.NewSingleIngressWithTLS(host, "/", host, []string{host}, f.Namespace, "http-svc", 80, nil)
		f.EnsureIngress(ing)

		time.Sleep(waitForLuaSync)

		ensureHTTPSRequest(fmt.Sprintf("%s?id=dummy_log_splitter_foo_bar", f.GetURL(framework.HTTPS)), host, "ingress.local")

		_, err := framework.CreateIngressTLSSecret(f.KubeClientSet,
			ing.Spec.TLS[0].Hosts,
			ing.Spec.TLS[0].SecretName,
			ing.Namespace)
		Expect(err).ToNot(HaveOccurred())

		By("configuring certificate_by_lua and skipping Nginx configuration of the new certificate")
		f.WaitForNginxServer(host,
			func(server string) bool {
				return strings.Contains(server, "ssl_certificate_by_lua_block") &&
					!strings.Contains(server, fmt.Sprintf("ssl_certificate /etc/ingress-controller/ssl/%s-%s.pem;", ing.Namespace, host)) &&
					!strings.Contains(server, fmt.Sprintf("ssl_certificate_key /etc/ingress-controller/ssl/%s-%s.pem;", ing.Namespace, host)) &&
					strings.Contains(server, "listen 443")
			})

		time.Sleep(waitForLuaSync)

		By("serving the configured certificate on HTTPS endpoint")
		ensureHTTPSRequest(f.GetURL(framework.HTTPS), host, host)

		log, err := f.NginxLogs()
		Expect(err).ToNot(HaveOccurred())
		Expect(log).ToNot(BeEmpty())
		index := strings.Index(log, "id=dummy_log_splitter_foo_bar")
		restOfLogs := log[index:]

		By("skipping Nginx reload")
		Expect(restOfLogs).ToNot(ContainSubstring(logRequireBackendReload))
		Expect(restOfLogs).ToNot(ContainSubstring(logBackendReloadSuccess))
	})

	Context("given an ingress with TLS correctly configured", func() {
		BeforeEach(func() {
			ing := f.EnsureIngress(framework.NewSingleIngressWithTLS(host, "/", host, []string{host}, f.Namespace, "http-svc", 80, nil))

			time.Sleep(waitForLuaSync)

			ensureHTTPSRequest(f.GetURL(framework.HTTPS), host, "ingress.local")

			_, err := framework.CreateIngressTLSSecret(f.KubeClientSet,
				ing.Spec.TLS[0].Hosts,
				ing.Spec.TLS[0].SecretName,
				ing.Namespace)
			Expect(err).ToNot(HaveOccurred())
			time.Sleep(waitForLuaSync)

			By("configuring certificate_by_lua and skipping Nginx configuration of the new certificate")
			f.WaitForNginxServer(ing.Spec.TLS[0].Hosts[0],
				func(server string) bool {
					return strings.Contains(server, "ssl_certificate_by_lua_block") &&
						!strings.Contains(server, fmt.Sprintf("ssl_certificate /etc/ingress-controller/ssl/%s-%s.pem;", ing.Namespace, host)) &&
						!strings.Contains(server, fmt.Sprintf("ssl_certificate_key /etc/ingress-controller/ssl/%s-%s.pem;", ing.Namespace, host)) &&
						strings.Contains(server, "listen 443")
				})

			time.Sleep(waitForLuaSync)

			By("serving the configured certificate on HTTPS endpoint")
			ensureHTTPSRequest(f.GetURL(framework.HTTPS), host, host)
		})

		It("supports requests with domain with trailing dot", func() {
			ensureHTTPSRequest(f.GetURL(framework.HTTPS), host+".", host)
		})

		It("picks up the updated certificate without reloading", func() {
			ing, err := f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.Namespace).Get(host, metav1.GetOptions{})
			Expect(err).ToNot(HaveOccurred())

			ensureHTTPSRequest(fmt.Sprintf("%s?id=dummy_log_splitter_foo_bar", f.GetURL(framework.HTTPS)), host, host)

			_, err = framework.CreateIngressTLSSecret(f.KubeClientSet,
				ing.Spec.TLS[0].Hosts,
				ing.Spec.TLS[0].SecretName,
				ing.Namespace)
			Expect(err).ToNot(HaveOccurred())
			time.Sleep(waitForLuaSync)

			By("configuring certificate_by_lua and skipping Nginx configuration of the new certificate")
			f.WaitForNginxServer(ing.Spec.TLS[0].Hosts[0],
				func(server string) bool {
					return strings.Contains(server, "ssl_certificate_by_lua_block") &&
						!strings.Contains(server, fmt.Sprintf("ssl_certificate /etc/ingress-controller/ssl/%s-%s.pem;", ing.Namespace, host)) &&
						!strings.Contains(server, fmt.Sprintf("ssl_certificate_key /etc/ingress-controller/ssl/%s-%s.pem;", ing.Namespace, host)) &&
						strings.Contains(server, "listen 443")
				})

			time.Sleep(waitForLuaSync)

			By("serving the configured certificate on HTTPS endpoint")
			ensureHTTPSRequest(f.GetURL(framework.HTTPS), host, host)

			log, err := f.NginxLogs()
			Expect(err).ToNot(HaveOccurred())
			Expect(log).ToNot(BeEmpty())
			index := strings.Index(log, "id=dummy_log_splitter_foo_bar")
			restOfLogs := log[index:]

			By("skipping Nginx reload")
			Expect(restOfLogs).ToNot(ContainSubstring(logRequireBackendReload))
			Expect(restOfLogs).ToNot(ContainSubstring(logBackendReloadSuccess))
		})

		It("falls back to using default certificate when secret gets deleted without reloading", func() {
			ing, err := f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.Namespace).Get(host, metav1.GetOptions{})

			ensureHTTPSRequest(fmt.Sprintf("%s?id=dummy_log_splitter_foo_bar", f.GetURL(framework.HTTPS)), host, host)

			f.KubeClientSet.CoreV1().Secrets(ing.Namespace).Delete(ing.Spec.TLS[0].SecretName, nil)
			Expect(err).ToNot(HaveOccurred())
			time.Sleep(waitForLuaSync)

			By("configuring certificate_by_lua and skipping Nginx configuration of the new certificate")
			f.WaitForNginxServer(ing.Spec.TLS[0].Hosts[0],
				func(server string) bool {
					return strings.Contains(server, "ssl_certificate_by_lua_block") &&
						strings.Contains(server, "ssl_certificate /etc/ingress-controller/ssl/default-fake-certificate.pem;") &&
						strings.Contains(server, "ssl_certificate_key /etc/ingress-controller/ssl/default-fake-certificate.pem;") &&
						strings.Contains(server, "listen 443")
				})

			time.Sleep(waitForLuaSync)

			By("serving the default certificate on HTTPS endpoint")
			ensureHTTPSRequest(f.GetURL(framework.HTTPS), host, "ingress.local")

			log, err := f.NginxLogs()
			Expect(err).ToNot(HaveOccurred())
			Expect(log).ToNot(BeEmpty())
			index := strings.Index(log, "id=dummy_log_splitter_foo_bar")
			restOfLogs := log[index:]

			By("skipping Nginx reload")
			Expect(restOfLogs).ToNot(ContainSubstring(logRequireBackendReload))
			Expect(restOfLogs).ToNot(ContainSubstring(logBackendReloadSuccess))
		})

		It("picks up a non-certificate only change", func() {
			newHost := "foo2.com"
			ing, err := f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.Namespace).Get(host, metav1.GetOptions{})
			Expect(err).NotTo(HaveOccurred())
			ing.Spec.Rules[0].Host = newHost
			_, err = f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.Namespace).Update(ing)
			Expect(err).ToNot(HaveOccurred())
			time.Sleep(waitForLuaSync)

			By("serving the configured certificate on HTTPS endpoint")
			ensureHTTPSRequest(f.GetURL(framework.HTTPS), newHost, "ingress.local")
		})

		It("removes HTTPS configuration when we delete TLS spec", func() {
			ing, err := f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.Namespace).Get(host, metav1.GetOptions{})
			Expect(err).NotTo(HaveOccurred())
			ing.Spec.TLS = []extensions.IngressTLS{}
			_, err = f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.Namespace).Update(ing)
			Expect(err).ToNot(HaveOccurred())
			time.Sleep(waitForLuaSync)

			ensureRequest(f, host)
		})
	})
})
<commit_msg>add comment to the test<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package lua

import (
	"fmt"
	"strings"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	extensions "k8s.io/api/extensions/v1beta1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"k8s.io/ingress-nginx/test/e2e/framework"
)

var _ = framework.IngressNginxDescribe("Dynamic Certificate", func() {
	f := framework.NewDefaultFramework("dynamic-certificate")
	host := "foo.com"

	BeforeEach(func() {
		f.NewEchoDeploymentWithReplicas(1)
	})

	It("picks up the certificate when we add TLS spec to existing ingress", func() {
		ensureIngress(f, host, "http-svc")

		ing, err := f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.Namespace).Get(host, metav1.GetOptions{})
		Expect(err).ToNot(HaveOccurred())
		ing.Spec.TLS = []extensions.IngressTLS{
			{
				Hosts:      []string{host},
				SecretName: host,
			},
		}
		_, err = framework.CreateIngressTLSSecret(f.KubeClientSet,
			ing.Spec.TLS[0].Hosts,
			ing.Spec.TLS[0].SecretName,
			ing.Namespace)
		Expect(err).ToNot(HaveOccurred())
		_, err = f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.Namespace).Update(ing)
		Expect(err).ToNot(HaveOccurred())
		time.Sleep(waitForLuaSync)

		ensureHTTPSRequest(f.GetURL(framework.HTTPS), host, host)
	})

	It("picks up the previously missing secret for a given ingress without reloading", func() {
		ing := framework.NewSingleIngressWithTLS(host, "/", host, []string{host}, f.Namespace, "http-svc", 80, nil)
		f.EnsureIngress(ing)

		time.Sleep(waitForLuaSync)

		ensureHTTPSRequest(fmt.Sprintf("%s?id=dummy_log_splitter_foo_bar", f.GetURL(framework.HTTPS)), host, "ingress.local")

		_, err := framework.CreateIngressTLSSecret(f.KubeClientSet,
			ing.Spec.TLS[0].Hosts,
			ing.Spec.TLS[0].SecretName,
			ing.Namespace)
		Expect(err).ToNot(HaveOccurred())

		By("configuring certificate_by_lua and skipping Nginx configuration of the new certificate")
		f.WaitForNginxServer(host,
			func(server string) bool {
				return strings.Contains(server, "ssl_certificate_by_lua_block") &&
					!strings.Contains(server, fmt.Sprintf("ssl_certificate /etc/ingress-controller/ssl/%s-%s.pem;", ing.Namespace, host)) &&
					!strings.Contains(server, fmt.Sprintf("ssl_certificate_key /etc/ingress-controller/ssl/%s-%s.pem;", ing.Namespace, host)) &&
					strings.Contains(server, "listen 443")
			})

		time.Sleep(waitForLuaSync)

		By("serving the configured certificate on HTTPS endpoint")
		ensureHTTPSRequest(f.GetURL(framework.HTTPS), host, host)

		log, err := f.NginxLogs()
		Expect(err).ToNot(HaveOccurred())
		Expect(log).ToNot(BeEmpty())
		index := strings.Index(log, "id=dummy_log_splitter_foo_bar")
		restOfLogs := log[index:]

		By("skipping Nginx reload")
		Expect(restOfLogs).ToNot(ContainSubstring(logRequireBackendReload))
		Expect(restOfLogs).ToNot(ContainSubstring(logBackendReloadSuccess))
	})

	Context("given an ingress with TLS correctly configured", func() {
		BeforeEach(func() {
			ing := f.EnsureIngress(framework.NewSingleIngressWithTLS(host, "/", host, []string{host}, f.Namespace, "http-svc", 80, nil))

			time.Sleep(waitForLuaSync)

			ensureHTTPSRequest(f.GetURL(framework.HTTPS), host, "ingress.local")

			_, err := framework.CreateIngressTLSSecret(f.KubeClientSet,
				ing.Spec.TLS[0].Hosts,
				ing.Spec.TLS[0].SecretName,
				ing.Namespace)
			Expect(err).ToNot(HaveOccurred())
			time.Sleep(waitForLuaSync)

			By("configuring certificate_by_lua and skipping Nginx configuration of the new certificate")
			f.WaitForNginxServer(ing.Spec.TLS[0].Hosts[0],
				func(server string) bool {
					return strings.Contains(server, "ssl_certificate_by_lua_block") &&
						!strings.Contains(server, fmt.Sprintf("ssl_certificate /etc/ingress-controller/ssl/%s-%s.pem;", ing.Namespace, host)) &&
						!strings.Contains(server, fmt.Sprintf("ssl_certificate_key /etc/ingress-controller/ssl/%s-%s.pem;", ing.Namespace, host)) &&
						strings.Contains(server, "listen 443")
				})

			time.Sleep(waitForLuaSync)

			By("serving the configured certificate on HTTPS endpoint")
			ensureHTTPSRequest(f.GetURL(framework.HTTPS), host, host)
		})

		/*
			TODO(elvinefendi): this test currently does not work as expected
			because Go transport code strips (https://github.com/golang/go/blob/431b5c69ca214ce4291f008c1ce2a50b22bc2d2d/src/crypto/tls/handshake_messages.go#L424)
			trailing dot from SNI as suggest by the standard (https://tools.ietf.org/html/rfc6066#section-3).
		*/
		It("supports requests with domain with trailing dot", func() {
			ensureHTTPSRequest(f.GetURL(framework.HTTPS), host+".", host)
		})

		It("picks up the updated certificate without reloading", func() {
			ing, err := f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.Namespace).Get(host, metav1.GetOptions{})
			Expect(err).ToNot(HaveOccurred())

			ensureHTTPSRequest(fmt.Sprintf("%s?id=dummy_log_splitter_foo_bar", f.GetURL(framework.HTTPS)), host, host)

			_, err = framework.CreateIngressTLSSecret(f.KubeClientSet,
				ing.Spec.TLS[0].Hosts,
				ing.Spec.TLS[0].SecretName,
				ing.Namespace)
			Expect(err).ToNot(HaveOccurred())
			time.Sleep(waitForLuaSync)

			By("configuring certificate_by_lua and skipping Nginx configuration of the new certificate")
			f.WaitForNginxServer(ing.Spec.TLS[0].Hosts[0],
				func(server string) bool {
					return strings.Contains(server, "ssl_certificate_by_lua_block") &&
						!strings.Contains(server, fmt.Sprintf("ssl_certificate /etc/ingress-controller/ssl/%s-%s.pem;", ing.Namespace, host)) &&
						!strings.Contains(server, fmt.Sprintf("ssl_certificate_key /etc/ingress-controller/ssl/%s-%s.pem;", ing.Namespace, host)) &&
						strings.Contains(server, "listen 443")
				})

			time.Sleep(waitForLuaSync)

			By("serving the configured certificate on HTTPS endpoint")
			ensureHTTPSRequest(f.GetURL(framework.HTTPS), host, host)

			log, err := f.NginxLogs()
			Expect(err).ToNot(HaveOccurred())
			Expect(log).ToNot(BeEmpty())
			index := strings.Index(log, "id=dummy_log_splitter_foo_bar")
			restOfLogs := log[index:]

			By("skipping Nginx reload")
			Expect(restOfLogs).ToNot(ContainSubstring(logRequireBackendReload))
			Expect(restOfLogs).ToNot(ContainSubstring(logBackendReloadSuccess))
		})

		It("falls back to using default certificate when secret gets deleted without reloading", func() {
			ing, err := f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.Namespace).Get(host, metav1.GetOptions{})

			ensureHTTPSRequest(fmt.Sprintf("%s?id=dummy_log_splitter_foo_bar", f.GetURL(framework.HTTPS)), host, host)

			f.KubeClientSet.CoreV1().Secrets(ing.Namespace).Delete(ing.Spec.TLS[0].SecretName, nil)
			Expect(err).ToNot(HaveOccurred())
			time.Sleep(waitForLuaSync)

			By("configuring certificate_by_lua and skipping Nginx configuration of the new certificate")
			f.WaitForNginxServer(ing.Spec.TLS[0].Hosts[0],
				func(server string) bool {
					return strings.Contains(server, "ssl_certificate_by_lua_block") &&
						strings.Contains(server, "ssl_certificate /etc/ingress-controller/ssl/default-fake-certificate.pem;") &&
						strings.Contains(server, "ssl_certificate_key /etc/ingress-controller/ssl/default-fake-certificate.pem;") &&
						strings.Contains(server, "listen 443")
				})

			time.Sleep(waitForLuaSync)

			By("serving the default certificate on HTTPS endpoint")
			ensureHTTPSRequest(f.GetURL(framework.HTTPS), host, "ingress.local")

			log, err := f.NginxLogs()
			Expect(err).ToNot(HaveOccurred())
			Expect(log).ToNot(BeEmpty())
			index := strings.Index(log, "id=dummy_log_splitter_foo_bar")
			restOfLogs := log[index:]

			By("skipping Nginx reload")
			Expect(restOfLogs).ToNot(ContainSubstring(logRequireBackendReload))
			Expect(restOfLogs).ToNot(ContainSubstring(logBackendReloadSuccess))
		})

		It("picks up a non-certificate only change", func() {
			newHost := "foo2.com"
			ing, err := f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.Namespace).Get(host, metav1.GetOptions{})
			Expect(err).NotTo(HaveOccurred())
			ing.Spec.Rules[0].Host = newHost
			_, err = f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.Namespace).Update(ing)
			Expect(err).ToNot(HaveOccurred())
			time.Sleep(waitForLuaSync)

			By("serving the configured certificate on HTTPS endpoint")
			ensureHTTPSRequest(f.GetURL(framework.HTTPS), newHost, "ingress.local")
		})

		It("removes HTTPS configuration when we delete TLS spec", func() {
			ing, err := f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.Namespace).Get(host, metav1.GetOptions{})
			Expect(err).NotTo(HaveOccurred())
			ing.Spec.TLS = []extensions.IngressTLS{}
			_, err = f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.Namespace).Update(ing)
			Expect(err).ToNot(HaveOccurred())
			time.Sleep(waitForLuaSync)

			ensureRequest(f, host)
		})
	})
})
<|endoftext|>


<commit_before>//go:build integration
// +build integration

/*
Copyright 2021 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"context"
	"fmt"
	"os/exec"
	"strings"
	"testing"
	"time"
)

const (
	mountGID   = "0"
	mountMSize = "6543"
	mountMode  = "0777"
	mountPort  = "46464"
	mountUID   = "0"
)

// TestMountStart tests using the mount command on start
func TestMountStart(t *testing.T) {
	if NoneDriver() {
		t.Skip("skipping: none driver does not support mount")
	}

	type validateFunc func(context.Context, *testing.T, string)
	profile1 := UniqueProfileName("mount-start-1")
	profile2 := UniqueProfileName("mount-start-2")
	ctx, cancel := context.WithTimeout(context.Background(), Minutes(15))
	defer Cleanup(t, profile1, cancel)
	defer Cleanup(t, profile2, cancel)

	// Serial tests
	t.Run("serial", func(t *testing.T) {
		tests := []struct {
			name      string
			validator validateFunc
			profile   string
		}{
			{"StartWithMountFirst", validateStartWithMount, profile1},
			{"StartWithMountSecond", validateStartWithMount, profile2},
			{"VerifyMountFirst", validateMount, profile1},
			{"VerifyMountSecond", validateMount, profile2},
			{"DeleteFirst", validateDelete, profile1},
			{"VerifyMountPostDelete", validateMount, profile2},
			{"Stop", validateMountStop, profile2},
			{"RestartStopped", validateRestart, profile2},
			{"VerifyMountPostStop", validateMount, profile2},
		}

		for _, test := range tests {
			if ctx.Err() == context.DeadlineExceeded {
				t.Fatalf("Unable to run more tests (deadline exceeded)")
			}

			t.Run(test.name, func(t *testing.T) {
				test.validator(ctx, t, test.profile)
			})
		}
	})
}

// validateStartWithMount starts a cluster with mount enabled
func validateStartWithMount(ctx context.Context, t *testing.T, profile string) {
	defer PostMortemLogs(t, profile)

	args := []string{"start", "-p", profile, "--memory=2048", "--mount", "--mount-gid", mountGID, "--mount-msize", mountMSize, "--mount-mode", mountMode, "--mount-port", mountPort, "--mount-uid", mountUID}
	args = append(args, StartArgs()...)
	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Fatalf("failed to start minikube with args: %q : %v", rr.Command(), err)
	}
}

// validateMount checks if the cluster has a folder mounted
func validateMount(ctx context.Context, t *testing.T, profile string) {
	defer PostMortemLogs(t, profile)

	sshArgs := []string{"-p", profile, "ssh", "--"}

	args := sshArgs
	args = append(args, "ls", "/minikube-host")
	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Fatalf("mount failed: %q : %v", rr.Command(), err)
	}

	// Docker has it's own mounting method, it doesn't respect the mounting flags
	if DockerDriver() {
		return
	}

	args = sshArgs
	args = append(args, "stat", "--format", "'%a'", "/minikube-host")
	rr, err = Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Fatalf("failed to get directory mode: %v", err)
	}

	want := "777"
	if !strings.Contains(rr.Output(), want) {
		t.Errorf("wanted mode to be %q; got: %q", want, rr.Output())
	}

	// give time for 9P process to be listed as running on Windows
	time.Sleep(5 * time.Second)

	args = sshArgs
	args = append(args, "mount", "|", "grep", "9p")
	rr, err = Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Fatalf("failed to get mount information: %v", err)
	}

	flags := []struct {
		key      string
		expected string
	}{
		{"gid", mountGID},
		{"msize", mountMSize},
		{"port", mountPort},
		{"uid", mountUID},
	}

	for _, flag := range flags {
		want := fmt.Sprintf("%s=%s", flag.key, flag.expected)
		if !strings.Contains(rr.Output(), want) {
			t.Errorf("wanted gid to be: %q; got: %q", want, rr.Output())
		}
	}
}

// validateMountStop stops a cluster
func validateMountStop(ctx context.Context, t *testing.T, profile string) {
	defer PostMortemLogs(t, profile)

	args := []string{"stop", "-p", profile}
	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Fatalf("stop failed: %q : %v", rr.Command(), err)
	}
}

// validateRestart restarts a cluster
func validateRestart(ctx context.Context, t *testing.T, profile string) {
	defer PostMortemLogs(t, profile)

	args := []string{"start", "-p", profile}
	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Fatalf("restart failed: %q : %v", rr.Command(), err)
	}
}
<commit_msg>skip Hyper-V mount details check<commit_after>//go:build integration
// +build integration

/*
Copyright 2021 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"context"
	"fmt"
	"os/exec"
	"strings"
	"testing"
	"time"
)

const (
	mountGID   = "0"
	mountMSize = "6543"
	mountMode  = "0777"
	mountPort  = "46464"
	mountUID   = "0"
)

// TestMountStart tests using the mount command on start
func TestMountStart(t *testing.T) {
	if NoneDriver() {
		t.Skip("skipping: none driver does not support mount")
	}

	type validateFunc func(context.Context, *testing.T, string)
	profile1 := UniqueProfileName("mount-start-1")
	profile2 := UniqueProfileName("mount-start-2")
	ctx, cancel := context.WithTimeout(context.Background(), Minutes(15))
	defer Cleanup(t, profile1, cancel)
	defer Cleanup(t, profile2, cancel)

	// Serial tests
	t.Run("serial", func(t *testing.T) {
		tests := []struct {
			name      string
			validator validateFunc
			profile   string
		}{
			{"StartWithMountFirst", validateStartWithMount, profile1},
			{"StartWithMountSecond", validateStartWithMount, profile2},
			{"VerifyMountFirst", validateMount, profile1},
			{"VerifyMountSecond", validateMount, profile2},
			{"DeleteFirst", validateDelete, profile1},
			{"VerifyMountPostDelete", validateMount, profile2},
			{"Stop", validateMountStop, profile2},
			{"RestartStopped", validateRestart, profile2},
			{"VerifyMountPostStop", validateMount, profile2},
		}

		for _, test := range tests {
			if ctx.Err() == context.DeadlineExceeded {
				t.Fatalf("Unable to run more tests (deadline exceeded)")
			}

			t.Run(test.name, func(t *testing.T) {
				test.validator(ctx, t, test.profile)
			})
		}
	})
}

// validateStartWithMount starts a cluster with mount enabled
func validateStartWithMount(ctx context.Context, t *testing.T, profile string) {
	defer PostMortemLogs(t, profile)

	args := []string{"start", "-p", profile, "--memory=2048", "--mount", "--mount-gid", mountGID, "--mount-msize", mountMSize, "--mount-mode", mountMode, "--mount-port", mountPort, "--mount-uid", mountUID}
	args = append(args, StartArgs()...)
	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Fatalf("failed to start minikube with args: %q : %v", rr.Command(), err)
	}
}

// validateMount checks if the cluster has a folder mounted
func validateMount(ctx context.Context, t *testing.T, profile string) {
	defer PostMortemLogs(t, profile)

	sshArgs := []string{"-p", profile, "ssh", "--"}

	args := sshArgs
	args = append(args, "ls", "/minikube-host")
	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Fatalf("mount failed: %q : %v", rr.Command(), err)
	}

	// Docker has it's own mounting method, it doesn't respect the mounting flags
	if DockerDriver() {
		return
	}

	args = sshArgs
	args = append(args, "stat", "--format", "'%a'", "/minikube-host")
	rr, err = Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Fatalf("failed to get directory mode: %v", err)
	}

	want := "777"
	if !strings.Contains(rr.Output(), want) {
		t.Errorf("wanted mode to be %q; got: %q", want, rr.Output())
	}

	// We can't get the mount details with Hyper-V
	if HyperVDriver() {
		return
        }

	args = sshArgs
	args = append(args, "mount", "|", "grep", "9p")
	rr, err = Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Fatalf("failed to get mount information: %v", err)
	}

	flags := []struct {
		key      string
		expected string
	}{
		{"gid", mountGID},
		{"msize", mountMSize},
		{"port", mountPort},
		{"uid", mountUID},
	}

	for _, flag := range flags {
		want := fmt.Sprintf("%s=%s", flag.key, flag.expected)
		if !strings.Contains(rr.Output(), want) {
			t.Errorf("wanted gid to be: %q; got: %q", want, rr.Output())
		}
	}
}

// validateMountStop stops a cluster
func validateMountStop(ctx context.Context, t *testing.T, profile string) {
	defer PostMortemLogs(t, profile)

	args := []string{"stop", "-p", profile}
	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Fatalf("stop failed: %q : %v", rr.Command(), err)
	}
}

// validateRestart restarts a cluster
func validateRestart(ctx context.Context, t *testing.T, profile string) {
	defer PostMortemLogs(t, profile)

	args := []string{"start", "-p", profile}
	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Fatalf("restart failed: %q : %v", rr.Command(), err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package agentlib_test

import (
	"bufio"
	"io/ioutil"
	"net"
	"os"
	"path/filepath"
	"reflect"
	"testing"
	"time"

	"v.io/v23/security"
	"v.io/x/ref"
	vsecurity "v.io/x/ref/lib/security"
	_ "v.io/x/ref/runtime/factories/generic"
	"v.io/x/ref/services/agent"
	"v.io/x/ref/services/agent/agentlib"
	"v.io/x/ref/services/agent/internal/constants"
	"v.io/x/ref/test/v23test"
)

func createFakeV23AgentdDir(t *testing.T, sh *v23test.Shell) string {
	d := sh.MakeTempDir()
	if err := ioutil.WriteFile(filepath.Join(d, "v23agentd"), []byte("junk"), 0700); err != nil {
		t.Fatalf("failed to create fake agent: %v", err)
	}
	return d
}

// TestV23AgentPrincipal tests LoadPrincipal, which spawns an agent process.
func TestV23AgentPrincipal(t *testing.T) {
	v23test.SkipUnlessRunningIntegrationTests(t)

	sh := v23test.NewShell(t, nil)
	sh.PropagateChildOutput = true
	// Make sure we enable launching an agent when loading credentials.
	os.Unsetenv(ref.EnvCredentialsNoAgent)
	// This is not strictly needed since we don't create child processes
	// using the shell that need to load credentials using the agent, but
	// it's good practice to use the same setting in the parent and any
	// children.
	delete(sh.Vars, ref.EnvCredentialsNoAgent)
	defer sh.Cleanup()

	agentd := v23test.BuildGoPkg(sh, "v.io/x/ref/services/agent/v23agentd")

	// Create a credentials directory with an unencrypted principal.
	credsDir := sh.MakeTempDir()
	principal, err := vsecurity.CreatePersistentPrincipal(credsDir, nil)
	if err != nil {
		t.Fatal(err)
	}
	if err := vsecurity.InitDefaultBlessings(principal, "happy"); err != nil {
		t.Fatal(err)
	}

	// loadPrincipal uses LoadPrincipal and verifies the blessing is as
	// expected.
	loadPrincipal := func() agent.Principal {
		p, err := agentlib.LoadPrincipal(credsDir)
		if err != nil {
			t.Fatalf("LoadPrincipal(%v) failed: %v", credsDir, err)
		}
		blessings, _ := p.BlessingStore().Default()
		if got, want := security.BlessingNames(p, blessings), []string{"happy"}; !reflect.DeepEqual(got, want) {
			t.Errorf("Got %v want %v", got, want)
		}
		return p
	}

	origPATH := os.Getenv("PATH")
	badAgentDir := createFakeV23AgentdDir(t, sh)
	badAgentPATH := func() {
		// Set up the PATH with an invalid agent.
		if err = os.Setenv("PATH", badAgentDir+":"+origPATH); err != nil {
			t.Fatalf("Setenv failed: %v", err)
		}
	}
	goodAgentPATH := func() {
		// Put v23agentd we built above in the PATH.
		if err = os.Setenv("PATH", filepath.Dir(agentd)+":"+origPATH); err != nil {
			t.Fatalf("Setenv failed: %v", err)
		}
	}

	badAgentPATH()
	// Load the principal locally (since an agent can't be started
	// successfully).
	p1 := loadPrincipal()

	// A subsequent attempt to load the principal should fail since the
	// local loading is exclusive.
	if _, err := agentlib.LoadPrincipal(credsDir); err == nil {
		t.Fatalf("Should have not been able to load the principal.")
	}

	// Even if we put a valid agent in the PATH, it should still not be able
	// to load the principal due to the exclusivity.
	goodAgentPATH()
	if _, err := agentlib.LoadPrincipal(credsDir); err == nil {
		t.Fatalf("Should have not been able to load the principal.")
	}

	// After letting go of the locally loaded principal, a subsequent local
	// load will succeed.
	p1.Close()
	badAgentPATH()
	p1 = loadPrincipal()
	p1.Close()

	// Use the agent now.
	goodAgentPATH()
	p1 = loadPrincipal()
	// Set up a way to communicate with the agent's commands socket.
	cmds, err := net.Dial("unix", filepath.Join(constants.AgentDir(credsDir), "commands"))
	if err != nil {
		t.Fatal(err)
	}
	defer cmds.Close()
	cmdsRead := bufio.NewScanner(cmds)
	sendCmd := func(cmd string) {
		if _, err := cmds.Write([]byte(cmd + "\n")); err != nil {
			t.Fatal(err)
		}
	}
	cmdReply := func() string {
		if !cmdsRead.Scan() {
			t.Fatal("failed to read reply")
		}
		if err := cmdsRead.Err(); err != nil {
			t.Fatal(err)
		}
		return cmdsRead.Text()
	}
	expectReply := func(expected string) {
		if reply := cmdReply(); reply != expected {
			t.Fatalf("expected %v, got %v", expected, reply)
		}
	}

	// There ought to be one client connected to the agent (p1).
	sendCmd("CONNECTIONS")
	expectReply("1")
	// The agent is not idle.
	sendCmd("IDLE")
	expectReply("0.00")

	p2 := loadPrincipal()

	// Now there are two clients (p1 and p2).
	sendCmd("CONNECTIONS")
	expectReply("2")

	// Close p2 and recreate a new principal.
	p2.Close()
	p2 = loadPrincipal()

	// Still two clients.
	sendCmd("CONNECTIONS")
	expectReply("2")

	p1.Close()
	p2.Close()

	// Now the agent should become idle.
	sendCmd("IDLE")
	timeout := time.After(20 * time.Second)
	for cmdReply() == "0.00" {
		sendCmd("IDLE")
		time.Sleep(10 * time.Millisecond)
		select {
		case <-timeout:
			t.Logf("Warning: still waiting to become idle after %v", timeout)
		default:
		}
	}

	// Shorten the grace period to 1 second; the agent should exit.
	sendCmd("GRACE=1")
	expectReply("OK")
	// Scan will block until the agent exits.
	if cmdsRead.Scan() {
		t.Errorf("Agent didn't seem to have exited")
	}

	p1 = loadPrincipal()

	// The agent restarted, so re-establish the commands socket connection.
	cmds, err = net.Dial("unix", filepath.Join(constants.AgentDir(credsDir), "commands"))
	if err != nil {
		t.Fatal(err)
	}
	defer cmds.Close()
	cmdsRead = bufio.NewScanner(cmds)

	// One connection (p1).
	sendCmd("CONNECTIONS")
	expectReply("1")

	// Tell the agent to exit.
	sendCmd("EXIT")
	// Either it replied "OK", or it exited before it was able to reply "OK"
	// (in which case the socket connection would have broken).
	if cmdsRead.Scan() && cmdsRead.Text() != "OK" {
		t.Fatal("failed to OK")
	}
	if err := cmdsRead.Err(); err != nil {
		t.Fatal(err)
	}
}
<commit_msg>x/ref/services/agent/agentlib: make sure that agent locking is always enabled for the test (#74)<commit_after>// Copyright 2016 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package agentlib_test

import (
	"bufio"
	"io/ioutil"
	"net"
	"os"
	"path/filepath"
	"reflect"
	"testing"
	"time"

	"v.io/v23/security"
	"v.io/x/ref"
	vsecurity "v.io/x/ref/lib/security"
	_ "v.io/x/ref/runtime/factories/generic"
	"v.io/x/ref/services/agent"
	"v.io/x/ref/services/agent/agentlib"
	"v.io/x/ref/services/agent/internal/constants"
	"v.io/x/ref/test/v23test"
)

func createFakeV23AgentdDir(t *testing.T, sh *v23test.Shell) string {
	d := sh.MakeTempDir()
	if err := ioutil.WriteFile(filepath.Join(d, "v23agentd"), []byte("junk"), 0700); err != nil {
		t.Fatalf("failed to create fake agent: %v", err)
	}
	return d
}

// TestV23AgentPrincipal tests LoadPrincipal, which spawns an agent process.
func TestV23AgentPrincipal(t *testing.T) {
	v23test.SkipUnlessRunningIntegrationTests(t)

	sh := v23test.NewShell(t, nil)
	sh.PropagateChildOutput = true
	// Make sure we enable launching an agent when loading credentials.
	os.Unsetenv(ref.EnvCredentialsNoAgent)
	// Make sure that locking is enabled.
	os.Unsetenv(ref.EnvDisableCredentialsLocking)
	// This is not strictly needed since we don't create child processes
	// using the shell that need to load credentials using the agent, but
	// it's good practice to use the same setting in the parent and any
	// children.
	delete(sh.Vars, ref.EnvCredentialsNoAgent)
	delete(sh.Vars, ref.EnvDisableCredentialsLocking)
	defer sh.Cleanup()

	agentd := v23test.BuildGoPkg(sh, "v.io/x/ref/services/agent/v23agentd")

	// Create a credentials directory with an unencrypted principal.
	credsDir := sh.MakeTempDir()
	principal, err := vsecurity.CreatePersistentPrincipal(credsDir, nil)
	if err != nil {
		t.Fatal(err)
	}
	if err := vsecurity.InitDefaultBlessings(principal, "happy"); err != nil {
		t.Fatal(err)
	}

	// loadPrincipal uses LoadPrincipal and verifies the blessing is as
	// expected.
	loadPrincipal := func() agent.Principal {
		p, err := agentlib.LoadPrincipal(credsDir)
		if err != nil {
			t.Fatalf("LoadPrincipal(%v) failed: %v", credsDir, err)
		}
		blessings, _ := p.BlessingStore().Default()
		if got, want := security.BlessingNames(p, blessings), []string{"happy"}; !reflect.DeepEqual(got, want) {
			t.Errorf("Got %v want %v", got, want)
		}
		return p
	}

	origPATH := os.Getenv("PATH")
	badAgentDir := createFakeV23AgentdDir(t, sh)
	badAgentPATH := func() {
		// Set up the PATH with an invalid agent.
		if err = os.Setenv("PATH", badAgentDir+":"+origPATH); err != nil {
			t.Fatalf("Setenv failed: %v", err)
		}
	}
	goodAgentPATH := func() {
		// Put v23agentd we built above in the PATH.
		if err = os.Setenv("PATH", filepath.Dir(agentd)+":"+origPATH); err != nil {
			t.Fatalf("Setenv failed: %v", err)
		}
	}

	badAgentPATH()
	// Load the principal locally (since an agent can't be started
	// successfully).
	p1 := loadPrincipal()

	// A subsequent attempt to load the principal should fail since the
	// local loading is exclusive.
	if _, err := agentlib.LoadPrincipal(credsDir); err == nil {
		t.Fatalf("Should have not been able to load the principal.")
	}

	// Even if we put a valid agent in the PATH, it should still not be able
	// to load the principal due to the exclusivity.
	goodAgentPATH()
	if _, err := agentlib.LoadPrincipal(credsDir); err == nil {
		t.Fatalf("Should have not been able to load the principal.")
	}

	// After letting go of the locally loaded principal, a subsequent local
	// load will succeed.
	p1.Close()
	badAgentPATH()
	p1 = loadPrincipal()
	p1.Close()

	// Use the agent now.
	goodAgentPATH()
	p1 = loadPrincipal()
	// Set up a way to communicate with the agent's commands socket.
	cmds, err := net.Dial("unix", filepath.Join(constants.AgentDir(credsDir), "commands"))
	if err != nil {
		t.Fatal(err)
	}
	defer cmds.Close()
	cmdsRead := bufio.NewScanner(cmds)
	sendCmd := func(cmd string) {
		if _, err := cmds.Write([]byte(cmd + "\n")); err != nil {
			t.Fatal(err)
		}
	}
	cmdReply := func() string {
		if !cmdsRead.Scan() {
			t.Fatal("failed to read reply")
		}
		if err := cmdsRead.Err(); err != nil {
			t.Fatal(err)
		}
		return cmdsRead.Text()
	}
	expectReply := func(expected string) {
		if reply := cmdReply(); reply != expected {
			t.Fatalf("expected %v, got %v", expected, reply)
		}
	}

	// There ought to be one client connected to the agent (p1).
	sendCmd("CONNECTIONS")
	expectReply("1")
	// The agent is not idle.
	sendCmd("IDLE")
	expectReply("0.00")

	p2 := loadPrincipal()

	// Now there are two clients (p1 and p2).
	sendCmd("CONNECTIONS")
	expectReply("2")

	// Close p2 and recreate a new principal.
	p2.Close()
	p2 = loadPrincipal()

	// Still two clients.
	sendCmd("CONNECTIONS")
	expectReply("2")

	p1.Close()
	p2.Close()

	// Now the agent should become idle.
	sendCmd("IDLE")
	timeout := time.After(20 * time.Second)
	for cmdReply() == "0.00" {
		sendCmd("IDLE")
		time.Sleep(10 * time.Millisecond)
		select {
		case <-timeout:
			t.Logf("Warning: still waiting to become idle after %v", timeout)
		default:
		}
	}

	// Shorten the grace period to 1 second; the agent should exit.
	sendCmd("GRACE=1")
	expectReply("OK")
	// Scan will block until the agent exits.
	if cmdsRead.Scan() {
		t.Errorf("Agent didn't seem to have exited")
	}

	p1 = loadPrincipal()

	// The agent restarted, so re-establish the commands socket connection.
	cmds, err = net.Dial("unix", filepath.Join(constants.AgentDir(credsDir), "commands"))
	if err != nil {
		t.Fatal(err)
	}
	defer cmds.Close()
	cmdsRead = bufio.NewScanner(cmds)

	// One connection (p1).
	sendCmd("CONNECTIONS")
	expectReply("1")

	// Tell the agent to exit.
	sendCmd("EXIT")
	// Either it replied "OK", or it exited before it was able to reply "OK"
	// (in which case the socket connection would have broken).
	if cmdsRead.Scan() && cmdsRead.Text() != "OK" {
		t.Fatal("failed to OK")
	}
	if err := cmdsRead.Err(); err != nil {
		t.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package use

import (
	"log"
	"text/template"
)

// FuncMap is a template.FuncMap, see docs there. Redefined to minimize imports.
type FuncMap template.FuncMap

// Pkg described single "package" of related functions.
// "Package" can be included into the map with every function, prefixed with
// Prefix. This is namespaces mechanism for poor.
// MapNames is a function invoking for each key within Funcs to convert names
// using custom rules. This can be used to adapt names to use with different
// interpreters.
type Pkg struct {
	Prefix  string
	MapName func(string) string
	Funcs   FuncMap
}

// Packages collects functions from all "packages" in arguments into one
// FuncMap, converting every function name in "package" with Pkg.MapName and
// prefixing it with Pkg.Prefix.
func Packages(pkgs ...Pkg) FuncMap {
	l := 0
	for _, p := range pkgs {
		l += len(p.Funcs)
	}
	m := make(FuncMap, l)
	for _, p := range pkgs {
		for nm, f := range p.Funcs {
			if p.MapName != nil {
				nm = p.MapName(nm)
			}
			name := p.Prefix + nm
			if f2, ok := m[name]; ok {
				log.Printf("use: name clash for '%s', %T, %T", name, f, f2)
			}
			m[name] = f
		}
	}
	return m
}
<commit_msg>Some docs<commit_after>/*
Package use provides simple namespaces for custom functions.
*/
package use

import (
	"log"
	"text/template"
)

// FuncMap is a template.FuncMap, see docs there. Redefined to minimize imports.
type FuncMap template.FuncMap

// Pkg described single "package" of related functions.
// "Package" can be included into the map with every function, prefixed with
// Prefix. This is namespaces mechanism for poor.
// MapNames is a function invoking for each key within Funcs to convert names
// using custom rules. This can be used to adapt names to use with different
// interpreters.
type Pkg struct {
	// Prefix added to function name (can be empty).
	Prefix string
	// Function to translate function name (can be nil).
	MapName func(string) string
	// Map of functions.
	Funcs FuncMap
}

// Packages collects functions from all "packages" in arguments into one
// FuncMap, converting every function name in "package" with Pkg.MapName and
// prefixing it with Pkg.Prefix.
func Packages(pkgs ...Pkg) FuncMap {
	l := 0
	for _, p := range pkgs {
		l += len(p.Funcs)
	}
	m := make(FuncMap, l)
	for _, p := range pkgs {
		for nm, f := range p.Funcs {
			if p.MapName != nil {
				nm = p.MapName(nm)
			}
			name := p.Prefix + nm
			if f2, ok := m[name]; ok {
				log.Printf("use: name clash for '%s', %T, %T", name, f, f2)
			}
			m[name] = f
		}
	}
	return m
}
<|endoftext|>


<commit_before>/*
Package uuid implements a fast representation of UUIDs (Universally Unique Identifiers) and integrates with JSON and SQL drivers.

This package supports reading of multiple formats of UUIDs, including but not limited to:

	a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11
	A0EEBC99-9C0B-4EF8-BB6D-6BB9BD380A11
	{a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11}
	a0eebc999c0b4ef8bb6d6bb9bd380a11
	a0ee-bc99-9c0b-4ef8-bb6d-6bb9-bd38-0a11
	{a0eebc99-9c0b4ef8-bb6d6bb9-bd380a11}

The parsing-speed of UUIDs in this package is achieved in several ways:

A lookup table converts hexadecimal digits to bytes.

Scanning and parsing is done in place without allocating anything.

Resulting bytes are written to the UUID as it is parsed. On parse errors
this will leave the UUID only partially populated with data from the
input string, leaving the rest of the UUID unmodified.

This package just ignores non-hexadecimal digits when scanning. This can cause
some odd representations of hexadecimal data to be parsed as valid UUIDs, and
longer strings like these will parse successfully:

	a0eebc99,9c0b,4ef8,bb6d,6bb9bd380a11
	a0eebc99This9cIs0b4eOKf8bb6d6bb9bdLOL380a11
	a0-ee-bc-99-9c-0b-4e-f8-bb-6d-6b-b9-bd-38-0a-11

However, the hexadecimal digits MUST come in pairs, and the total number of bytes
represented by them MUST equal 16, or it will generate a parse error.
For example, UUIDs like these will not parse:

	a0eebc999-c0b-4ef8-bb6d-6bb9bd380a11
	a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a
	a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a111
	a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a1111

All string-creating functions will generate UUIDs in the canonical format of:

	a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11

*/
package uuid

import (
	"crypto/rand"
	"fmt"
)

// UUID represents a Universally-Unique-Identifier.
type UUID [16]byte

// zero is the zero-UUID, every single byte set to 0.
var zero = [16]byte{}

// ScanError contains the scanner-state for when the error occurred.
type ScanError struct {
	// Scanned is the number of bytes of the source string which has been considered.
	Scanned int
	// Written is the number of decoded hexadecimal bytes which has
	// been written to the UUID instance.
	Written int
	// Length is the length of the source string.
	Length  int
}

// ErrTooShort occurs when the supplied string does not contain enough
// hexadecimal characters to represent a UUID.
type ErrTooShort ScanError

func (e *ErrTooShort) Error() string {
	return fmt.Sprintf("invalid UUID: too few bytes (scanned characters: %d, written bytes: %d, string length: %d)", e.Scanned, e.Written, e.Length)
}

// ErrTooLong occurs when the supplied string contains more than the
// required number of hexadecimal characters to represent a UUID.
type ErrTooLong ScanError

func (e *ErrTooLong) Error() string {
	return fmt.Sprintf("invalid UUID: too many bytes (scanned characters: %d, written bytes: %d, string length: %d)", e.Scanned, e.Written, e.Length)
}

// ErrUneven occurs when a hexadecimal digit is not part of a pair, making it
// impossible to decode it to a byte.
type ErrUneven ScanError

func (e *ErrUneven) Error() string {
	return fmt.Sprintf("invalid UUID: uneven hexadecimal bytes (scanned characters: %d, written bytes: %d, string length: %d)", e.Scanned, e.Written, e.Length)
}

// hexchar2byte contains the integer byte-value represented by a hexadecimal character,
// 255 if it is an invalid character.
var hexchar2byte = []byte{
	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 255, 255, 255, 255, 255, 255,
	255, 10, 11, 12, 13, 14, 15, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	255, 10, 11, 12, 13, 14, 15, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
}

// V4 creates a new random UUID with data from crypto/rand.Read().
func V4() (UUID, error) {
	u := UUID{}

	_, err := rand.Read(u[:])
	if err != nil {
		return u, err
	}

	u[8] = (u[8] | 0x80) & 0xBF
	u[6] = (u[6] | 0x40) & 0x4F

	return u, nil
}

// FromString reads a UUID into a new UUID instance.
func FromString(str string) (UUID, error) {
	u := UUID{}

	err := u.SetString(str)

	return u, err
}

// SetString reads the supplied string-representation of the UUID into the instance.
// On invalid UUID an error is returned and the UUID state will be undetermined.
// This function will ignore all non-hexadecimal digits.
func (u *UUID) SetString(str string) error {
	i := 0
	x := 0
	c := len(str)

	for x < c {
		a := hexchar2byte[str[x]]
		if a == 255 {
			// Invalid char, skip
			x++

			continue
		}

		// We need to perform this check after the attempted hex-read in case
		// we have trailing "}" characters
		if i >= 16 {
			return &ErrTooLong{x, i, c}
		}
		if x+1 >= c {
			// Not enough to scan
			return &ErrTooShort{x, i, c}
		}

		b := hexchar2byte[str[x+1]]
		if b == 255 {
			// Uneven hexadecimal byte
			return &ErrUneven{x, i, c}
		}

		u[i] = (a << 4) | b

		x += 2
		i++
	}

	if i != 16 {
		// Can only be too short here
		return &ErrTooShort{x, i, c}
	}

	return nil
}

// IsZero returns true if the UUID is zero.
func (u UUID) IsZero() bool {
	return u == zero
}

// String returns the string representation of the UUID.
// This method returns the canonical representation of
// ``xxxxxxxx-xxxx-Mxxx-Nxxx-xxxxxxxxxxxx``.
func (u UUID) String() string {
	return fmt.Sprintf("%8.x-%4.x-%4.x-%4.x-%12.x", u[0:4], u[4:6], u[6:8], u[8:10], u[10:16])
}
<commit_msg>Slight change of wording<commit_after>/*
Package uuid implements a fast representation of UUIDs (Universally Unique Identifiers) and integrates with JSON and SQL drivers.

This package supports reading of multiple formats of UUIDs, including but not limited to:

	a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11
	A0EEBC99-9C0B-4EF8-BB6D-6BB9BD380A11
	{a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11}
	a0eebc999c0b4ef8bb6d6bb9bd380a11
	a0ee-bc99-9c0b-4ef8-bb6d-6bb9-bd38-0a11
	{a0eebc99-9c0b4ef8-bb6d6bb9-bd380a11}

The parsing-speed of UUIDs in this package is achieved in several ways:

A lookup table converts hexadecimal digits to bytes.

Scanning and parsing is done in place without allocating anything.

Resulting bytes are written to the UUID as it is parsed. On parse errors
this will leave the UUID only partially populated with data from the
input string, leaving the rest of the UUID unmodified.

This package just ignores non-hexadecimal digits when scanning. This can cause
some odd representations of hexadecimal data to be parsed as valid UUIDs, and
longer strings like these will parse successfully:

	a0eebc99,9c0b,4ef8,bb6d,6bb9bd380a11
	a0eebc99This9cIs0b4eOKf8bb6d6bb9bdLOL380a11
	a0-ee-bc-99-9c-0b-4e-f8-bb-6d-6b-b9-bd-38-0a-11

However, the hexadecimal digits MUST come in pairs, and the total number of bytes
represented by them MUST equal 16, or it will generate a parse error.
For example, invalid UUIDs like these will not parse:

	a0eebc999-c0b-4ef8-bb6d-6bb9bd380a11
	a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a
	a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a111
	a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a1111

All string-creating functions will generate UUIDs in the canonical format of:

	a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11

*/
package uuid

import (
	"crypto/rand"
	"fmt"
)

// UUID represents a Universally-Unique-Identifier.
type UUID [16]byte

// zero is the zero-UUID, every single byte set to 0.
var zero = [16]byte{}

// ScanError contains the scanner-state for when the error occurred.
type ScanError struct {
	// Scanned is the number of bytes of the source string which has been considered.
	Scanned int
	// Written is the number of decoded hexadecimal bytes which has
	// been written to the UUID instance.
	Written int
	// Length is the length of the source string.
	Length int
}

// ErrTooShort occurs when the supplied string does not contain enough
// hexadecimal characters to represent a UUID.
type ErrTooShort ScanError

func (e *ErrTooShort) Error() string {
	return fmt.Sprintf("invalid UUID: too few bytes (scanned characters: %d, written bytes: %d, string length: %d)", e.Scanned, e.Written, e.Length)
}

// ErrTooLong occurs when the supplied string contains more than the
// required number of hexadecimal characters to represent a UUID.
type ErrTooLong ScanError

func (e *ErrTooLong) Error() string {
	return fmt.Sprintf("invalid UUID: too many bytes (scanned characters: %d, written bytes: %d, string length: %d)", e.Scanned, e.Written, e.Length)
}

// ErrUneven occurs when a hexadecimal digit is not part of a pair, making it
// impossible to decode it to a byte.
type ErrUneven ScanError

func (e *ErrUneven) Error() string {
	return fmt.Sprintf("invalid UUID: uneven hexadecimal bytes (scanned characters: %d, written bytes: %d, string length: %d)", e.Scanned, e.Written, e.Length)
}

// hexchar2byte contains the integer byte-value represented by a hexadecimal character,
// 255 if it is an invalid character.
var hexchar2byte = []byte{
	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 255, 255, 255, 255, 255, 255,
	255, 10, 11, 12, 13, 14, 15, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	255, 10, 11, 12, 13, 14, 15, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
}

// V4 creates a new random UUID with data from crypto/rand.Read().
func V4() (UUID, error) {
	u := UUID{}

	_, err := rand.Read(u[:])
	if err != nil {
		return u, err
	}

	u[8] = (u[8] | 0x80) & 0xBF
	u[6] = (u[6] | 0x40) & 0x4F

	return u, nil
}

// FromString reads a UUID into a new UUID instance.
func FromString(str string) (UUID, error) {
	u := UUID{}

	err := u.SetString(str)

	return u, err
}

// SetString reads the supplied string-representation of the UUID into the instance.
// On invalid UUID an error is returned and the UUID state will be undetermined.
// This function will ignore all non-hexadecimal digits.
func (u *UUID) SetString(str string) error {
	i := 0
	x := 0
	c := len(str)

	for x < c {
		a := hexchar2byte[str[x]]
		if a == 255 {
			// Invalid char, skip
			x++

			continue
		}

		// We need to perform this check after the attempted hex-read in case
		// we have trailing "}" characters
		if i >= 16 {
			return &ErrTooLong{x, i, c}
		}
		if x+1 >= c {
			// Not enough to scan
			return &ErrTooShort{x, i, c}
		}

		b := hexchar2byte[str[x+1]]
		if b == 255 {
			// Uneven hexadecimal byte
			return &ErrUneven{x, i, c}
		}

		u[i] = (a << 4) | b

		x += 2
		i++
	}

	if i != 16 {
		// Can only be too short here
		return &ErrTooShort{x, i, c}
	}

	return nil
}

// IsZero returns true if the UUID is zero.
func (u UUID) IsZero() bool {
	return u == zero
}

// String returns the string representation of the UUID.
// This method returns the canonical representation of
// ``xxxxxxxx-xxxx-Mxxx-Nxxx-xxxxxxxxxxxx``.
func (u UUID) String() string {
	return fmt.Sprintf("%8.x-%4.x-%4.x-%4.x-%12.x", u[0:4], u[4:6], u[6:8], u[8:10], u[10:16])
}
<|endoftext|>


<commit_before>// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
// Helpers used by multiple sub-commands.
// Author: mpolden

package cmd

import (
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/vespa-engine/vespa/client/go/vespa"
)

var exitFunc = os.Exit // To allow overriding Exit in tests

func fatalErrHint(err error, hints ...string) {
	printErrHint(err, hints...)
	exitFunc(1)
}

func fatalErr(err error, msg ...interface{}) {
	printErr(err, msg...)
	exitFunc(1)
}

func printErrHint(err error, hints ...string) {
	if err != nil {
		printErr(nil, err.Error())
	}
	for _, hint := range hints {
		fmt.Fprintln(stderr, color.Cyan("Hint:"), hint)
	}
}

func printErr(err error, msg ...interface{}) {
	if len(msg) > 0 {
		fmt.Fprintln(stderr, color.Red("Error:"), fmt.Sprint(msg...))
	}
	if err != nil {
		fmt.Fprintln(stderr, color.Yellow(err))
	}
}

func printSuccess(msg ...interface{}) {
	log.Print(color.Green("Success: "), fmt.Sprint(msg...))
}

func vespaCliHome() (string, error) {
	home := os.Getenv("VESPA_CLI_HOME")
	if home == "" {
		userHome, err := os.UserHomeDir()
		if err != nil {
			return "", err
		}
		home = filepath.Join(userHome, ".vespa")
	}
	if err := os.MkdirAll(home, 0700); err != nil {
		return "", err
	}
	return home, nil
}

func vespaCliCacheDir() (string, error) {
	cacheDir := os.Getenv("VESPA_CLI_CACHE_DIR")
	if cacheDir == "" {
		userCacheDir, err := os.UserCacheDir()
		if err != nil {
			return "", err
		}
		cacheDir = filepath.Join(userCacheDir, "vespa")
	}
	if err := os.MkdirAll(cacheDir, 0755); err != nil {
		return "", err
	}
	return cacheDir, nil
}

func deploymentFromArgs() vespa.Deployment {
	zone, err := vespa.ZoneFromString(zoneArg)
	if err != nil {
		fatalErrHint(err, "Zone format is <env>.<region>")
	}
	app := getApplication()
	return vespa.Deployment{Application: app, Zone: zone}
}

func applicationSource(args []string) string {
	if len(args) > 0 {
		return args[0]
	}
	return "."
}

func getApplication() vespa.ApplicationID {
	cfg, err := LoadConfig()
	if err != nil {
		fatalErr(err, "Could not load config")
		return vespa.ApplicationID{}
	}
	app, err := cfg.Get(applicationFlag)
	if err != nil {
		fatalErrHint(err, "No application specified. Try the --"+applicationFlag+" flag")
		return vespa.ApplicationID{}
	}
	application, err := vespa.ApplicationFromString(app)
	if err != nil {
		fatalErrHint(err, "Application format is <tenant>.<app>.<instance>")
		return vespa.ApplicationID{}
	}
	return application
}

func getTargetType() string {
	cfg, err := LoadConfig()
	if err != nil {
		fatalErr(err, "Could not load config")
		return ""
	}
	target, err := cfg.Get(targetFlag)
	if err != nil {
		fatalErr(err, "A valid target must be specified")
	}
	return target
}

func getService(service string, sessionOrRunID int64, cluster string) *vespa.Service {
	t := getTarget()
	timeout := time.Duration(waitSecsArg) * time.Second
	if timeout > 0 {
		log.Printf("Waiting up to %d %s for service to become available ...", color.Cyan(waitSecsArg), color.Cyan("seconds"))
	}
	s, err := t.Service(service, timeout, sessionOrRunID, cluster)
	if err != nil {
		fatalErr(err, "Invalid service: ", service)
	}
	return s
}

func getEndpointsOverride() string { return os.Getenv("VESPA_CLI_ENDPOINTS") }

func getSystem() string { return os.Getenv("VESPA_CLI_CLOUD_SYSTEM") }

func getSystemName() string {
	if getSystem() == "publiccd" {
		return "publiccd"
	}
	return "public"
}

func getConsoleURL() string {
	if getSystem() == "publiccd" {
		return "https://console-cd.vespa.oath.cloud"
	}
	return "https://console.vespa.oath.cloud"

}

func getApiURL() string {
	if getSystem() == "publiccd" {
		return "https://api.vespa-external-cd.aws.oath.cloud:4443"
	}
	return "https://api.vespa-external.aws.oath.cloud:4443"
}

func getTarget() vespa.Target {
	targetType := getTargetType()
	if strings.HasPrefix(targetType, "http") {
		return vespa.CustomTarget(targetType)
	}
	switch targetType {
	case "local":
		return vespa.LocalTarget()
	case "cloud":
		cfg, err := LoadConfig()
		if err != nil {
			fatalErr(err, "Could not load config")
			return nil
		}
		deployment := deploymentFromArgs()

		var apiKey []byte = nil
		apiKey, err = ioutil.ReadFile(cfg.APIKeyPath(deployment.Application.Tenant))
		if !vespa.Auth0AccessTokenEnabled() {
			if err != nil {
				fatalErrHint(err, "Deployment to cloud requires an API key. Try 'vespa api-key'")
			}
		}
		privateKeyFile, err := cfg.PrivateKeyPath(deployment.Application)
		if err != nil {
			fatalErr(err)
			return nil
		}
		certificateFile, err := cfg.CertificatePath(deployment.Application)
		if err != nil {
			fatalErr(err)
			return nil
		}
		kp, err := tls.LoadX509KeyPair(certificateFile, privateKeyFile)
		if err != nil {
			fatalErrHint(err, "Deployment to cloud requires a certificate. Try 'vespa cert'")
		}
		var cloudAuth string
		if vespa.Auth0AccessTokenEnabled() {
			cloudAuth, err = cfg.Get(cloudAuthFlag)
			if err != nil {
				if apiKey != nil {
					cloudAuth = "api-key"
				} else {
					cloudAuth = "access-token"
				}
			}
		} else {
			cloudAuth = ""
		}

		return vespa.CloudTarget(getApiURL(), deployment, apiKey,
			vespa.TLSOptions{
				KeyPair:         kp,
				CertificateFile: certificateFile,
				PrivateKeyFile:  privateKeyFile,
			},
			vespa.LogOptions{
				Writer: stdout,
				Level:  vespa.LogLevel(logLevelArg),
			},
			cfg.AuthConfigPath(),
			getSystemName(),
			cloudAuth,
			getEndpointsFromEnv())
	}
	fatalErrHint(fmt.Errorf("Invalid target: %s", targetType), "Valid targets are 'local', 'cloud' or an URL")
	return nil
}

func waitForService(service string, sessionOrRunID int64) {
	s := getService(service, sessionOrRunID, "")
	timeout := time.Duration(waitSecsArg) * time.Second
	if timeout > 0 {
		log.Printf("Waiting up to %d %s for service to become ready ...", color.Cyan(waitSecsArg), color.Cyan("seconds"))
	}
	status, err := s.Wait(timeout)
	if status/100 == 2 {
		log.Print(s.Description(), " at ", color.Cyan(s.BaseURL), " is ", color.Green("ready"))
	} else {
		if err == nil {
			err = fmt.Errorf("Status %d", status)
		}
		fatalErr(err, s.Description(), " at ", color.Cyan(s.BaseURL), " is ", color.Red("not ready"))
	}
}

func getDeploymentOpts(cfg *Config, pkg vespa.ApplicationPackage, target vespa.Target) vespa.DeploymentOpts {
	opts := vespa.DeploymentOpts{ApplicationPackage: pkg, Target: target}
	if opts.IsCloud() {
		deployment := deploymentFromArgs()
		if !opts.ApplicationPackage.HasCertificate() {
			fatalErrHint(fmt.Errorf("Missing certificate in application package"), "Applications in Vespa Cloud require a certificate", "Try 'vespa cert'")
			return opts
		}
		var err error
		opts.APIKey, err = cfg.ReadAPIKey(deployment.Application.Tenant)
		if !vespa.Auth0AccessTokenEnabled() {
			if err != nil {
				fatalErrHint(err, "Deployment to cloud requires an API key. Try 'vespa api-key'")
				return opts
			}
		}
		opts.Deployment = deployment
	}
	return opts
}

func getEndpointsFromEnv() map[string]string {
	endpointsString := getEndpointsOverride()
	if endpointsString == "" {
		return nil
	}

	var endpoints endpoints
	urlsByCluster := make(map[string]string)
	if err := json.Unmarshal([]byte(endpointsString), &endpoints); err != nil {
		fatalErrHint(err, "Endpoints must be valid JSON")
	}
	if len(endpoints.Endpoints) == 0 {
		fatalErr(fmt.Errorf("endpoints must be non-empty"))
	}
	for _, endpoint := range endpoints.Endpoints {
		urlsByCluster[endpoint.Cluster] = endpoint.URL
	}
	return urlsByCluster
}

type endpoints struct {
	Endpoints []endpoint `json:"endpoints"`
}

type endpoint struct {
	Cluster string `json:"cluster"`
	URL     string `json:"url"`
}
<commit_msg>No need for API key if endpoints are overridden<commit_after>// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
// Helpers used by multiple sub-commands.
// Author: mpolden

package cmd

import (
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/vespa-engine/vespa/client/go/vespa"
)

var exitFunc = os.Exit // To allow overriding Exit in tests

func fatalErrHint(err error, hints ...string) {
	printErrHint(err, hints...)
	exitFunc(1)
}

func fatalErr(err error, msg ...interface{}) {
	printErr(err, msg...)
	exitFunc(1)
}

func printErrHint(err error, hints ...string) {
	if err != nil {
		printErr(nil, err.Error())
	}
	for _, hint := range hints {
		fmt.Fprintln(stderr, color.Cyan("Hint:"), hint)
	}
}

func printErr(err error, msg ...interface{}) {
	if len(msg) > 0 {
		fmt.Fprintln(stderr, color.Red("Error:"), fmt.Sprint(msg...))
	}
	if err != nil {
		fmt.Fprintln(stderr, color.Yellow(err))
	}
}

func printSuccess(msg ...interface{}) {
	log.Print(color.Green("Success: "), fmt.Sprint(msg...))
}

func vespaCliHome() (string, error) {
	home := os.Getenv("VESPA_CLI_HOME")
	if home == "" {
		userHome, err := os.UserHomeDir()
		if err != nil {
			return "", err
		}
		home = filepath.Join(userHome, ".vespa")
	}
	if err := os.MkdirAll(home, 0700); err != nil {
		return "", err
	}
	return home, nil
}

func vespaCliCacheDir() (string, error) {
	cacheDir := os.Getenv("VESPA_CLI_CACHE_DIR")
	if cacheDir == "" {
		userCacheDir, err := os.UserCacheDir()
		if err != nil {
			return "", err
		}
		cacheDir = filepath.Join(userCacheDir, "vespa")
	}
	if err := os.MkdirAll(cacheDir, 0755); err != nil {
		return "", err
	}
	return cacheDir, nil
}

func deploymentFromArgs() vespa.Deployment {
	zone, err := vespa.ZoneFromString(zoneArg)
	if err != nil {
		fatalErrHint(err, "Zone format is <env>.<region>")
	}
	app := getApplication()
	return vespa.Deployment{Application: app, Zone: zone}
}

func applicationSource(args []string) string {
	if len(args) > 0 {
		return args[0]
	}
	return "."
}

func getApplication() vespa.ApplicationID {
	cfg, err := LoadConfig()
	if err != nil {
		fatalErr(err, "Could not load config")
		return vespa.ApplicationID{}
	}
	app, err := cfg.Get(applicationFlag)
	if err != nil {
		fatalErrHint(err, "No application specified. Try the --"+applicationFlag+" flag")
		return vespa.ApplicationID{}
	}
	application, err := vespa.ApplicationFromString(app)
	if err != nil {
		fatalErrHint(err, "Application format is <tenant>.<app>.<instance>")
		return vespa.ApplicationID{}
	}
	return application
}

func getTargetType() string {
	cfg, err := LoadConfig()
	if err != nil {
		fatalErr(err, "Could not load config")
		return ""
	}
	target, err := cfg.Get(targetFlag)
	if err != nil {
		fatalErr(err, "A valid target must be specified")
	}
	return target
}

func getService(service string, sessionOrRunID int64, cluster string) *vespa.Service {
	t := getTarget()
	timeout := time.Duration(waitSecsArg) * time.Second
	if timeout > 0 {
		log.Printf("Waiting up to %d %s for service to become available ...", color.Cyan(waitSecsArg), color.Cyan("seconds"))
	}
	s, err := t.Service(service, timeout, sessionOrRunID, cluster)
	if err != nil {
		fatalErr(err, "Invalid service: ", service)
	}
	return s
}

func getEndpointsOverride() string { return os.Getenv("VESPA_CLI_ENDPOINTS") }

func getSystem() string { return os.Getenv("VESPA_CLI_CLOUD_SYSTEM") }

func getSystemName() string {
	if getSystem() == "publiccd" {
		return "publiccd"
	}
	return "public"
}

func getConsoleURL() string {
	if getSystem() == "publiccd" {
		return "https://console-cd.vespa.oath.cloud"
	}
	return "https://console.vespa.oath.cloud"

}

func getApiURL() string {
	if getSystem() == "publiccd" {
		return "https://api.vespa-external-cd.aws.oath.cloud:4443"
	}
	return "https://api.vespa-external.aws.oath.cloud:4443"
}

func getTarget() vespa.Target {
	targetType := getTargetType()
	if strings.HasPrefix(targetType, "http") {
		return vespa.CustomTarget(targetType)
	}
	switch targetType {
	case "local":
		return vespa.LocalTarget()
	case "cloud":
		cfg, err := LoadConfig()
		if err != nil {
			fatalErr(err, "Could not load config")
			return nil
		}
		deployment := deploymentFromArgs()
		endpoints := getEndpointsFromEnv()

		var apiKey []byte = nil
		apiKey, err = ioutil.ReadFile(cfg.APIKeyPath(deployment.Application.Tenant))
		if !vespa.Auth0AccessTokenEnabled() && endpoints == nil {
			if err != nil {
				fatalErrHint(err, "Deployment to cloud requires an API key. Try 'vespa api-key'")
			}
		}
		privateKeyFile, err := cfg.PrivateKeyPath(deployment.Application)
		if err != nil {
			fatalErr(err)
			return nil
		}
		certificateFile, err := cfg.CertificatePath(deployment.Application)
		if err != nil {
			fatalErr(err)
			return nil
		}
		kp, err := tls.LoadX509KeyPair(certificateFile, privateKeyFile)
		if err != nil {
			fatalErrHint(err, "Deployment to cloud requires a certificate. Try 'vespa cert'")
		}
		var cloudAuth string
		if vespa.Auth0AccessTokenEnabled() {
			cloudAuth, err = cfg.Get(cloudAuthFlag)
			if err != nil {
				if apiKey != nil {
					cloudAuth = "api-key"
				} else {
					cloudAuth = "access-token"
				}
			}
		} else {
			cloudAuth = ""
		}

		return vespa.CloudTarget(getApiURL(), deployment, apiKey,
			vespa.TLSOptions{
				KeyPair:         kp,
				CertificateFile: certificateFile,
				PrivateKeyFile:  privateKeyFile,
			},
			vespa.LogOptions{
				Writer: stdout,
				Level:  vespa.LogLevel(logLevelArg),
			},
			cfg.AuthConfigPath(),
			getSystemName(),
			cloudAuth,
			endpoints)
	}
	fatalErrHint(fmt.Errorf("Invalid target: %s", targetType), "Valid targets are 'local', 'cloud' or an URL")
	return nil
}

func waitForService(service string, sessionOrRunID int64) {
	s := getService(service, sessionOrRunID, "")
	timeout := time.Duration(waitSecsArg) * time.Second
	if timeout > 0 {
		log.Printf("Waiting up to %d %s for service to become ready ...", color.Cyan(waitSecsArg), color.Cyan("seconds"))
	}
	status, err := s.Wait(timeout)
	if status/100 == 2 {
		log.Print(s.Description(), " at ", color.Cyan(s.BaseURL), " is ", color.Green("ready"))
	} else {
		if err == nil {
			err = fmt.Errorf("Status %d", status)
		}
		fatalErr(err, s.Description(), " at ", color.Cyan(s.BaseURL), " is ", color.Red("not ready"))
	}
}

func getDeploymentOpts(cfg *Config, pkg vespa.ApplicationPackage, target vespa.Target) vespa.DeploymentOpts {
	opts := vespa.DeploymentOpts{ApplicationPackage: pkg, Target: target}
	if opts.IsCloud() {
		deployment := deploymentFromArgs()
		if !opts.ApplicationPackage.HasCertificate() {
			fatalErrHint(fmt.Errorf("Missing certificate in application package"), "Applications in Vespa Cloud require a certificate", "Try 'vespa cert'")
			return opts
		}
		var err error
		opts.APIKey, err = cfg.ReadAPIKey(deployment.Application.Tenant)
		if !vespa.Auth0AccessTokenEnabled() {
			if err != nil {
				fatalErrHint(err, "Deployment to cloud requires an API key. Try 'vespa api-key'")
				return opts
			}
		}
		opts.Deployment = deployment
	}
	return opts
}

func getEndpointsFromEnv() map[string]string {
	endpointsString := getEndpointsOverride()
	if endpointsString == "" {
		return nil
	}

	var endpoints endpoints
	urlsByCluster := make(map[string]string)
	if err := json.Unmarshal([]byte(endpointsString), &endpoints); err != nil {
		fatalErrHint(err, "Endpoints must be valid JSON")
	}
	if len(endpoints.Endpoints) == 0 {
		fatalErr(fmt.Errorf("endpoints must be non-empty"))
	}
	for _, endpoint := range endpoints.Endpoints {
		urlsByCluster[endpoint.Cluster] = endpoint.URL
	}
	return urlsByCluster
}

type endpoints struct {
	Endpoints []endpoint `json:"endpoints"`
}

type endpoint struct {
	Cluster string `json:"cluster"`
	URL     string `json:"url"`
}
<|endoftext|>


<commit_before>package inliner

import (
	"fmt"
	"testing"
)

// Simple rule inlining with two declarations
func TestInliner(t *testing.T) {
	input := `<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">

<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>

      Inline me please!

`

	expectedOutput := `<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>

      Inline me please!

`

	output, err := Inline(input)
	if err != nil {
		t.Fatal("Failed to inline html:", err)
	}

	if output != expectedOutput {
		t.Fatal(fmt.Sprintf("CSS inliner error\nExpected:\n\"%s\"\nGot:\n\"%s\"", expectedOutput, output))
	}
}

// Alreadu inlined style has more priority than

      Inline me please!

`

	expectedOutput := `<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>

      Inline me please!

`

	output, err := Inline(input)
	if err != nil {
		t.Fatal("Failed to inline html:", err)
	}

	if output != expectedOutput {
		t.Fatal(fmt.Sprintf("CSS inliner error\nExpected:\n\"%s\"\nGot:\n\"%s\"", expectedOutput, output))
	}
}

// Pseudo-class selectors can't be inlined
func TestNotInlinable(t *testing.T) {
	input := `<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">

<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>

      Superbe website

`

	expectedOutput := `<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>

      Superbe website

`

	output, err := Inline(input)
	if err != nil {
		t.Fatal("Failed to inline html:", err)
	}

	if output != expectedOutput {
		t.Fatal(fmt.Sprintf("CSS inliner error\nExpected:\n\"%s\"\nGot:\n\"%s\"", expectedOutput, output))
	}
}

// Some styles causes insertion of additional element attributes
func TestStyleToAttr(t *testing.T) {
	input := `<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">

<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>

      Inline me please!

`

	expectedOutput := `<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>

      Inline me please!

`

	output, err := Inline(input)
	if err != nil {
		t.Fatal("Failed to inline html:", err)
	}

	if output != expectedOutput {
		t.Fatal(fmt.Sprintf("CSS inliner error\nExpected:\n\"%s\"\nGot:\n\"%s\"", expectedOutput, output))
	}
}
<commit_msg>More test<commit_after>package inliner

import (
	"fmt"
	"testing"
)

// Simple rule inlining with two declarations
func TestInliner(t *testing.T) {
	input := `<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">

<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>

      Inline me please!

`

	expectedOutput := `<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>

      Inline me please!

`

	output, err := Inline(input)
	if err != nil {
		t.Fatal("Failed to inline html:", err)
	}

	if output != expectedOutput {
		t.Fatal(fmt.Sprintf("CSS inliner error\nExpected:\n\"%s\"\nGot:\n\"%s\"", expectedOutput, output))
	}
}

// Already inlined style has more priority than

      Inline me please!

`

	expectedOutput := `<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>

      Inline me please!

`

	output, err := Inline(input)
	if err != nil {
		t.Fatal("Failed to inline html:", err)
	}

	if output != expectedOutput {
		t.Fatal(fmt.Sprintf("CSS inliner error\nExpected:\n\"%s\"\nGot:\n\"%s\"", expectedOutput, output))
	}
}

// Pseudo-class selectors can't be inlined
func TestNotInlinable(t *testing.T) {
	input := `<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">

<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>

      Superbe website

`

	expectedOutput := `<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>

      Superbe website

`

	output, err := Inline(input)
	if err != nil {
		t.Fatal("Failed to inline html:", err)
	}

	if output != expectedOutput {
		t.Fatal(fmt.Sprintf("CSS inliner error\nExpected:\n\"%s\"\nGot:\n\"%s\"", expectedOutput, output))
	}
}

// Some styles causes insertion of additional element attributes
func TestStyleToAttr(t *testing.T) {
	input := `<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">

<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>

test

test

test

test

test

test

test

test

`

	expectedOutput := `<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>

test

test

test

test

test

test

test

test

`

	output, err := Inline(input)
	if err != nil {
		t.Fatal("Failed to inline html:", err)
	}

	if output != expectedOutput {
		t.Fatal(fmt.Sprintf("CSS inliner error\nExpected:\n\"%s\"\nGot:\n\"%s\"", expectedOutput, output))
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	//	"flag"
	"fmt"
	"io/ioutil"
	//	"os"
)

type JsonImport struct {
	// File containing the Json
	Filename string

	flattened map[string]string
}

func (ji *JsonImport) ParseFlags(args []string) {

}

func (ji *JsonImport) keysFromJson(nested interface{}, prefix string) {
	for k, v := range nested.(map[string]interface{}) {
		newPrefix := fmt.Sprintf("%s/%s", prefix, k)
		switch v.(type) {
		case string:
			ji.flattened[newPrefix] = v.(string)
		case interface{}:
			ji.keysFromJson(v, newPrefix)
		}
	}
}

func (ji *JsonImport) Run() {
	var unmarshalled map[string]interface{}

	ji.flattened = make(map[string]string)

	fileOutput, err := ioutil.ReadFile(ji.Filename)
	if err != nil {
		panic(err)
	}

	// Import the json file
	if err := json.Unmarshal(fileOutput, &unmarshalled); err != nil {
		panic(err)
	}

	ji.keysFromJson(unmarshalled, "")

	// Set the consul values
}
<commit_msg>Actually upload the keys<commit_after>package main

import (
	"encoding/json"
	//	"flag"
	"fmt"
	"io/ioutil"
	//	"os"
	"github.com/hashicorp/consul/api"
)

type JsonImport struct {
	// Prefix to load the config under. If empty then loads to the
	// root kv node.
	Prefix string
	// File containing the Json
	Filename string

	flattened map[string]string
}

func (ji *JsonImport) ParseFlags(args []string) {

}

func (ji *JsonImport) keysFromJson(nested interface{}, prefix string) {
	for k, v := range nested.(map[string]interface{}) {
		newPrefix := fmt.Sprintf("%s/%s", prefix, k)
		switch v.(type) {
		case string:
			ji.flattened[newPrefix] = v.(string)
		case interface{}:
			ji.keysFromJson(v, newPrefix)
		}
	}
}

func (ji *JsonImport) readFile() (unmarshalled map[string]interface{}) {
	fileOutput, err := ioutil.ReadFile(ji.Filename)
	if err != nil {
		panic(err)
	}

	// Import the json file
	if err := json.Unmarshal(fileOutput, &unmarshalled); err != nil {
		panic(err)
	}

	return
}

func (ji *JsonImport) prefixedKey(key string) string {
	return fmt.Sprintf("%s/%s", ji.Prefix, key)
}

func (ji *JsonImport) setConsulValues() {
	client, _ := api.NewClient(api.DefaultConfig())
	kv := client.KV() // Lookup the pair

	for k, v := range ji.flattened {
		p := &api.KVPair{
			Key:   ji.prefixedKey(k),
			Value: []byte(v),
		}

		_, err := kv.Put(p, nil)
		if err != nil {
			panic(err)
		}
	}
}

func (ji *JsonImport) Run() {
	unmarshalled := ji.readFile()

	ji.flattened = make(map[string]string)
	ji.keysFromJson(unmarshalled, "")
	ji.setConsulValues()
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"encoding/csv"
	"html/template"
	"os"
	"time"
)

type Import struct {
	Id uint64
	PostId uint64
	Source string
	Identifier string
}

func NewImport() (*Import) {
	return &Import{0, 0, "", ""}
}

func ImportBySourceIdentifier(source, identifier string) (*Import, error) {
	row := db.QueryRow("SELECT id, post, source, identifier FROM import WHERE source = $1 AND identifier = $2 LIMIT 1",
		source, identifier)

	var id, postid uint64
	err := row.Scan(&id, &postid, &source, &identifier)
	if err != nil {
		return nil, err
	}

	i := &Import{id, postid, source, identifier}
	return i, nil
}

func (i *Import) Save() (err error) {
	if i.Id == 0 {
		row := db.QueryRow("INSERT INTO import (post, source, identifier) values ($1, $2, $3) RETURNING id",
			i.PostId, i.Source, i.Identifier)
		var id uint64
		err = row.Scan(&id)
		if err != nil {
			return
		}
		i.Id = id
	} else {
		_, err = db.Exec("UPDATE import SET post = $2, source = $3, identifier = $4 WHERE id = $1",
			i.Id, i.PostId, i.Source, i.Identifier)
	}
	return
}

func ImportThinkup(path string) {
	logr.Debugln("Importing from Thinkup export", path)
	port, err := os.Open(path)
	if err != nil {
		logr.Errln("Error opening", path, "for import:", err.Error())
		return
	}

	r := csv.NewReader(port)
	// There may be missing header columns, so turn off field count checking.
	r.FieldsPerRecord = -1

	head, err := r.Read()
	if err != nil {
		logr.Errln("Error reading from import file", path, ":", err.Error())
		return
	}

	count := 0
	for {
		record, err := r.Read()
		if err != nil {
			break
		}

		data := make(map[string]string)
		for i, field := range head {
			data[field] = record[i]
		}

		// TODO: import replies, once there's something reasonable to import them as.
		if data["in_reply_to_post_id"] != "" {
			logr.Debugln("Skipping post (twitter,", data["post_id"], ") as it is a reply")
			continue
		}
		// TODO: import repeats, once there's something reasonable to import them as.
		if data["in_retweet_of_post_id"] != "" {
			logr.Debugln("Skipping post (twitter,", data["post_id"], ") as it is a repeat")
			continue
		}

		// okay now what
		im, err := ImportBySourceIdentifier("twitter", data["post_id"])
		if err == sql.ErrNoRows {
			im = NewImport()
			im.Source = "twitter"
			im.Identifier = data["post_id"]
		} else if err != nil {
			logr.Errln("Error searching for existing imported post (twitter,", data["post_id"], "):", err.Error())
			return
		}

		var post *Post
		if im.PostId != 0 {
			post, err = PostById(im.PostId)
			if err != nil {
				logr.Errln("Error loading already-imported post", im.PostId, "for twitter post", im.Identifier, ":", err.Error())
				return
			}
		} else {
			post = NewPost()
		}

		post.Posted, err = time.Parse("2006-01-02 15:04:05", data["pub_date"])
		if err != nil {
			logr.Errln("Error parsing publish time", data["pub_date"], "for twitter post", data["post_id"], ":", err.Error())
			return
		}

		// TODO: make the links link.
		post.Html = template.HTMLEscapeString(data["post_text"])

		// TODO: store the source?
		// TODO: store the geoplace

		err = post.Save()
		if err != nil {
			logr.Errln("Error saving imported post:", err.Error())
			return
		}

		im.PostId = post.Id
		err = im.Save()
		if err != nil {
			logr.Errln("Error saving import notation for post", im.PostId, ":", err.Error())
			return
		}

		logr.Debugln("Imported post (twitter,", im.Identifier, ")")
		count++
	}
	if err != nil {
		logr.Errln("Error reading import records:", err.Error())
		return
	}

	logr.Debugln("Finished importing", count, "posts!")
}
<commit_msg>Honor line feeds in imported posts<commit_after>package main

import (
	"database/sql"
	"encoding/csv"
	"html/template"
	"os"
	"strings"
	"time"
)

type Import struct {
	Id uint64
	PostId uint64
	Source string
	Identifier string
}

func NewImport() (*Import) {
	return &Import{0, 0, "", ""}
}

func ImportBySourceIdentifier(source, identifier string) (*Import, error) {
	row := db.QueryRow("SELECT id, post, source, identifier FROM import WHERE source = $1 AND identifier = $2 LIMIT 1",
		source, identifier)

	var id, postid uint64
	err := row.Scan(&id, &postid, &source, &identifier)
	if err != nil {
		return nil, err
	}

	i := &Import{id, postid, source, identifier}
	return i, nil
}

func (i *Import) Save() (err error) {
	if i.Id == 0 {
		row := db.QueryRow("INSERT INTO import (post, source, identifier) values ($1, $2, $3) RETURNING id",
			i.PostId, i.Source, i.Identifier)
		var id uint64
		err = row.Scan(&id)
		if err != nil {
			return
		}
		i.Id = id
	} else {
		_, err = db.Exec("UPDATE import SET post = $2, source = $3, identifier = $4 WHERE id = $1",
			i.Id, i.PostId, i.Source, i.Identifier)
	}
	return
}

func ImportThinkup(path string) {
	logr.Debugln("Importing from Thinkup export", path)
	port, err := os.Open(path)
	if err != nil {
		logr.Errln("Error opening", path, "for import:", err.Error())
		return
	}

	r := csv.NewReader(port)
	// There may be missing header columns, so turn off field count checking.
	r.FieldsPerRecord = -1

	head, err := r.Read()
	if err != nil {
		logr.Errln("Error reading from import file", path, ":", err.Error())
		return
	}

	count := 0
	for {
		record, err := r.Read()
		if err != nil {
			break
		}

		data := make(map[string]string)
		for i, field := range head {
			data[field] = record[i]
		}

		// TODO: import replies, once there's something reasonable to import them as.
		if data["in_reply_to_post_id"] != "" {
			logr.Debugln("Skipping post (twitter,", data["post_id"], ") as it is a reply")
			continue
		}
		// TODO: import repeats, once there's something reasonable to import them as.
		if data["in_retweet_of_post_id"] != "" {
			logr.Debugln("Skipping post (twitter,", data["post_id"], ") as it is a repeat")
			continue
		}

		// okay now what
		im, err := ImportBySourceIdentifier("twitter", data["post_id"])
		if err == sql.ErrNoRows {
			im = NewImport()
			im.Source = "twitter"
			im.Identifier = data["post_id"]
		} else if err != nil {
			logr.Errln("Error searching for existing imported post (twitter,", data["post_id"], "):", err.Error())
			return
		}

		var post *Post
		if im.PostId != 0 {
			post, err = PostById(im.PostId)
			if err != nil {
				logr.Errln("Error loading already-imported post", im.PostId, "for twitter post", im.Identifier, ":", err.Error())
				return
			}
		} else {
			post = NewPost()
		}

		post.Posted, err = time.Parse("2006-01-02 15:04:05", data["pub_date"])
		if err != nil {
			logr.Errln("Error parsing publish time", data["pub_date"], "for twitter post", data["post_id"], ":", err.Error())
			return
		}

		// TODO: make the links link.
		html := template.HTMLEscapeString(data["post_text"])
		html = strings.Replace(html, "\n", "<br>\n", -1)
		post.Html = html

		// TODO: store the source?
		// TODO: store the geoplace

		err = post.Save()
		if err != nil {
			logr.Errln("Error saving imported post:", err.Error())
			return
		}

		im.PostId = post.Id
		err = im.Save()
		if err != nil {
			logr.Errln("Error saving import notation for post", im.PostId, ":", err.Error())
			return
		}

		logr.Debugln("Imported post (twitter,", im.Identifier, ")")
		count++
	}
	if err != nil {
		logr.Errln("Error reading import records:", err.Error())
		return
	}

	logr.Debugln("Finished importing", count, "posts!")
}
<|endoftext|>


<commit_before>package buffer

import (
	"bufio"
	"bytes"
	"errors"
	"io"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"runtime"
	"unicode"

	"github.com/zyedidia/micro/v2/internal/config"
	"github.com/zyedidia/micro/v2/internal/screen"
	"github.com/zyedidia/micro/v2/internal/util"
	"golang.org/x/text/encoding"
	"golang.org/x/text/encoding/htmlindex"
	"golang.org/x/text/transform"
)

// LargeFileThreshold is the number of bytes when fastdirty is forced
// because hashing is too slow
const LargeFileThreshold = 50000

// overwriteFile opens the given file for writing, truncating if one exists, and then calls
// the supplied function with the file as io.Writer object, also making sure the file is
// closed afterwards.
func overwriteFile(name string, enc encoding.Encoding, fn func(io.Writer) error, withSudo bool) (err error) {
	var writeCloser io.WriteCloser

	if withSudo {
		cmd := exec.Command(config.GlobalSettings["sucmd"].(string), "dd", "bs=4k", "of="+name)

		if writeCloser, err = cmd.StdinPipe(); err != nil {
			return
		}

		c := make(chan os.Signal, 1)
		signal.Notify(c, os.Interrupt)
		go func() {
			<-c
			cmd.Process.Kill()
		}()

		defer func() {
			screenb := screen.TempFini()
			if e := cmd.Run(); e != nil && err == nil {
				err = e
			}
			screen.TempStart(screenb)
		}()
	} else if writeCloser, err = os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644); err != nil {
		return
	}

	w := bufio.NewWriter(transform.NewWriter(writeCloser, enc.NewEncoder()))
	err = fn(w)
	w.Flush()

	if e := writeCloser.Close(); e != nil && err == nil {
		err = e
	}

	return
}

// Save saves the buffer to its default path
func (b *Buffer) Save() error {
	return b.SaveAs(b.Path)
}

// SaveAs saves the buffer to a specified path (filename), creating the file if it does not exist
func (b *Buffer) SaveAs(filename string) error {
	return b.saveToFile(filename, false)
}

func (b *Buffer) SaveWithSudo() error {
	return b.SaveAsWithSudo(b.Path)
}

func (b *Buffer) SaveAsWithSudo(filename string) error {
	return b.saveToFile(filename, true)
}

func (b *Buffer) saveToFile(filename string, withSudo bool) error {
	var err error
	if b.Type.Readonly {
		return errors.New("Cannot save readonly buffer")
	}
	if b.Type.Scratch {
		return errors.New("Cannot save scratch buffer")
	}
	if withSudo && runtime.GOOS == "windows" {
		return errors.New("Save with sudo not supported on Windows")
	}

	if b.Settings["rmtrailingws"].(bool) {
		for i, l := range b.lines {
			leftover := util.CharacterCount(bytes.TrimRightFunc(l.data, unicode.IsSpace))

			linelen := util.CharacterCount(l.data)
			b.Remove(Loc{leftover, i}, Loc{linelen, i})
		}

		b.RelocateCursors()
	}

	if b.Settings["eofnewline"].(bool) {
		end := b.End()
		if b.RuneAt(Loc{end.X - 1, end.Y}) != '\n' {
			b.insert(end, []byte{'\n'})
		}
	}

	// Update the last time this file was updated after saving
	defer func() {
		b.ModTime, _ = util.GetModTime(filename)
		err = b.Serialize()
	}()

	// Removes any tilde and replaces with the absolute path to home
	absFilename, _ := util.ReplaceHome(filename)

	// Get the leading path to the file | "." is returned if there's no leading path provided
	if dirname := filepath.Dir(absFilename); dirname != "." {
		// Check if the parent dirs don't exist
		if _, statErr := os.Stat(dirname); os.IsNotExist(statErr) {
			// Prompt to make sure they want to create the dirs that are missing
			if b.Settings["mkparents"].(bool) {
				// Create all leading dir(s) since they don't exist
				if mkdirallErr := os.MkdirAll(dirname, os.ModePerm); mkdirallErr != nil {
					// If there was an error creating the dirs
					return mkdirallErr
				}
			} else {
				return errors.New("Parent dirs don't exist, enable 'mkparents' for auto creation")
			}
		}
	}

	var fileSize int

	enc, err := htmlindex.Get(b.Settings["encoding"].(string))
	if err != nil {
		return err
	}

	fwriter := func(file io.Writer) (e error) {
		if len(b.lines) == 0 {
			return
		}

		// end of line
		var eol []byte
		if b.Endings == FFDos {
			eol = []byte{'\r', '\n'}
		} else {
			eol = []byte{'\n'}
		}

		// write lines
		if fileSize, e = file.Write(b.lines[0].data); e != nil {
			return
		}

		for _, l := range b.lines[1:] {
			if _, e = file.Write(eol); e != nil {
				return
			}
			if _, e = file.Write(l.data); e != nil {
				return
			}
			fileSize += len(eol) + len(l.data)
		}
		return
	}

	if err = overwriteFile(absFilename, enc, fwriter, withSudo); err != nil {
		return err
	}

	if !b.Settings["fastdirty"].(bool) {
		if fileSize > LargeFileThreshold {
			// For large files 'fastdirty' needs to be on
			b.Settings["fastdirty"] = true
		} else {
			calcHash(b, &b.origHash)
		}
	}

	b.Path = filename
	absPath, _ := filepath.Abs(filename)
	b.AbsPath = absPath
	b.isModified = false
	b.UpdateRules()
	return err
}
<commit_msg>Fix save with sudo for large file sizes<commit_after>package buffer

import (
	"bufio"
	"bytes"
	"errors"
	"io"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"runtime"
	"unicode"

	"github.com/zyedidia/micro/v2/internal/config"
	"github.com/zyedidia/micro/v2/internal/screen"
	"github.com/zyedidia/micro/v2/internal/util"
	"golang.org/x/text/encoding"
	"golang.org/x/text/encoding/htmlindex"
	"golang.org/x/text/transform"
)

// LargeFileThreshold is the number of bytes when fastdirty is forced
// because hashing is too slow
const LargeFileThreshold = 50000

// overwriteFile opens the given file for writing, truncating if one exists, and then calls
// the supplied function with the file as io.Writer object, also making sure the file is
// closed afterwards.
func overwriteFile(name string, enc encoding.Encoding, fn func(io.Writer) error, withSudo bool) (err error) {
	var writeCloser io.WriteCloser
	var screenb bool
	var cmd *exec.Cmd

	if withSudo {
		cmd = exec.Command(config.GlobalSettings["sucmd"].(string), "dd", "bs=4k", "of="+name)

		if writeCloser, err = cmd.StdinPipe(); err != nil {
			return
		}

		c := make(chan os.Signal, 1)
		signal.Notify(c, os.Interrupt)
		go func() {
			<-c
			cmd.Process.Kill()
		}()

		screenb = screen.TempFini()
		// need to start the process now, otherwise when we flush the file
		// contents to its stdin it might hang because the kernel's pipe size
		// is too small to handle the full file contents all at once
		if e := cmd.Start(); e != nil && err == nil {
			return err
		}
	} else if writeCloser, err = os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644); err != nil {
		return
	}

	w := bufio.NewWriter(transform.NewWriter(writeCloser, enc.NewEncoder()))
	err = fn(w)
	w.Flush()

	if e := writeCloser.Close(); e != nil && err == nil {
		err = e
	}

	if withSudo {
		// wait for dd to finish and restart the screen if we used sudo
		err := cmd.Wait()
		if err != nil {
			return err
		}
		screen.TempStart(screenb)
	}

	return
}

// Save saves the buffer to its default path
func (b *Buffer) Save() error {
	return b.SaveAs(b.Path)
}

// SaveAs saves the buffer to a specified path (filename), creating the file if it does not exist
func (b *Buffer) SaveAs(filename string) error {
	return b.saveToFile(filename, false)
}

func (b *Buffer) SaveWithSudo() error {
	return b.SaveAsWithSudo(b.Path)
}

func (b *Buffer) SaveAsWithSudo(filename string) error {
	return b.saveToFile(filename, true)
}

func (b *Buffer) saveToFile(filename string, withSudo bool) error {
	var err error
	if b.Type.Readonly {
		return errors.New("Cannot save readonly buffer")
	}
	if b.Type.Scratch {
		return errors.New("Cannot save scratch buffer")
	}
	if withSudo && runtime.GOOS == "windows" {
		return errors.New("Save with sudo not supported on Windows")
	}

	if b.Settings["rmtrailingws"].(bool) {
		for i, l := range b.lines {
			leftover := util.CharacterCount(bytes.TrimRightFunc(l.data, unicode.IsSpace))

			linelen := util.CharacterCount(l.data)
			b.Remove(Loc{leftover, i}, Loc{linelen, i})
		}

		b.RelocateCursors()
	}

	if b.Settings["eofnewline"].(bool) {
		end := b.End()
		if b.RuneAt(Loc{end.X - 1, end.Y}) != '\n' {
			b.insert(end, []byte{'\n'})
		}
	}

	// Update the last time this file was updated after saving
	defer func() {
		b.ModTime, _ = util.GetModTime(filename)
		err = b.Serialize()
	}()

	// Removes any tilde and replaces with the absolute path to home
	absFilename, _ := util.ReplaceHome(filename)

	// Get the leading path to the file | "." is returned if there's no leading path provided
	if dirname := filepath.Dir(absFilename); dirname != "." {
		// Check if the parent dirs don't exist
		if _, statErr := os.Stat(dirname); os.IsNotExist(statErr) {
			// Prompt to make sure they want to create the dirs that are missing
			if b.Settings["mkparents"].(bool) {
				// Create all leading dir(s) since they don't exist
				if mkdirallErr := os.MkdirAll(dirname, os.ModePerm); mkdirallErr != nil {
					// If there was an error creating the dirs
					return mkdirallErr
				}
			} else {
				return errors.New("Parent dirs don't exist, enable 'mkparents' for auto creation")
			}
		}
	}

	var fileSize int

	enc, err := htmlindex.Get(b.Settings["encoding"].(string))
	if err != nil {
		return err
	}

	fwriter := func(file io.Writer) (e error) {
		if len(b.lines) == 0 {
			return
		}

		// end of line
		var eol []byte
		if b.Endings == FFDos {
			eol = []byte{'\r', '\n'}
		} else {
			eol = []byte{'\n'}
		}

		// write lines
		if fileSize, e = file.Write(b.lines[0].data); e != nil {
			return
		}

		for _, l := range b.lines[1:] {
			if _, e = file.Write(eol); e != nil {
				return
			}
			if _, e = file.Write(l.data); e != nil {
				return
			}
			fileSize += len(eol) + len(l.data)
		}
		return
	}

	if err = overwriteFile(absFilename, enc, fwriter, withSudo); err != nil {
		return err
	}

	if !b.Settings["fastdirty"].(bool) {
		if fileSize > LargeFileThreshold {
			// For large files 'fastdirty' needs to be on
			b.Settings["fastdirty"] = true
		} else {
			calcHash(b, &b.origHash)
		}
	}

	b.Path = filename
	absPath, _ := filepath.Abs(filename)
	b.AbsPath = absPath
	b.isModified = false
	b.UpdateRules()
	return err
}
<|endoftext|>