<commit_before>package integration_test

import (
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"

	"github.com/concourse/fly/ui"
	"github.com/fatih/color"
	"github.com/onsi/gomega/gexec"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Fly CLI", func() {
	var (
		flyCmd *exec.Cmd
		flyrc  string
	)

	BeforeEach(func() {
		flyrc = filepath.Join(userHomeDir(), ".flyrc")

		flyFixtureFile, err := os.OpenFile("./fixtures/flyrc.yml", os.O_RDONLY, 0600)
		Expect(err).NotTo(HaveOccurred())

		flyFixtureData, err := ioutil.ReadAll(flyFixtureFile)
		Expect(err).NotTo(HaveOccurred())

		err = ioutil.WriteFile(flyrc, flyFixtureData, 0600)
		Expect(err).NotTo(HaveOccurred())

		flyCmd = exec.Command(flyPath, "targets")
	})

	AfterEach(func() {
		os.RemoveAll(flyrc)
	})

	Describe("targets", func() {
		Context("when there are targets in the .flyrc", func() {
			It("displays all the targets with their token expiration", func() {
				sess, err := gexec.Start(flyCmd, GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				Eventually(sess).Should(gexec.Exit(0))

				Expect(sess.Out).To(PrintTable(ui.Table{
					Headers: ui.TableRow{
						{Contents: "name", Color: color.New(color.Bold)},
						{Contents: "url", Color: color.New(color.Bold)},
						{Contents: "expiry", Color: color.New(color.Bold)},
					},
					Data: []ui.TableRow{
						{{Contents: "another-test"}, {Contents: "https://example.com/another-test"}, {Contents: "Sat, 19 Mar 2016 01:54:30 UTC"}},
						{{Contents: "no-token"}, {Contents: "https://example.com/no-token"}, {Contents: "n/a"}},
						{{Contents: "omt"}, {Contents: "https://example.com/omt"}, {Contents: "Mon, 21 Mar 2016 01:54:30 UTC"}},
						{{Contents: "test"}, {Contents: "https://example.com/test"}, {Contents: "Fri, 25 Mar 2016 23:29:57 UTC"}},
					},
				}))
			})
		})

		Context("when no targets are available", func() {
			BeforeEach(func() {
				os.RemoveAll(flyrc)
			})

			It("prints an empty table", func() {
				sess, err := gexec.Start(flyCmd, GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				Eventually(sess).Should(gexec.Exit(0))
				Expect(sess.Out).To(PrintTable(ui.Table{
					Headers: ui.TableRow{
						{Contents: "name", Color: color.New(color.Bold)},
						{Contents: "url", Color: color.New(color.Bold)},
						{Contents: "expiry", Color: color.New(color.Bold)},
					},
					Data: []ui.TableRow{}}))
			})
		})
	})
})
<commit_msg>make flyrc location portable to windows<commit_after>package integration_test

import (
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"

	"github.com/concourse/fly/ui"
	"github.com/fatih/color"
	"github.com/onsi/gomega/gexec"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Fly CLI", func() {
	var (
		flyCmd *exec.Cmd
		flyrc  string
		tmpDir string
	)

	BeforeEach(func() {
		var err error
		tmpDir, err = ioutil.TempDir("", "fly-test")
		Expect(err).NotTo(HaveOccurred())

		if runtime.GOOS == "windows" {
			os.Setenv("USERPROFILE", tmpDir)
		} else {
			os.Setenv("HOME", tmpDir)
		}

		flyrc = filepath.Join(userHomeDir(), ".flyrc")

		flyFixtureFile, err := os.OpenFile("./fixtures/flyrc.yml", os.O_RDONLY, 0600)
		Expect(err).NotTo(HaveOccurred())

		flyFixtureData, err := ioutil.ReadAll(flyFixtureFile)
		Expect(err).NotTo(HaveOccurred())

		err = ioutil.WriteFile(flyrc, flyFixtureData, 0600)
		Expect(err).NotTo(HaveOccurred())

		flyCmd = exec.Command(flyPath, "targets")
	})

	AfterEach(func() {
		os.RemoveAll(tmpDir)
	})

	Describe("targets", func() {
		Context("when there are targets in the .flyrc", func() {
			It("displays all the targets with their token expiration", func() {
				sess, err := gexec.Start(flyCmd, GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				Eventually(sess).Should(gexec.Exit(0))

				Expect(sess.Out).To(PrintTable(ui.Table{
					Headers: ui.TableRow{
						{Contents: "name", Color: color.New(color.Bold)},
						{Contents: "url", Color: color.New(color.Bold)},
						{Contents: "expiry", Color: color.New(color.Bold)},
					},
					Data: []ui.TableRow{
						{{Contents: "another-test"}, {Contents: "https://example.com/another-test"}, {Contents: "Sat, 19 Mar 2016 01:54:30 UTC"}},
						{{Contents: "no-token"}, {Contents: "https://example.com/no-token"}, {Contents: "n/a"}},
						{{Contents: "omt"}, {Contents: "https://example.com/omt"}, {Contents: "Mon, 21 Mar 2016 01:54:30 UTC"}},
						{{Contents: "test"}, {Contents: "https://example.com/test"}, {Contents: "Fri, 25 Mar 2016 23:29:57 UTC"}},
					},
				}))
			})
		})

		Context("when no targets are available", func() {
			BeforeEach(func() {
				os.RemoveAll(flyrc)
			})

			It("prints an empty table", func() {
				sess, err := gexec.Start(flyCmd, GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				Eventually(sess).Should(gexec.Exit(0))
				Expect(sess.Out).To(PrintTable(ui.Table{
					Headers: ui.TableRow{
						{Contents: "name", Color: color.New(color.Bold)},
						{Contents: "url", Color: color.New(color.Bold)},
						{Contents: "expiry", Color: color.New(color.Bold)},
					},
					Data: []ui.TableRow{}}))
			})
		})
	})
})
<|endoftext|>


<commit_before>package version

// Version of Functions
var Version = "0.3.588"
<commit_msg>fnserver: 0.3.589 release [skip ci]<commit_after>package version

// Version of Functions
var Version = "0.3.589"
<|endoftext|>


<commit_before>// errchk $G $D/$F.go

// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Static error messages about interface conversions.

package main

type T struct { a int }
var t *T

type I interface { M() }
var i I

type I2 interface { M(); N(); }
var i2 I2

type E interface { }
var e E

func main() {
	e = t;	// ok
	t = e;	// ERROR "need explicit|need type assertion"

	// neither of these can work,
	// because i has an extra method
	// that t does not, so i cannot contain a t.
	i = t;	// ERROR "missing|incompatible|is not"
	t = i;	// ERROR "missing|incompatible|is not"

	i = i2;	// ok
	i2 = i;	// ERROR "need explicit|need type assertion"

	i = I(i2);	// ok
	i2 = I2(i);	// ERROR "need explicit|need type assertion"

	e = E(t);	// ok
	t = T(e);	// ERROR "need explicit|need type assertion"
}
<commit_msg>Recognize gccgo error message.<commit_after>// errchk $G $D/$F.go

// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Static error messages about interface conversions.

package main

type T struct { a int }
var t *T

type I interface { M() }
var i I

type I2 interface { M(); N(); }
var i2 I2

type E interface { }
var e E

func main() {
	e = t;	// ok
	t = e;	// ERROR "need explicit|need type assertion"

	// neither of these can work,
	// because i has an extra method
	// that t does not, so i cannot contain a t.
	i = t;	// ERROR "missing|incompatible|is not"
	t = i;	// ERROR "missing|incompatible|is not"

	i = i2;	// ok
	i2 = i;	// ERROR "need explicit|need type assertion"

	i = I(i2);	// ok
	i2 = I2(i);	// ERROR "need explicit|need type assertion"

	e = E(t);	// ok
	t = T(e);	// ERROR "need explicit|need type assertion|incompatible"
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package assets

import (
	"bytes"
	"fmt"
	"net/url"
	"os"
	"path"
	"strings"

	"github.com/blang/semver"
	"github.com/golang/glog"

	"k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/apis/kops/util"
	"k8s.io/kops/pkg/featureflag"
	"k8s.io/kops/pkg/kubemanifest"
	"k8s.io/kops/pkg/values"
	"k8s.io/kops/util/pkg/hashing"
	"k8s.io/kops/util/pkg/vfs"
)

// RewriteManifests controls whether we rewrite manifests
// Because manifest rewriting converts everything to and from YAML, we normalize everything by doing so
var RewriteManifests = featureflag.New("RewriteManifests", featureflag.Bool(true))

// AssetBuilder discovers and remaps assets.
type AssetBuilder struct {
	ContainerAssets []*ContainerAsset
	FileAssets      []*FileAsset
	AssetsLocation  *kops.Assets
	// TODO we'd like to use cloudup.Phase here, but that introduces a go cyclic dependency
	Phase string

	// KubernetesVersion is the version of kubernetes we are installing
	KubernetesVersion semver.Version
}

// ContainerAsset models a container's location.
type ContainerAsset struct {
	// DockerImage will be the name of the container we should run.
	// This is used to copy a container to a ContainerRegistry.
	DockerImage string
	// CanonicalLocation will be the source location of the container.
	CanonicalLocation string
}

// FileAsset models a file's location.
type FileAsset struct {
	// FileURL is the URL of a file that is accessed by a Kubernetes cluster.
	FileURL *url.URL
	// CanonicalFileURL is the source URL of a file. This is used to copy a file to a FileRepository.
	CanonicalFileURL *url.URL
	// SHAValue is the SHA hash of the FileAsset.
	SHAValue string
}

// NewAssetBuilder creates a new AssetBuilder.
func NewAssetBuilder(cluster *kops.Cluster, phase string) *AssetBuilder {
	a := &AssetBuilder{
		AssetsLocation: cluster.Spec.Assets,
		Phase:          phase,
	}

	version, err := util.ParseKubernetesVersion(cluster.Spec.KubernetesVersion)
	if err != nil {
		// This should have already been validated
		glog.Fatalf("unexpected error from ParseKubernetesVersion %s: %v", cluster.Spec.KubernetesVersion, err)
	}
	a.KubernetesVersion = *version

	return a
}

// RemapManifest transforms a kubernetes manifest.
// Whenever we are building a Task that includes a manifest, we should pass it through RemapManifest first.
// This will:
// * rewrite the images if they are being redirected to a mirror, and ensure the image is uploaded
func (a *AssetBuilder) RemapManifest(data []byte) ([]byte, error) {
	if !RewriteManifests.Enabled() {
		return data, nil
	}

	manifests, err := kubemanifest.LoadManifestsFrom(data)
	if err != nil {
		return nil, err
	}

	var yamlSeparator = []byte("\n---\n\n")
	var remappedManifests [][]byte
	for _, manifest := range manifests {
		if err := manifest.RemapImages(a.RemapImage); err != nil {
			return nil, fmt.Errorf("error remapping images: %v", err)
		}

		y, err := manifest.ToYAML()
		if err != nil {
			return nil, fmt.Errorf("error re-marshalling manifest: %v", err)
		}

		remappedManifests = append(remappedManifests, y)
	}

	return bytes.Join(remappedManifests, yamlSeparator), nil
}

// RemapImage normalizes a containers location if a user sets the AssetsLocation ContainerRegistry location.
func (a *AssetBuilder) RemapImage(image string) (string, error) {
	asset := &ContainerAsset{}

	asset.DockerImage = image

	// The k8s.gcr.io prefix is an alias, but for CI builds we run from a docker load,
	// and we only double-tag from 1.10 onwards.
	// For versions prior to 1.10, remap k8s.gcr.io to the old name.
	// This also means that we won't start using the aliased names on existing clusters,
	// which could otherwise be surprising to users.
	if !util.IsKubernetesGTE("1.10", a.KubernetesVersion) && strings.HasPrefix(image, "k8s.gcr.io/") {
		image = "gcr.io/google_containers/" + strings.TrimPrefix(image, "k8s.gcr.io/")
	}

	if strings.HasPrefix(image, "kope/dns-controller:") {
		// To use user-defined DNS Controller:
		// 1. DOCKER_REGISTRY=[your docker hub repo] make dns-controller-push
		// 2. export DNSCONTROLLER_IMAGE=[your docker hub repo]
		// 3. make kops and create/apply cluster
		override := os.Getenv("DNSCONTROLLER_IMAGE")
		if override != "" {
			image = override
		}
	}

	if a.AssetsLocation != nil && a.AssetsLocation.ContainerRegistry != nil {
		registryMirror := *a.AssetsLocation.ContainerRegistry
		normalized := image

		// Remove the 'standard' kubernetes image prefix, just for sanity
		normalized = strings.TrimPrefix(normalized, "k8s.gcr.io/")

		// We can't nest arbitrarily
		// Some risk of collisions, but also -- and __ in the names appear to be blocked by docker hub
		normalized = strings.Replace(normalized, "/", "-", -1)
		asset.DockerImage = registryMirror + "/" + normalized

		asset.CanonicalLocation = image

		// Run the new image
		image = asset.DockerImage
	}

	a.ContainerAssets = append(a.ContainerAssets, asset)
	return image, nil
}

// RemapFileAndSHA returns a remapped url for the file, if AssetsLocation is defined.
// It also returns the SHA hash of the file.
func (a *AssetBuilder) RemapFileAndSHA(fileURL *url.URL) (*url.URL, *hashing.Hash, error) {
	if fileURL == nil {
		return nil, nil, fmt.Errorf("unable to remap an nil URL")
	}

	fileAsset := &FileAsset{
		FileURL: fileURL,
	}

	if a.AssetsLocation != nil && a.AssetsLocation.FileRepository != nil {
		fileAsset.CanonicalFileURL = fileURL

		normalizedFileURL, err := a.normalizeURL(fileURL)
		if err != nil {
			return nil, nil, err
		}

		fileAsset.FileURL = normalizedFileURL

		glog.V(4).Infof("adding remapped file: %+v", fileAsset)
	}

	h, err := a.findHash(fileAsset)
	if err != nil {
		return nil, nil, err
	}
	fileAsset.SHAValue = h.Hex()

	a.FileAssets = append(a.FileAssets, fileAsset)
	glog.V(8).Infof("adding file: %+v", fileAsset)

	return fileAsset.FileURL, h, nil
}

// TODO - remove this method as CNI does now have a SHA file

// RemapFileAndSHAValue is used exclusively to remap the cni tarball, as the tarball does not have a sha file in object storage.
func (a *AssetBuilder) RemapFileAndSHAValue(fileURL *url.URL, shaValue string) (*url.URL, error) {
	if fileURL == nil {
		return nil, fmt.Errorf("unable to remap a nil URL")
	}

	fileAsset := &FileAsset{
		FileURL:  fileURL,
		SHAValue: shaValue,
	}

	if a.AssetsLocation != nil && a.AssetsLocation.FileRepository != nil {
		fileAsset.CanonicalFileURL = fileURL

		normalizedFile, err := a.normalizeURL(fileURL)
		if err != nil {
			return nil, err
		}

		fileAsset.FileURL = normalizedFile
		glog.V(4).Infof("adding remapped file: %q", fileAsset.FileURL.String())
	}

	a.FileAssets = append(a.FileAssets, fileAsset)

	return fileAsset.FileURL, nil
}

// FindHash returns the hash value of a FileAsset.
func (a *AssetBuilder) findHash(file *FileAsset) (*hashing.Hash, error) {

	// If the phase is "assets" we use the CanonicalFileURL,
	// but during other phases we use the hash from the FileRepository or the base kops path.
	// We do not want to just test for CanonicalFileURL as it is defined in
	// other phases, but is not used to test for the SHA.
	// This prevents a chicken and egg problem where the file is not yet in the FileRepository.
	//
	// assets phase -> get the sha file from the source / CanonicalFileURL
	// any other phase -> get the sha file from the kops base location or the FileRepository
	//
	// TLDR; we use the file.CanonicalFileURL during assets phase, and use file.FileUrl the
	// rest of the time. If not we get a chicken and the egg problem where we are reading the sha file
	// before it exists.
	u := file.FileURL
	if a.Phase == "assets" && file.CanonicalFileURL != nil {
		u = file.CanonicalFileURL
	}

	if u == nil {
		return nil, fmt.Errorf("file url is not defined")
	}

	for _, ext := range []string{".sha1"} {
		hashURL := u.String() + ext
		b, err := vfs.Context.ReadFile(hashURL)
		if err != nil {
			glog.Infof("error reading hash file %q: %v", hashURL, err)
			continue
		}
		hashString := strings.TrimSpace(string(b))
		glog.V(2).Infof("Found hash %q for %q", hashString, u)

		return hashing.FromString(hashString)
	}

	if a.AssetsLocation != nil && a.AssetsLocation.FileRepository != nil {
		return nil, fmt.Errorf("you may have not staged your files correctly, please execute kops update cluster using the assets phase")
	}
	return nil, fmt.Errorf("cannot determine hash for %q (have you specified a valid file location?)", u)
}

func (a *AssetBuilder) normalizeURL(file *url.URL) (*url.URL, error) {

	if a.AssetsLocation == nil || a.AssetsLocation.FileRepository == nil {
		return nil, fmt.Errorf("assetLocation and fileRepository cannot be nil to normalize an file asset URL")
	}

	f := values.StringValue(a.AssetsLocation.FileRepository)

	if f == "" {
		return nil, fmt.Errorf("assetsLocation fileRepository cannot be an empty string")
	}

	fileRepo, err := url.Parse(f)
	if err != nil {
		return nil, fmt.Errorf("unable to parse file repository URL %q: %v", values.StringValue(a.AssetsLocation.FileRepository), err)
	}

	fileRepo.Path = path.Join(fileRepo.Path, file.Path)

	return fileRepo, nil
}
<commit_msg>Typo fix in error message<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package assets

import (
	"bytes"
	"fmt"
	"net/url"
	"os"
	"path"
	"strings"

	"github.com/blang/semver"
	"github.com/golang/glog"

	"k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/apis/kops/util"
	"k8s.io/kops/pkg/featureflag"
	"k8s.io/kops/pkg/kubemanifest"
	"k8s.io/kops/pkg/values"
	"k8s.io/kops/util/pkg/hashing"
	"k8s.io/kops/util/pkg/vfs"
)

// RewriteManifests controls whether we rewrite manifests
// Because manifest rewriting converts everything to and from YAML, we normalize everything by doing so
var RewriteManifests = featureflag.New("RewriteManifests", featureflag.Bool(true))

// AssetBuilder discovers and remaps assets.
type AssetBuilder struct {
	ContainerAssets []*ContainerAsset
	FileAssets      []*FileAsset
	AssetsLocation  *kops.Assets
	// TODO we'd like to use cloudup.Phase here, but that introduces a go cyclic dependency
	Phase string

	// KubernetesVersion is the version of kubernetes we are installing
	KubernetesVersion semver.Version
}

// ContainerAsset models a container's location.
type ContainerAsset struct {
	// DockerImage will be the name of the container we should run.
	// This is used to copy a container to a ContainerRegistry.
	DockerImage string
	// CanonicalLocation will be the source location of the container.
	CanonicalLocation string
}

// FileAsset models a file's location.
type FileAsset struct {
	// FileURL is the URL of a file that is accessed by a Kubernetes cluster.
	FileURL *url.URL
	// CanonicalFileURL is the source URL of a file. This is used to copy a file to a FileRepository.
	CanonicalFileURL *url.URL
	// SHAValue is the SHA hash of the FileAsset.
	SHAValue string
}

// NewAssetBuilder creates a new AssetBuilder.
func NewAssetBuilder(cluster *kops.Cluster, phase string) *AssetBuilder {
	a := &AssetBuilder{
		AssetsLocation: cluster.Spec.Assets,
		Phase:          phase,
	}

	version, err := util.ParseKubernetesVersion(cluster.Spec.KubernetesVersion)
	if err != nil {
		// This should have already been validated
		glog.Fatalf("unexpected error from ParseKubernetesVersion %s: %v", cluster.Spec.KubernetesVersion, err)
	}
	a.KubernetesVersion = *version

	return a
}

// RemapManifest transforms a kubernetes manifest.
// Whenever we are building a Task that includes a manifest, we should pass it through RemapManifest first.
// This will:
// * rewrite the images if they are being redirected to a mirror, and ensure the image is uploaded
func (a *AssetBuilder) RemapManifest(data []byte) ([]byte, error) {
	if !RewriteManifests.Enabled() {
		return data, nil
	}

	manifests, err := kubemanifest.LoadManifestsFrom(data)
	if err != nil {
		return nil, err
	}

	var yamlSeparator = []byte("\n---\n\n")
	var remappedManifests [][]byte
	for _, manifest := range manifests {
		if err := manifest.RemapImages(a.RemapImage); err != nil {
			return nil, fmt.Errorf("error remapping images: %v", err)
		}

		y, err := manifest.ToYAML()
		if err != nil {
			return nil, fmt.Errorf("error re-marshalling manifest: %v", err)
		}

		remappedManifests = append(remappedManifests, y)
	}

	return bytes.Join(remappedManifests, yamlSeparator), nil
}

// RemapImage normalizes a containers location if a user sets the AssetsLocation ContainerRegistry location.
func (a *AssetBuilder) RemapImage(image string) (string, error) {
	asset := &ContainerAsset{}

	asset.DockerImage = image

	// The k8s.gcr.io prefix is an alias, but for CI builds we run from a docker load,
	// and we only double-tag from 1.10 onwards.
	// For versions prior to 1.10, remap k8s.gcr.io to the old name.
	// This also means that we won't start using the aliased names on existing clusters,
	// which could otherwise be surprising to users.
	if !util.IsKubernetesGTE("1.10", a.KubernetesVersion) && strings.HasPrefix(image, "k8s.gcr.io/") {
		image = "gcr.io/google_containers/" + strings.TrimPrefix(image, "k8s.gcr.io/")
	}

	if strings.HasPrefix(image, "kope/dns-controller:") {
		// To use user-defined DNS Controller:
		// 1. DOCKER_REGISTRY=[your docker hub repo] make dns-controller-push
		// 2. export DNSCONTROLLER_IMAGE=[your docker hub repo]
		// 3. make kops and create/apply cluster
		override := os.Getenv("DNSCONTROLLER_IMAGE")
		if override != "" {
			image = override
		}
	}

	if a.AssetsLocation != nil && a.AssetsLocation.ContainerRegistry != nil {
		registryMirror := *a.AssetsLocation.ContainerRegistry
		normalized := image

		// Remove the 'standard' kubernetes image prefix, just for sanity
		normalized = strings.TrimPrefix(normalized, "k8s.gcr.io/")

		// We can't nest arbitrarily
		// Some risk of collisions, but also -- and __ in the names appear to be blocked by docker hub
		normalized = strings.Replace(normalized, "/", "-", -1)
		asset.DockerImage = registryMirror + "/" + normalized

		asset.CanonicalLocation = image

		// Run the new image
		image = asset.DockerImage
	}

	a.ContainerAssets = append(a.ContainerAssets, asset)
	return image, nil
}

// RemapFileAndSHA returns a remapped url for the file, if AssetsLocation is defined.
// It also returns the SHA hash of the file.
func (a *AssetBuilder) RemapFileAndSHA(fileURL *url.URL) (*url.URL, *hashing.Hash, error) {
	if fileURL == nil {
		return nil, nil, fmt.Errorf("unable to remap a nil URL")
	}

	fileAsset := &FileAsset{
		FileURL: fileURL,
	}

	if a.AssetsLocation != nil && a.AssetsLocation.FileRepository != nil {
		fileAsset.CanonicalFileURL = fileURL

		normalizedFileURL, err := a.normalizeURL(fileURL)
		if err != nil {
			return nil, nil, err
		}

		fileAsset.FileURL = normalizedFileURL

		glog.V(4).Infof("adding remapped file: %+v", fileAsset)
	}

	h, err := a.findHash(fileAsset)
	if err != nil {
		return nil, nil, err
	}
	fileAsset.SHAValue = h.Hex()

	a.FileAssets = append(a.FileAssets, fileAsset)
	glog.V(8).Infof("adding file: %+v", fileAsset)

	return fileAsset.FileURL, h, nil
}

// TODO - remove this method as CNI does now have a SHA file

// RemapFileAndSHAValue is used exclusively to remap the cni tarball, as the tarball does not have a sha file in object storage.
func (a *AssetBuilder) RemapFileAndSHAValue(fileURL *url.URL, shaValue string) (*url.URL, error) {
	if fileURL == nil {
		return nil, fmt.Errorf("unable to remap a nil URL")
	}

	fileAsset := &FileAsset{
		FileURL:  fileURL,
		SHAValue: shaValue,
	}

	if a.AssetsLocation != nil && a.AssetsLocation.FileRepository != nil {
		fileAsset.CanonicalFileURL = fileURL

		normalizedFile, err := a.normalizeURL(fileURL)
		if err != nil {
			return nil, err
		}

		fileAsset.FileURL = normalizedFile
		glog.V(4).Infof("adding remapped file: %q", fileAsset.FileURL.String())
	}

	a.FileAssets = append(a.FileAssets, fileAsset)

	return fileAsset.FileURL, nil
}

// FindHash returns the hash value of a FileAsset.
func (a *AssetBuilder) findHash(file *FileAsset) (*hashing.Hash, error) {

	// If the phase is "assets" we use the CanonicalFileURL,
	// but during other phases we use the hash from the FileRepository or the base kops path.
	// We do not want to just test for CanonicalFileURL as it is defined in
	// other phases, but is not used to test for the SHA.
	// This prevents a chicken and egg problem where the file is not yet in the FileRepository.
	//
	// assets phase -> get the sha file from the source / CanonicalFileURL
	// any other phase -> get the sha file from the kops base location or the FileRepository
	//
	// TLDR; we use the file.CanonicalFileURL during assets phase, and use file.FileUrl the
	// rest of the time. If not we get a chicken and the egg problem where we are reading the sha file
	// before it exists.
	u := file.FileURL
	if a.Phase == "assets" && file.CanonicalFileURL != nil {
		u = file.CanonicalFileURL
	}

	if u == nil {
		return nil, fmt.Errorf("file url is not defined")
	}

	for _, ext := range []string{".sha1"} {
		hashURL := u.String() + ext
		b, err := vfs.Context.ReadFile(hashURL)
		if err != nil {
			glog.Infof("error reading hash file %q: %v", hashURL, err)
			continue
		}
		hashString := strings.TrimSpace(string(b))
		glog.V(2).Infof("Found hash %q for %q", hashString, u)

		return hashing.FromString(hashString)
	}

	if a.AssetsLocation != nil && a.AssetsLocation.FileRepository != nil {
		return nil, fmt.Errorf("you may have not staged your files correctly, please execute kops update cluster using the assets phase")
	}
	return nil, fmt.Errorf("cannot determine hash for %q (have you specified a valid file location?)", u)
}

func (a *AssetBuilder) normalizeURL(file *url.URL) (*url.URL, error) {

	if a.AssetsLocation == nil || a.AssetsLocation.FileRepository == nil {
		return nil, fmt.Errorf("assetLocation and fileRepository cannot be nil to normalize an file asset URL")
	}

	f := values.StringValue(a.AssetsLocation.FileRepository)

	if f == "" {
		return nil, fmt.Errorf("assetsLocation fileRepository cannot be an empty string")
	}

	fileRepo, err := url.Parse(f)
	if err != nil {
		return nil, fmt.Errorf("unable to parse file repository URL %q: %v", values.StringValue(a.AssetsLocation.FileRepository), err)
	}

	fileRepo.Path = path.Join(fileRepo.Path, file.Path)

	return fileRepo, nil
}
<|endoftext|>


<commit_before>package runtime

import (
	"fmt"
	"os"
	"strings"

	"github.com/go-clang/gen"
)

// PrepareFunctionName prepares C function naming to Go function name.
func PrepareFunctionName(g *gen.Generation, f *gen.Function) string {
	fname := f.Name
	fname = strings.TrimPrefix(fname, "clang_")

	// Trim some whitelisted prefixes by their function name
	if fn := strings.TrimPrefix(fname, "indexLoc_"); len(fn) != len(fname) {
		fname = fn
	} else if fn := strings.TrimPrefix(fname, "index_"); len(fn) != len(fname) {
		fname = fn
	} else if fn := strings.TrimPrefix(fname, "Location_"); len(fn) != len(fname) {
		fname = fn
	} else if fn := strings.TrimPrefix(fname, "Range_"); len(fn) != len(fname) {
		fname = fn
	} else if fn := strings.TrimPrefix(fname, "remap_"); len(fn) != len(fname) {
		fname = fn
	}

	// Trim some whitelisted prefixes by their types
	if len(f.Parameters) > 0 && g.IsEnumOrStruct(f.Parameters[0].Type.GoName) {
		switch f.Parameters[0].Type.GoName {
		case "CodeCompleteResults":
			fname = strings.TrimPrefix(fname, "codeComplete")
		case "CompletionString":
			if f.CName == "clang_getNumCompletionChunks" {
				fname = "NumChunks"
			} else {
				fname = strings.TrimPrefix(fname, "getCompletion")
			}
		case "SourceRange":
			fname = strings.TrimPrefix(fname, "getRange")
		}
	}

	return fname
}

// PrepareFunctionName2 prepares C function naming to Go function name.
func PrepareFunctionName2(g *gen.Generation, f *gen.Function) string {
	fname := f.Name
	fname = strings.TrimPrefix(fname, "clang_")

	var trimFuncNames = []string{
		"indexLoc_",
		"index_",
		"Location_",
		"Range_",
		"remap_",
	}
	// trim some allowlisted prefixes by their function name
	for _, name := range trimFuncNames {
		idx := strings.LastIndex(fname, name)
		if idx != -1 {
			fname = fname[idx:]
			break
		}
	}

	// trim some allowlisted prefixes by their types
	if len(f.Parameters) > 0 && g.IsEnumOrStruct(f.Parameters[0].Type.GoName) {
		switch f.Parameters[0].Type.GoName {
		case "CodeCompleteResults":
			fname = strings.TrimPrefix(fname, "codeComplete")

		case "CompletionString":
			if f.CName == "clang_getNumCompletionChunks" {
				fname = "NumChunks"
			} else {
				fname = strings.TrimPrefix(fname, "getCompletion")
			}

		case "SourceRange":
			fname = strings.TrimPrefix(fname, "getRange")
		}
	}

	return fname
}

// PrepareFunction prepares C function to Go function.
func PrepareFunction(f *gen.Function) {
	for i := range f.Parameters {
		p := &f.Parameters[i]

		if f.CName == "clang_getRemappingsFromFileList" {
			switch p.CName {
			case "filePaths":
				p.Type.IsSlice = true
			case "numFiles":
				p.Type.LengthOfSlice = "filePaths"
			}

			continue
		}

		// Whiteflag types that are return arguments
		if p.Type.PointerLevel == 1 && (p.Type.GoName == "File" || p.Type.GoName == "FileUniqueID" || p.Type.GoName == "IdxClientFile" || p.Type.GoName == "cxstring" || p.Type.GoName == gen.GoInt32 || p.Type.GoName == gen.GoUInt32 || p.Type.GoName == "CompilationDatabase_Error" || p.Type.GoName == "PlatformAvailability" || p.Type.GoName == "SourceRange" || p.Type.GoName == "LoadDiag_Error") {
			p.Type.IsReturnArgument = true
		}
		if p.Type.PointerLevel == 2 && (p.Type.GoName == "Token" || p.Type.GoName == "Cursor") {
			p.Type.IsReturnArgument = true
		}

		if f.CName == "clang_disposeOverriddenCursors" && p.CName == "overridden" {
			p.Type.IsSlice = true
		}

		// If this is an array length parameter we need to find its partner
		paCName := gen.ArrayNameFromLength(p.CName)

		if paCName != "" {
			for j := range f.Parameters {
				pa := &f.Parameters[j]

				if strings.ToLower(pa.CName) == strings.ToLower(paCName) {
					if pa.Type.GoName == "struct CXUnsavedFile" || pa.Type.GoName == "UnsavedFile" {
						pa.Type.GoName = "UnsavedFile"
						pa.Type.CGoName = "struct_CXUnsavedFile"
					} else if pa.Type.CGoName == gen.CSChar && pa.Type.PointerLevel == 2 {
					} else if pa.Type.GoName == "CompletionResult" {
					} else if pa.Type.GoName == "Token" {
					} else if pa.Type.GoName == "Cursor" {
					} else {
						break
					}

					p.Type.LengthOfSlice = pa.Name
					pa.Type.IsSlice = true

					if pa.Type.IsReturnArgument && p.Type.PointerLevel > 0 {
						p.Type.IsReturnArgument = true
					}

					break
				}
			}
		}
	}

	for i := range f.Parameters {
		p := &f.Parameters[i]

		if p.Type.CGoName == gen.CSChar && p.Type.PointerLevel == 2 && !p.Type.IsSlice {
			p.Type.IsReturnArgument = true
		}
	}
}

// PrepareFunction2 prepares C function to Go function.
func PrepareFunction2(f *gen.Function) {
	for i := range f.Parameters {
		p := &f.Parameters[i]

		if f.CName == "clang_getRemappingsFromFileList" {
			switch p.CName {
			case "filePaths":
				p.Type.IsSlice = true

			case "numFiles":
				p.Type.LengthOfSlice = "filePaths"
			}

			continue
		}

		// allowflag types that are return arguments
		switch p.Type.PointerLevel {
		case 1:
			switch p.Type.GoName {
			case gen.GoInt32,
				gen.GoUInt32,
				"File",
				"FileUniqueID",
				"IdxClientFile",
				"cxstring",
				"CompilationDatabase_Error",
				"PlatformAvailability",
				"SourceRange",
				"LoadDiag_Error":

				p.Type.IsReturnArgument = true
			}

		case 2:
			switch p.Type.GoName {
			case "Token",
				"Cursor":

				p.Type.IsReturnArgument = true
			}
		}

		if f.CName == "clang_disposeOverriddenCursors" && p.CName == "overridden" {
			p.Type.IsSlice = true
		}

		// if this is an array length parameter we need to find its partner
		paCName := gen.ArrayNameFromLength(p.CName)

		if paCName != "" {
			for j := range f.Parameters {
				pa := &f.Parameters[j]

				if strings.EqualFold(pa.CName, paCName) {
					switch {
					case pa.Type.CGoName == gen.CSChar && pa.Type.PointerLevel == 2:
						// nothing to do

					case pa.Type.GoName == "CompletionResult":
						// nothing to do

					case pa.Type.GoName == "Token":
						// nothing to do

					case pa.Type.GoName == "Cursor":
						// nothing to do

					case pa.Type.GoName == "struct CXUnsavedFile" || pa.Type.GoName == "UnsavedFile":
						pa.Type.GoName = "UnsavedFile"
						pa.Type.CGoName = "struct_CXUnsavedFile"

					default:
						break
					}

					p.Type.LengthOfSlice = pa.Name
					pa.Type.IsSlice = true

					if pa.Type.IsReturnArgument && p.Type.PointerLevel > 0 {
						p.Type.IsReturnArgument = true
					}

					break
				}
			}
		}
	}

	for i := range f.Parameters {
		p := &f.Parameters[i]

		if p.Type.CGoName == gen.CSChar && p.Type.PointerLevel == 2 && !p.Type.IsSlice {
			p.Type.IsReturnArgument = true
		}
	}
}

// FilterFunction reports whether the f function filtered to a particular condition.
func FilterFunction(f *gen.Function) bool {
	switch f.CName {
	case "clang_CompileCommand_getMappedSourceContent", "clang_CompileCommand_getMappedSourcePath", "clang_CompileCommand_getNumMappedSources":
		// some functions are not compiled in the library see https://lists.launchpad.net/desktop-packages/msg75835.html for a never resolved bug report
		fmt.Fprintf(os.Stderr, "Ignore function %q because it is not compiled within libClang\n", f.CName)

		return false

	case "clang_executeOnThread", "clang_getInclusions":
		// some functions can not be handled automatically by us
		fmt.Fprintf(os.Stderr, "Ignore function %q because it cannot be handled automatically\n", f.CName)

		return false

	case "clang_annotateTokens", "clang_getCursorPlatformAvailability", "clang_visitChildren":
		// some functions are simply manually implemented
		fmt.Fprintf(os.Stderr, "Ignore function %q because it is manually implemented\n", f.CName)

		return false
	}

	// TODO(go-clang): if this function is from CXString.h we ignore it https://github.com/go-clang/gen/issues/25
	for i := range f.IncludeFiles {
		if strings.HasSuffix(i, "CXString.h") {
			return false
		}
	}

	return true
}

// FilterFunctionParameter reports whether the p function parameter filtered to a particular condition.
func FilterFunctionParameter(p gen.FunctionParameter) bool {
	// these pointers are ok
	if p.Type.PointerLevel == 1 {
		if p.Type.CGoName == gen.CSChar {
			return false
		}

		switch p.Type.GoName {
		case "UnsavedFile",
			"CodeCompleteResults",
			"CursorKind",
			"IdxContainerInfo",
			"IdxDeclInfo",
			"IndexerCallbacks",
			"TranslationUnit",
			"IdxEntityInfo",
			"IdxAttrInfo":

			return false
		}
	}

	return true
}

// FixedFunctionName changes the function name under certain conditions.
func FixedFunctionName(f *gen.Function) string {
	// needs to be renamed manually since clang_getTranslationUnitCursor will conflict with clang_getCursor
	if f.CName == "clang_getTranslationUnitCursor" {
		return "TranslationUnitCursor"
	}

	return ""
}

// PrepareStructMembers prepares struct member names.
func PrepareStructMembers(s *gen.Struct) {
	for _, m := range s.Members {
		if (strings.HasPrefix(m.CName, "has") || strings.HasPrefix(m.CName, "is")) && m.Type.GoName == gen.GoInt32 {
			m.Type.GoName = gen.GoBool
		}

		// if this is an array length parameter we need to find its partner
		maCName := gen.ArrayNameFromLength(m.CName)

		if maCName != "" {
			for _, ma := range s.Members {
				if strings.ToLower(ma.CName) == strings.ToLower(maCName) {
					m.Type.LengthOfSlice = ma.CName
					ma.Type.IsSlice = true
					// TODO(go-clang): wrong usage but needed for the getter generation...
					// maybe refactor this LengthOfSlice alltogether?
					// https://github.com/go-clang/gen/issues/49
					ma.Type.LengthOfSlice = m.CName

					break
				}
			}
		}
	}

	prepareStructMembersArrayStruct(s)
}

// prepareStructMembersArrayStruct checks if the struct has two member variables, one is an array and the other a plain
// int/uint with size/length/count/len is its name because then this should be an array struct, and we connect them to handle a slice.
func prepareStructMembersArrayStruct(s *gen.Struct) {
	if len(s.Members) != 2 {
		return
	}

	if !arrayLengthCombination(&s.Members[0].Type, &s.Members[1].Type) && !arrayLengthCombination(&s.Members[1].Type, &s.Members[0].Type) {
		return
	}

	// if one of the members is already marked as array/slice another heuristic has already covered both members.
	switch {
	case s.Members[0].Type.IsArray,
		s.Members[1].Type.IsArray,
		s.Members[0].Type.IsSlice,
		s.Members[1].Type.IsSlice:

		return
	}

	var a *gen.StructMember
	var c *gen.StructMember

	if s.Members[0].Type.PointerLevel == 1 {
		a = s.Members[0]
		c = s.Members[1]
	} else {
		c = s.Members[0]
		a = s.Members[1]
	}

	lengthName := strings.ToLower(c.CName)
	if lengthName != "count" &&
		lengthName != "len" && lengthName != "length" && lengthName != "size" {

		return
	}

	c.Type.LengthOfSlice = a.CName
	a.Type.IsSlice = true
	// TODO(go-clang): wrong usage but needed for the getter generation...
	// maybe refactor this LengthOfSlice alltogether?
	// https://github.com/go-clang/gen/issues/49
	a.Type.LengthOfSlice = c.CName
}

// arrayLengthCombination reports whether the x and y to correct combination.
func arrayLengthCombination(x *gen.Type, y *gen.Type) bool {
	return x.PointerLevel == 1 && y.PointerLevel == 0 &&
		!gen.IsInteger(x) && gen.IsInteger(y)
}

// FilterStructMemberGetter reports whether the m struct member filtered to a particular condition.
func FilterStructMemberGetter(m *gen.StructMember) bool {
	// we do not want getters to *int_data members
	return !strings.HasSuffix(m.CName, "int_data")
}
<commit_msg>cmd/go-clang-gen/runtime: remove unused functions<commit_after>package runtime

import (
	"fmt"
	"os"
	"strings"

	"github.com/go-clang/gen"
)

// PrepareFunctionName prepares C function naming to Go function name.
func PrepareFunctionName(g *gen.Generation, f *gen.Function) string {
	fname := f.Name
	fname = strings.TrimPrefix(fname, "clang_")

	// Trim some whitelisted prefixes by their function name
	if fn := strings.TrimPrefix(fname, "indexLoc_"); len(fn) != len(fname) {
		fname = fn
	} else if fn := strings.TrimPrefix(fname, "index_"); len(fn) != len(fname) {
		fname = fn
	} else if fn := strings.TrimPrefix(fname, "Location_"); len(fn) != len(fname) {
		fname = fn
	} else if fn := strings.TrimPrefix(fname, "Range_"); len(fn) != len(fname) {
		fname = fn
	} else if fn := strings.TrimPrefix(fname, "remap_"); len(fn) != len(fname) {
		fname = fn
	}

	// Trim some whitelisted prefixes by their types
	if len(f.Parameters) > 0 && g.IsEnumOrStruct(f.Parameters[0].Type.GoName) {
		switch f.Parameters[0].Type.GoName {
		case "CodeCompleteResults":
			fname = strings.TrimPrefix(fname, "codeComplete")
		case "CompletionString":
			if f.CName == "clang_getNumCompletionChunks" {
				fname = "NumChunks"
			} else {
				fname = strings.TrimPrefix(fname, "getCompletion")
			}
		case "SourceRange":
			fname = strings.TrimPrefix(fname, "getRange")
		}
	}

	return fname
}

// PrepareFunction prepares C function to Go function.
func PrepareFunction(f *gen.Function) {
	for i := range f.Parameters {
		p := &f.Parameters[i]

		if f.CName == "clang_getRemappingsFromFileList" {
			switch p.CName {
			case "filePaths":
				p.Type.IsSlice = true
			case "numFiles":
				p.Type.LengthOfSlice = "filePaths"
			}

			continue
		}

		// Whiteflag types that are return arguments
		if p.Type.PointerLevel == 1 && (p.Type.GoName == "File" || p.Type.GoName == "FileUniqueID" || p.Type.GoName == "IdxClientFile" || p.Type.GoName == "cxstring" || p.Type.GoName == gen.GoInt32 || p.Type.GoName == gen.GoUInt32 || p.Type.GoName == "CompilationDatabase_Error" || p.Type.GoName == "PlatformAvailability" || p.Type.GoName == "SourceRange" || p.Type.GoName == "LoadDiag_Error") {
			p.Type.IsReturnArgument = true
		}
		if p.Type.PointerLevel == 2 && (p.Type.GoName == "Token" || p.Type.GoName == "Cursor") {
			p.Type.IsReturnArgument = true
		}

		if f.CName == "clang_disposeOverriddenCursors" && p.CName == "overridden" {
			p.Type.IsSlice = true
		}

		// If this is an array length parameter we need to find its partner
		paCName := gen.ArrayNameFromLength(p.CName)

		if paCName != "" {
			for j := range f.Parameters {
				pa := &f.Parameters[j]

				if strings.ToLower(pa.CName) == strings.ToLower(paCName) {
					if pa.Type.GoName == "struct CXUnsavedFile" || pa.Type.GoName == "UnsavedFile" {
						pa.Type.GoName = "UnsavedFile"
						pa.Type.CGoName = "struct_CXUnsavedFile"
					} else if pa.Type.CGoName == gen.CSChar && pa.Type.PointerLevel == 2 {
					} else if pa.Type.GoName == "CompletionResult" {
					} else if pa.Type.GoName == "Token" {
					} else if pa.Type.GoName == "Cursor" {
					} else {
						break
					}

					p.Type.LengthOfSlice = pa.Name
					pa.Type.IsSlice = true

					if pa.Type.IsReturnArgument && p.Type.PointerLevel > 0 {
						p.Type.IsReturnArgument = true
					}

					break
				}
			}
		}
	}

	for i := range f.Parameters {
		p := &f.Parameters[i]

		if p.Type.CGoName == gen.CSChar && p.Type.PointerLevel == 2 && !p.Type.IsSlice {
			p.Type.IsReturnArgument = true
		}
	}
}

// FilterFunction reports whether the f function filtered to a particular condition.
func FilterFunction(f *gen.Function) bool {
	switch f.CName {
	case "clang_CompileCommand_getMappedSourceContent", "clang_CompileCommand_getMappedSourcePath", "clang_CompileCommand_getNumMappedSources":
		// some functions are not compiled in the library see https://lists.launchpad.net/desktop-packages/msg75835.html for a never resolved bug report
		fmt.Fprintf(os.Stderr, "Ignore function %q because it is not compiled within libClang\n", f.CName)

		return false

	case "clang_executeOnThread", "clang_getInclusions":
		// some functions can not be handled automatically by us
		fmt.Fprintf(os.Stderr, "Ignore function %q because it cannot be handled automatically\n", f.CName)

		return false

	case "clang_annotateTokens", "clang_getCursorPlatformAvailability", "clang_visitChildren":
		// some functions are simply manually implemented
		fmt.Fprintf(os.Stderr, "Ignore function %q because it is manually implemented\n", f.CName)

		return false
	}

	// TODO(go-clang): if this function is from CXString.h we ignore it https://github.com/go-clang/gen/issues/25
	for i := range f.IncludeFiles {
		if strings.HasSuffix(i, "CXString.h") {
			return false
		}
	}

	return true
}

// FilterFunctionParameter reports whether the p function parameter filtered to a particular condition.
func FilterFunctionParameter(p gen.FunctionParameter) bool {
	// these pointers are ok
	if p.Type.PointerLevel == 1 {
		if p.Type.CGoName == gen.CSChar {
			return false
		}

		switch p.Type.GoName {
		case "UnsavedFile",
			"CodeCompleteResults",
			"CursorKind",
			"IdxContainerInfo",
			"IdxDeclInfo",
			"IndexerCallbacks",
			"TranslationUnit",
			"IdxEntityInfo",
			"IdxAttrInfo":

			return false
		}
	}

	return true
}

// FixedFunctionName changes the function name under certain conditions.
func FixedFunctionName(f *gen.Function) string {
	// needs to be renamed manually since clang_getTranslationUnitCursor will conflict with clang_getCursor
	if f.CName == "clang_getTranslationUnitCursor" {
		return "TranslationUnitCursor"
	}

	return ""
}

// PrepareStructMembers prepares struct member names.
func PrepareStructMembers(s *gen.Struct) {
	for _, m := range s.Members {
		if (strings.HasPrefix(m.CName, "has") || strings.HasPrefix(m.CName, "is")) && m.Type.GoName == gen.GoInt32 {
			m.Type.GoName = gen.GoBool
		}

		// if this is an array length parameter we need to find its partner
		maCName := gen.ArrayNameFromLength(m.CName)

		if maCName != "" {
			for _, ma := range s.Members {
				if strings.ToLower(ma.CName) == strings.ToLower(maCName) {
					m.Type.LengthOfSlice = ma.CName
					ma.Type.IsSlice = true
					// TODO(go-clang): wrong usage but needed for the getter generation...
					// maybe refactor this LengthOfSlice alltogether?
					// https://github.com/go-clang/gen/issues/49
					ma.Type.LengthOfSlice = m.CName

					break
				}
			}
		}
	}

	prepareStructMembersArrayStruct(s)
}

// prepareStructMembersArrayStruct checks if the struct has two member variables, one is an array and the other a plain
// int/uint with size/length/count/len is its name because then this should be an array struct, and we connect them to handle a slice.
func prepareStructMembersArrayStruct(s *gen.Struct) {
	if len(s.Members) != 2 {
		return
	}

	if !arrayLengthCombination(&s.Members[0].Type, &s.Members[1].Type) && !arrayLengthCombination(&s.Members[1].Type, &s.Members[0].Type) {
		return
	}

	// if one of the members is already marked as array/slice another heuristic has already covered both members.
	switch {
	case s.Members[0].Type.IsArray,
		s.Members[1].Type.IsArray,
		s.Members[0].Type.IsSlice,
		s.Members[1].Type.IsSlice:

		return
	}

	var a *gen.StructMember
	var c *gen.StructMember

	if s.Members[0].Type.PointerLevel == 1 {
		a = s.Members[0]
		c = s.Members[1]
	} else {
		c = s.Members[0]
		a = s.Members[1]
	}

	lengthName := strings.ToLower(c.CName)
	if lengthName != "count" &&
		lengthName != "len" && lengthName != "length" && lengthName != "size" {

		return
	}

	c.Type.LengthOfSlice = a.CName
	a.Type.IsSlice = true
	// TODO(go-clang): wrong usage but needed for the getter generation...
	// maybe refactor this LengthOfSlice alltogether?
	// https://github.com/go-clang/gen/issues/49
	a.Type.LengthOfSlice = c.CName
}

// arrayLengthCombination reports whether the x and y to correct combination.
func arrayLengthCombination(x *gen.Type, y *gen.Type) bool {
	return x.PointerLevel == 1 && y.PointerLevel == 0 &&
		!gen.IsInteger(x) && gen.IsInteger(y)
}

// FilterStructMemberGetter reports whether the m struct member filtered to a particular condition.
func FilterStructMemberGetter(m *gen.StructMember) bool {
	// we do not want getters to *int_data members
	return !strings.HasSuffix(m.CName, "int_data")
}
<|endoftext|>


<commit_before>package expr

import (
	"context"
	"encoding/json"
	"fmt"
	"time"

	"github.com/grafana/grafana-plugin-sdk-go/backend"
	"github.com/grafana/grafana-plugin-sdk-go/data"
	"github.com/grafana/grafana/pkg/expr/classic"
	"github.com/grafana/grafana/pkg/expr/mathexp"
	"github.com/grafana/grafana/pkg/infra/log"

	"gonum.org/v1/gonum/graph/simple"
)

var (
	logger = log.New("expr")
)

// baseNode includes commmon properties used across DPNodes.
type baseNode struct {
	id    int64
	refID string
}

type rawNode struct {
	RefID         string `json:"refId"`
	Query         map[string]interface{}
	QueryType     string
	TimeRange     TimeRange
	DatasourceUID string
}

func (rn *rawNode) GetDatasourceName() (string, error) {
	rawDs, ok := rn.Query["datasource"]
	if !ok {
		return "", nil
	}
	dsName, ok := rawDs.(string)
	if !ok {
		return "", fmt.Errorf("expted datasource identifier to be a string, got %T", rawDs)
	}
	return dsName, nil
}

func (rn *rawNode) GetCommandType() (c CommandType, err error) {
	rawType, ok := rn.Query["type"]
	if !ok {
		return c, fmt.Errorf("no expression command type in query for refId %v", rn.RefID)
	}
	typeString, ok := rawType.(string)
	if !ok {
		return c, fmt.Errorf("expected expression command type to be a string, got type %T", rawType)
	}
	return ParseCommandType(typeString)
}

// String returns a string representation of the node. In particular for
// %v formating in error messages.
func (b *baseNode) String() string {
	return b.refID
}

// CMDNode is a DPNode that holds an expression command.
type CMDNode struct {
	baseNode
	CMDType CommandType
	Command Command
}

// ID returns the id of the node so it can fulfill the gonum's graph Node interface.
func (b *baseNode) ID() int64 {
	return b.id
}

// RefID returns the refId of the node.
func (b *baseNode) RefID() string {
	return b.refID
}

// NodeType returns the data pipeline node type.
func (gn *CMDNode) NodeType() NodeType {
	return TypeCMDNode
}

// Execute runs the node and adds the results to vars. If the node requires
// other nodes they must have already been executed and their results must
// already by in vars.
func (gn *CMDNode) Execute(ctx context.Context, vars mathexp.Vars, s *Service) (mathexp.Results, error) {
	return gn.Command.Execute(ctx, vars)
}

func buildCMDNode(dp *simple.DirectedGraph, rn *rawNode) (*CMDNode, error) {
	commandType, err := rn.GetCommandType()
	if err != nil {
		return nil, fmt.Errorf("invalid expression command type in '%v'", rn.RefID)
	}

	node := &CMDNode{
		baseNode: baseNode{
			id:    dp.NewNode().ID(),
			refID: rn.RefID,
		},
		CMDType: commandType,
	}

	switch commandType {
	case TypeMath:
		node.Command, err = UnmarshalMathCommand(rn)
	case TypeReduce:
		node.Command, err = UnmarshalReduceCommand(rn)
	case TypeResample:
		node.Command, err = UnmarshalResampleCommand(rn)
	case TypeClassicConditions:
		node.Command, err = classic.UnmarshalConditionsCmd(rn.Query, rn.RefID)
	default:
		return nil, fmt.Errorf("expression command type '%v' in '%v' not implemented", commandType, rn.RefID)
	}
	if err != nil {
		return nil, err
	}

	return node, nil
}

const (
	defaultIntervalMS = int64(64)
	defaultMaxDP      = int64(5000)
)

// DSNode is a DPNode that holds a datasource request.
type DSNode struct {
	baseNode
	query         json.RawMessage
	datasourceID  int64
	datasourceUID string

	orgID      int64
	queryType  string
	timeRange  TimeRange
	intervalMS int64
	maxDP      int64
}

// NodeType returns the data pipeline node type.
func (dn *DSNode) NodeType() NodeType {
	return TypeDatasourceNode
}

func (s *Service) buildDSNode(dp *simple.DirectedGraph, rn *rawNode, orgID int64) (*DSNode, error) {
	encodedQuery, err := json.Marshal(rn.Query)
	if err != nil {
		return nil, err
	}

	dsNode := &DSNode{
		baseNode: baseNode{
			id:    dp.NewNode().ID(),
			refID: rn.RefID,
		},
		orgID:      orgID,
		query:      json.RawMessage(encodedQuery),
		queryType:  rn.QueryType,
		intervalMS: defaultIntervalMS,
		maxDP:      defaultMaxDP,
		timeRange:  rn.TimeRange,
	}

	rawDsID, ok := rn.Query["datasourceId"]
	switch ok {
	case true:
		floatDsID, ok := rawDsID.(float64)
		if !ok {
			return nil, fmt.Errorf("expected datasourceId to be a float64, got type %T for refId %v", rawDsID, rn.RefID)
		}
		dsNode.datasourceID = int64(floatDsID)
	default:
		if rn.DatasourceUID == "" {
			return nil, fmt.Errorf("neither datasourceId or datasourceUid in expression data source request for refId %v", rn.RefID)
		}
		dsNode.datasourceUID = rn.DatasourceUID
	}

	var floatIntervalMS float64
	if rawIntervalMS := rn.Query["intervalMs"]; ok {
		if floatIntervalMS, ok = rawIntervalMS.(float64); !ok {
			return nil, fmt.Errorf("expected intervalMs to be an float64, got type %T for refId %v", rawIntervalMS, rn.RefID)
		}
		dsNode.intervalMS = int64(floatIntervalMS)
	}

	var floatMaxDP float64
	if rawMaxDP := rn.Query["maxDataPoints"]; ok {
		if floatMaxDP, ok = rawMaxDP.(float64); !ok {
			return nil, fmt.Errorf("expected maxDataPoints to be an float64, got type %T for refId %v", rawMaxDP, rn.RefID)
		}
		dsNode.maxDP = int64(floatMaxDP)
	}

	return dsNode, nil
}

// Execute runs the node and adds the results to vars. If the node requires
// other nodes they must have already been executed and their results must
// already by in vars.
func (dn *DSNode) Execute(ctx context.Context, vars mathexp.Vars, s *Service) (mathexp.Results, error) {
	pc := backend.PluginContext{
		OrgID: dn.orgID,
		DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{
			ID:  dn.datasourceID,
			UID: dn.datasourceUID,
		},
	}

	q := []backend.DataQuery{
		{
			RefID:         dn.refID,
			MaxDataPoints: dn.maxDP,
			Interval:      time.Duration(int64(time.Millisecond) * dn.intervalMS),
			JSON:          dn.query,
			TimeRange: backend.TimeRange{
				From: dn.timeRange.From,
				To:   dn.timeRange.To,
			},
			QueryType: dn.queryType,
		},
	}

	resp, err := s.queryData(ctx, &backend.QueryDataRequest{
		PluginContext: pc,
		Queries:       q,
	})

	if err != nil {
		return mathexp.Results{}, err
	}

	vals := make([]mathexp.Value, 0)
	for refID, qr := range resp.Responses {
		if qr.Error != nil {
			return mathexp.Results{}, fmt.Errorf("failed to execute query %v: %w", refID, qr.Error)
		}

		if len(qr.Frames) == 1 {
			frame := qr.Frames[0]
			if frame.TimeSeriesSchema().Type == data.TimeSeriesTypeNot && isNumberTable(frame) {
				logger.Debug("expression datasource query (numberSet)", "query", refID)
				numberSet, err := extractNumberSet(frame)
				if err != nil {
					return mathexp.Results{}, err
				}
				for _, n := range numberSet {
					vals = append(vals, n)
				}

				return mathexp.Results{
					Values: vals,
				}, nil
			}
		}

		for _, frame := range qr.Frames {
			logger.Debug("expression datasource query (seriesSet)", "query", refID)
			series, err := WideToMany(frame)
			if err != nil {
				return mathexp.Results{}, err
			}
			for _, s := range series {
				vals = append(vals, s)
			}
		}
	}
	return mathexp.Results{
		Values: vals,
	}, nil
}

func isNumberTable(frame *data.Frame) bool {
	if frame == nil || frame.Fields == nil {
		return false
	}
	numericCount := 0
	stringCount := 0
	otherCount := 0
	for _, field := range frame.Fields {
		fType := field.Type()
		switch {
		case fType.Numeric():
			numericCount++
		case fType == data.FieldTypeString || fType == data.FieldTypeNullableString:
			stringCount++
		default:
			otherCount++
		}
	}
	return numericCount == 1 && otherCount == 0
}

func extractNumberSet(frame *data.Frame) ([]mathexp.Number, error) {
	numericField := 0
	stringFieldIdxs := []int{}
	stringFieldNames := []string{}
	for i, field := range frame.Fields {
		fType := field.Type()
		switch {
		case fType.Numeric():
			numericField = i
		case fType == data.FieldTypeString || fType == data.FieldTypeNullableString:
			stringFieldIdxs = append(stringFieldIdxs, i)
			stringFieldNames = append(stringFieldNames, field.Name)
		}
	}
	numbers := make([]mathexp.Number, frame.Rows())

	for rowIdx := 0; rowIdx < frame.Rows(); rowIdx++ {
		val, _ := frame.FloatAt(numericField, rowIdx)
		var labels data.Labels
		for i := 0; i < len(stringFieldIdxs); i++ {
			if i == 0 {
				labels = make(data.Labels)
			}
			key := stringFieldNames[i] // TODO check for duplicate string column names
			val, _ := frame.ConcreteAt(stringFieldIdxs[i], rowIdx)
			labels[key] = val.(string) // TODO check assertion / return error
		}

		n := mathexp.NewNumber("", labels)
		n.SetValue(&val)
		numbers[rowIdx] = n
	}
	return numbers, nil
}

// WideToMany converts a data package wide type Frame to one or multiple Series. A series
// is created for each value type column of wide frame.
//
// This might not be a good idea long term, but works now as an adapter/shim.
func WideToMany(frame *data.Frame) ([]mathexp.Series, error) {
	tsSchema := frame.TimeSeriesSchema()
	if tsSchema.Type != data.TimeSeriesTypeWide {
		return nil, fmt.Errorf("input data must be a wide series but got type %s (input refid)", tsSchema.Type)
	}

	if len(tsSchema.ValueIndices) == 1 {
		s, err := mathexp.SeriesFromFrame(frame)
		if err != nil {
			return nil, err
		}
		return []mathexp.Series{s}, nil
	}

	series := []mathexp.Series{}
	for _, valIdx := range tsSchema.ValueIndices {
		l := frame.Rows()
		f := data.NewFrameOfFieldTypes(frame.Name, l, frame.Fields[tsSchema.TimeIndex].Type(), frame.Fields[valIdx].Type())
		f.Fields[0].Name = frame.Fields[tsSchema.TimeIndex].Name
		f.Fields[1].Name = frame.Fields[valIdx].Name
		if frame.Fields[valIdx].Labels != nil {
			f.Fields[1].Labels = frame.Fields[valIdx].Labels.Copy()
		}
		for i := 0; i < l; i++ {
			f.SetRow(i, frame.Fields[tsSchema.TimeIndex].CopyAt(i), frame.Fields[valIdx].CopyAt(i))
		}
		s, err := mathexp.SeriesFromFrame(f)
		if err != nil {
			return nil, err
		}
		series = append(series, s)
	}

	return series, nil
}
<commit_msg>Chore: Fix typo in nodes.go (#35312)<commit_after>package expr

import (
	"context"
	"encoding/json"
	"fmt"
	"time"

	"github.com/grafana/grafana-plugin-sdk-go/backend"
	"github.com/grafana/grafana-plugin-sdk-go/data"
	"github.com/grafana/grafana/pkg/expr/classic"
	"github.com/grafana/grafana/pkg/expr/mathexp"
	"github.com/grafana/grafana/pkg/infra/log"

	"gonum.org/v1/gonum/graph/simple"
)

var (
	logger = log.New("expr")
)

// baseNode includes commmon properties used across DPNodes.
type baseNode struct {
	id    int64
	refID string
}

type rawNode struct {
	RefID         string `json:"refId"`
	Query         map[string]interface{}
	QueryType     string
	TimeRange     TimeRange
	DatasourceUID string
}

func (rn *rawNode) GetDatasourceName() (string, error) {
	rawDs, ok := rn.Query["datasource"]
	if !ok {
		return "", nil
	}
	dsName, ok := rawDs.(string)
	if !ok {
		return "", fmt.Errorf("expted datasource identifier to be a string, got %T", rawDs)
	}
	return dsName, nil
}

func (rn *rawNode) GetCommandType() (c CommandType, err error) {
	rawType, ok := rn.Query["type"]
	if !ok {
		return c, fmt.Errorf("no expression command type in query for refId %v", rn.RefID)
	}
	typeString, ok := rawType.(string)
	if !ok {
		return c, fmt.Errorf("expected expression command type to be a string, got type %T", rawType)
	}
	return ParseCommandType(typeString)
}

// String returns a string representation of the node. In particular for
// %v formatting in error messages.
func (b *baseNode) String() string {
	return b.refID
}

// CMDNode is a DPNode that holds an expression command.
type CMDNode struct {
	baseNode
	CMDType CommandType
	Command Command
}

// ID returns the id of the node so it can fulfill the gonum's graph Node interface.
func (b *baseNode) ID() int64 {
	return b.id
}

// RefID returns the refId of the node.
func (b *baseNode) RefID() string {
	return b.refID
}

// NodeType returns the data pipeline node type.
func (gn *CMDNode) NodeType() NodeType {
	return TypeCMDNode
}

// Execute runs the node and adds the results to vars. If the node requires
// other nodes they must have already been executed and their results must
// already by in vars.
func (gn *CMDNode) Execute(ctx context.Context, vars mathexp.Vars, s *Service) (mathexp.Results, error) {
	return gn.Command.Execute(ctx, vars)
}

func buildCMDNode(dp *simple.DirectedGraph, rn *rawNode) (*CMDNode, error) {
	commandType, err := rn.GetCommandType()
	if err != nil {
		return nil, fmt.Errorf("invalid expression command type in '%v'", rn.RefID)
	}

	node := &CMDNode{
		baseNode: baseNode{
			id:    dp.NewNode().ID(),
			refID: rn.RefID,
		},
		CMDType: commandType,
	}

	switch commandType {
	case TypeMath:
		node.Command, err = UnmarshalMathCommand(rn)
	case TypeReduce:
		node.Command, err = UnmarshalReduceCommand(rn)
	case TypeResample:
		node.Command, err = UnmarshalResampleCommand(rn)
	case TypeClassicConditions:
		node.Command, err = classic.UnmarshalConditionsCmd(rn.Query, rn.RefID)
	default:
		return nil, fmt.Errorf("expression command type '%v' in '%v' not implemented", commandType, rn.RefID)
	}
	if err != nil {
		return nil, err
	}

	return node, nil
}

const (
	defaultIntervalMS = int64(64)
	defaultMaxDP      = int64(5000)
)

// DSNode is a DPNode that holds a datasource request.
type DSNode struct {
	baseNode
	query         json.RawMessage
	datasourceID  int64
	datasourceUID string

	orgID      int64
	queryType  string
	timeRange  TimeRange
	intervalMS int64
	maxDP      int64
}

// NodeType returns the data pipeline node type.
func (dn *DSNode) NodeType() NodeType {
	return TypeDatasourceNode
}

func (s *Service) buildDSNode(dp *simple.DirectedGraph, rn *rawNode, orgID int64) (*DSNode, error) {
	encodedQuery, err := json.Marshal(rn.Query)
	if err != nil {
		return nil, err
	}

	dsNode := &DSNode{
		baseNode: baseNode{
			id:    dp.NewNode().ID(),
			refID: rn.RefID,
		},
		orgID:      orgID,
		query:      json.RawMessage(encodedQuery),
		queryType:  rn.QueryType,
		intervalMS: defaultIntervalMS,
		maxDP:      defaultMaxDP,
		timeRange:  rn.TimeRange,
	}

	rawDsID, ok := rn.Query["datasourceId"]
	switch ok {
	case true:
		floatDsID, ok := rawDsID.(float64)
		if !ok {
			return nil, fmt.Errorf("expected datasourceId to be a float64, got type %T for refId %v", rawDsID, rn.RefID)
		}
		dsNode.datasourceID = int64(floatDsID)
	default:
		if rn.DatasourceUID == "" {
			return nil, fmt.Errorf("neither datasourceId or datasourceUid in expression data source request for refId %v", rn.RefID)
		}
		dsNode.datasourceUID = rn.DatasourceUID
	}

	var floatIntervalMS float64
	if rawIntervalMS := rn.Query["intervalMs"]; ok {
		if floatIntervalMS, ok = rawIntervalMS.(float64); !ok {
			return nil, fmt.Errorf("expected intervalMs to be an float64, got type %T for refId %v", rawIntervalMS, rn.RefID)
		}
		dsNode.intervalMS = int64(floatIntervalMS)
	}

	var floatMaxDP float64
	if rawMaxDP := rn.Query["maxDataPoints"]; ok {
		if floatMaxDP, ok = rawMaxDP.(float64); !ok {
			return nil, fmt.Errorf("expected maxDataPoints to be an float64, got type %T for refId %v", rawMaxDP, rn.RefID)
		}
		dsNode.maxDP = int64(floatMaxDP)
	}

	return dsNode, nil
}

// Execute runs the node and adds the results to vars. If the node requires
// other nodes they must have already been executed and their results must
// already by in vars.
func (dn *DSNode) Execute(ctx context.Context, vars mathexp.Vars, s *Service) (mathexp.Results, error) {
	pc := backend.PluginContext{
		OrgID: dn.orgID,
		DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{
			ID:  dn.datasourceID,
			UID: dn.datasourceUID,
		},
	}

	q := []backend.DataQuery{
		{
			RefID:         dn.refID,
			MaxDataPoints: dn.maxDP,
			Interval:      time.Duration(int64(time.Millisecond) * dn.intervalMS),
			JSON:          dn.query,
			TimeRange: backend.TimeRange{
				From: dn.timeRange.From,
				To:   dn.timeRange.To,
			},
			QueryType: dn.queryType,
		},
	}

	resp, err := s.queryData(ctx, &backend.QueryDataRequest{
		PluginContext: pc,
		Queries:       q,
	})

	if err != nil {
		return mathexp.Results{}, err
	}

	vals := make([]mathexp.Value, 0)
	for refID, qr := range resp.Responses {
		if qr.Error != nil {
			return mathexp.Results{}, fmt.Errorf("failed to execute query %v: %w", refID, qr.Error)
		}

		if len(qr.Frames) == 1 {
			frame := qr.Frames[0]
			if frame.TimeSeriesSchema().Type == data.TimeSeriesTypeNot && isNumberTable(frame) {
				logger.Debug("expression datasource query (numberSet)", "query", refID)
				numberSet, err := extractNumberSet(frame)
				if err != nil {
					return mathexp.Results{}, err
				}
				for _, n := range numberSet {
					vals = append(vals, n)
				}

				return mathexp.Results{
					Values: vals,
				}, nil
			}
		}

		for _, frame := range qr.Frames {
			logger.Debug("expression datasource query (seriesSet)", "query", refID)
			series, err := WideToMany(frame)
			if err != nil {
				return mathexp.Results{}, err
			}
			for _, s := range series {
				vals = append(vals, s)
			}
		}
	}
	return mathexp.Results{
		Values: vals,
	}, nil
}

func isNumberTable(frame *data.Frame) bool {
	if frame == nil || frame.Fields == nil {
		return false
	}
	numericCount := 0
	stringCount := 0
	otherCount := 0
	for _, field := range frame.Fields {
		fType := field.Type()
		switch {
		case fType.Numeric():
			numericCount++
		case fType == data.FieldTypeString || fType == data.FieldTypeNullableString:
			stringCount++
		default:
			otherCount++
		}
	}
	return numericCount == 1 && otherCount == 0
}

func extractNumberSet(frame *data.Frame) ([]mathexp.Number, error) {
	numericField := 0
	stringFieldIdxs := []int{}
	stringFieldNames := []string{}
	for i, field := range frame.Fields {
		fType := field.Type()
		switch {
		case fType.Numeric():
			numericField = i
		case fType == data.FieldTypeString || fType == data.FieldTypeNullableString:
			stringFieldIdxs = append(stringFieldIdxs, i)
			stringFieldNames = append(stringFieldNames, field.Name)
		}
	}
	numbers := make([]mathexp.Number, frame.Rows())

	for rowIdx := 0; rowIdx < frame.Rows(); rowIdx++ {
		val, _ := frame.FloatAt(numericField, rowIdx)
		var labels data.Labels
		for i := 0; i < len(stringFieldIdxs); i++ {
			if i == 0 {
				labels = make(data.Labels)
			}
			key := stringFieldNames[i] // TODO check for duplicate string column names
			val, _ := frame.ConcreteAt(stringFieldIdxs[i], rowIdx)
			labels[key] = val.(string) // TODO check assertion / return error
		}

		n := mathexp.NewNumber("", labels)
		n.SetValue(&val)
		numbers[rowIdx] = n
	}
	return numbers, nil
}

// WideToMany converts a data package wide type Frame to one or multiple Series. A series
// is created for each value type column of wide frame.
//
// This might not be a good idea long term, but works now as an adapter/shim.
func WideToMany(frame *data.Frame) ([]mathexp.Series, error) {
	tsSchema := frame.TimeSeriesSchema()
	if tsSchema.Type != data.TimeSeriesTypeWide {
		return nil, fmt.Errorf("input data must be a wide series but got type %s (input refid)", tsSchema.Type)
	}

	if len(tsSchema.ValueIndices) == 1 {
		s, err := mathexp.SeriesFromFrame(frame)
		if err != nil {
			return nil, err
		}
		return []mathexp.Series{s}, nil
	}

	series := []mathexp.Series{}
	for _, valIdx := range tsSchema.ValueIndices {
		l := frame.Rows()
		f := data.NewFrameOfFieldTypes(frame.Name, l, frame.Fields[tsSchema.TimeIndex].Type(), frame.Fields[valIdx].Type())
		f.Fields[0].Name = frame.Fields[tsSchema.TimeIndex].Name
		f.Fields[1].Name = frame.Fields[valIdx].Name
		if frame.Fields[valIdx].Labels != nil {
			f.Fields[1].Labels = frame.Fields[valIdx].Labels.Copy()
		}
		for i := 0; i < l; i++ {
			f.SetRow(i, frame.Fields[tsSchema.TimeIndex].CopyAt(i), frame.Fields[valIdx].CopyAt(i))
		}
		s, err := mathexp.SeriesFromFrame(f)
		if err != nil {
			return nil, err
		}
		series = append(series, s)
	}

	return series, nil
}
<|endoftext|>


<commit_before>package google

import (
	"errors"
	"fmt"
	"strconv"
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
	"regexp"
)

func TestAccTpuTensorflowVersions_basic(t *testing.T) {
	t.Parallel()

	resource.Test(t, resource.TestCase{
		PreCheck:  func() { testAccPreCheck(t) },
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			{
				Config: testAccTpuTensorFlowVersionsConfig,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckGoogleTpuTensorflowVersions("data.google_tpu_tensorflow_versions.available"),
				),
			},
		},
	})
}

func testAccCheckGoogleTpuTensorflowVersions(n string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Can't find TPU Tensorflow versions data source: %s", n)
		}

		if rs.Primary.ID == "" {
			return errors.New("data source ID not set.")
		}

		count, ok := rs.Primary.Attributes["versions.#"]
		if !ok {
			return errors.New("can't find 'names' attribute")
		}

		cnt, err := strconv.Atoi(count)
		if err != nil {
			return errors.New("failed to read number of version")
		}
		if cnt < 2 {
			return fmt.Errorf("expected at least 2 versions, received %d, this is most likely a bug", cnt)
		}

		for i := 0; i < cnt; i++ {
			idx := fmt.Sprintf("versions.%d", i)
			v, ok := rs.Primary.Attributes[idx]
			if !ok {
				return fmt.Errorf("expected %q, version not found", idx)
			}

			if !regexp.MustCompile(`^([0-9]+\.)+[0-9]+$`).MatchString(v) {
				return fmt.Errorf("unexpected version format for %q, value is %v", idx, v)
			}
		}
		return nil
	}
}

var testAccTpuTensorFlowVersionsConfig = `
data "google_tpu_tensorflow_versions" "available" {}
`
<commit_msg>Remove version format validation (#808)<commit_after>package google

import (
	"errors"
	"fmt"
	"strconv"
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestAccTpuTensorflowVersions_basic(t *testing.T) {
	t.Parallel()

	resource.Test(t, resource.TestCase{
		PreCheck:  func() { testAccPreCheck(t) },
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			{
				Config: testAccTpuTensorFlowVersionsConfig,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckGoogleTpuTensorflowVersions("data.google_tpu_tensorflow_versions.available"),
				),
			},
		},
	})
}

func testAccCheckGoogleTpuTensorflowVersions(n string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Can't find TPU Tensorflow versions data source: %s", n)
		}

		if rs.Primary.ID == "" {
			return errors.New("data source ID not set.")
		}

		count, ok := rs.Primary.Attributes["versions.#"]
		if !ok {
			return errors.New("can't find 'names' attribute")
		}

		cnt, err := strconv.Atoi(count)
		if err != nil {
			return errors.New("failed to read number of version")
		}
		if cnt < 2 {
			return fmt.Errorf("expected at least 2 versions, received %d, this is most likely a bug", cnt)
		}

		for i := 0; i < cnt; i++ {
			idx := fmt.Sprintf("versions.%d", i)
			_, ok := rs.Primary.Attributes[idx]
			if !ok {
				return fmt.Errorf("expected %q, version not found", idx)
			}
		}
		return nil
	}
}

var testAccTpuTensorFlowVersionsConfig = `
data "google_tpu_tensorflow_versions" "available" {}
`
<|endoftext|>


<commit_before>// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"path"
	"regexp"
	"sync"
	"time"

	"github.com/golang/glog"
	"golang.org/x/net/context"
	"google.golang.org/grpc"

	pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1alpha"
)

const (
	// All NVIDIA GPUs cards should be mounted with nvidiactl and nvidia-uvm
	// If the driver installed correctly, these two devices will be there.
	nvidiaCtlDevice = "/dev/nvidiactl"
	nvidiaUVMDevice = "/dev/nvidia-uvm"
	// Optional device.
	nvidiaUVMToolsDevice = "/dev/nvidia-uvm-tools"
	devDirectory         = "/dev"
	nvidiaDeviceRE       = `^nvidia[0-9]*$`

	// Device plugin settings.
	pluginMountPath      = "/device-plugin"
	kubeletEndpoint      = "kubelet.sock"
	pluginEndpointPrefix = "nvidiaGPU"
	resourceName         = "nvidia.com/gpu"
	ContainerPathPrefix  = "/usr/local/nvidia"
	HostPathPrefix       = "/home/kubernetes/bin/nvidia"
)

// nvidiaGPUManager manages nvidia gpu devices.
type nvidiaGPUManager struct {
	defaultDevices []string
	devices        map[string]pluginapi.Device
	grpcServer     *grpc.Server
}

func NewNvidiaGPUManager() *nvidiaGPUManager {
	return &nvidiaGPUManager{
		devices: make(map[string]pluginapi.Device),
	}
}

// Discovers all NVIDIA GPU devices available on the local node by walking `/dev` directory.
func (ngm *nvidiaGPUManager) discoverGPUs() error {
	reg := regexp.MustCompile(nvidiaDeviceRE)
	files, err := ioutil.ReadDir(devDirectory)
	if err != nil {
		return err
	}
	for _, f := range files {
		if f.IsDir() {
			continue
		}
		if reg.MatchString(f.Name()) {
			glog.Infof("Found Nvidia GPU %q\n", f.Name())
			ngm.devices[f.Name()] = pluginapi.Device{f.Name(), pluginapi.Healthy}
		}
	}
	return nil
}

func (ngm *nvidiaGPUManager) GetDeviceState(DeviceName string) string {
	// TODO: calling Nvidia tools to figure out actual device state
	return pluginapi.Healthy
}

// Discovers Nvidia GPU devices and sets up device access environment.
func (ngm *nvidiaGPUManager) Start() error {
	if _, err := os.Stat(nvidiaCtlDevice); err != nil {
		return err
	}

	if _, err := os.Stat(nvidiaUVMDevice); err != nil {
		return err
	}

	ngm.defaultDevices = []string{nvidiaCtlDevice, nvidiaUVMDevice}

	if _, err := os.Stat(nvidiaUVMToolsDevice); err != nil {
		ngm.defaultDevices = append(ngm.defaultDevices, nvidiaUVMToolsDevice)
	}

	if err := ngm.discoverGPUs(); err != nil {
		return err
	}

	return nil
}

func Register(kubeletEndpoint, pluginEndpoint, resourceName string) error {
	conn, err := grpc.Dial(kubeletEndpoint, grpc.WithInsecure(),
		grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) {
			return net.DialTimeout("unix", addr, timeout)
		}))
	if err != nil {
		return fmt.Errorf("device-plugin: cannot connect to kubelet service: %v", err)
	}
	defer conn.Close()
	client := pluginapi.NewRegistrationClient(conn)

	request := &pluginapi.RegisterRequest{
		Version:      pluginapi.Version,
		Endpoint:     pluginEndpoint,
		ResourceName: resourceName,
	}

	if _, err = client.Register(context.Background(), request); err != nil {
		return fmt.Errorf("device-plugin: cannot register to kubelet service: %v", err)
	}
	return nil
}

// Implements DevicePlugin service functions
func (ngm *nvidiaGPUManager) ListAndWatch(emtpy *pluginapi.Empty, stream pluginapi.DevicePlugin_ListAndWatchServer) error {
	glog.Infoln("device-plugin: ListAndWatch start")
	changed := true
	for {
		for id, dev := range ngm.devices {
			state := ngm.GetDeviceState(id)
			if dev.Health != state {
				changed = true
				dev.Health = state
				ngm.devices[id] = dev
			}
		}
		if changed {
			resp := new(pluginapi.ListAndWatchResponse)
			for _, dev := range ngm.devices {
				resp.Devices = append(resp.Devices, &pluginapi.Device{dev.ID, dev.Health})
			}
			glog.Infof("ListAndWatch: send devices %v\n", resp)
			if err := stream.Send(resp); err != nil {
				glog.Warningf("device-plugin: cannot update device states: %v\n", err)
				ngm.grpcServer.Stop()
				return err
			}
		}
		changed = false
		time.Sleep(5 * time.Second)
	}
}

func (ngm *nvidiaGPUManager) Allocate(ctx context.Context, rqt *pluginapi.AllocateRequest) (*pluginapi.AllocateResponse, error) {
	resp := new(pluginapi.AllocateResponse)
	// Add all requested devices to Allocate Response
	for _, id := range rqt.DevicesIDs {
		dev, ok := ngm.devices[id]
		if !ok {
			return nil, fmt.Errorf("invalid allocation request with non-existing device %s", id)
		}
		if dev.Health != pluginapi.Healthy {
			return nil, fmt.Errorf("invalid allocation request with unhealthy device %s", id)
		}
		resp.Devices = append(resp.Devices, &pluginapi.DeviceSpec{
			HostPath:      "/dev/" + id,
			ContainerPath: "/dev/" + id,
			Permissions:   "mrw",
		})
	}
	// Add all default devices to Allocate Response
	for _, d := range ngm.defaultDevices {
		resp.Devices = append(resp.Devices, &pluginapi.DeviceSpec{
			HostPath:      d,
			ContainerPath: d,
			Permissions:   "mrw",
		})
	}

	resp.Mounts = append(resp.Mounts, &pluginapi.Mount{
		ContainerPath: path.Join(ContainerPathPrefix, "lib64"),
		HostPath:      path.Join(HostPathPrefix, "lib"),
		ReadOnly:      true,
	})
	resp.Mounts = append(resp.Mounts, &pluginapi.Mount{
		ContainerPath: path.Join(ContainerPathPrefix, "bin"),
		HostPath:      path.Join(HostPathPrefix, "bin"),
		ReadOnly:      true,
	})
	return resp, nil
}

func (ngm *nvidiaGPUManager) Serve(pMountPath, kEndpoint, pEndpointPrefix string) {
	for {
		pluginEndpoint := fmt.Sprintf("%s-%d.sock", pEndpointPrefix, time.Now().Unix())
		pluginEndpointPath := path.Join(pMountPath, pluginEndpoint)
		glog.Infof("starting device-plugin server at: %s\n", pluginEndpointPath)
		lis, err := net.Listen("unix", pluginEndpointPath)
		if err != nil {
			glog.Fatalf("starting device-plugin server failed: %v", err)
		}
		ngm.grpcServer = grpc.NewServer()
		pluginapi.RegisterDevicePluginServer(ngm.grpcServer, ngm)

		var wg sync.WaitGroup
		wg.Add(1)
		// Starts device plugin service.
		go func() {
			defer wg.Done()
			// Blocking call to accept incoming connections.
			err := ngm.grpcServer.Serve(lis)
			glog.Errorf("device-plugin server stopped serving: %v", err)
		}()

		// Wait till the grpcServer is ready to serve services.
		for len(ngm.grpcServer.GetServiceInfo()) <= 0 {
			time.Sleep(1 * time.Second)
		}
		glog.Infoln("device-plugin server started serving")

		// Registers with Kubelet.
		err = Register(path.Join(pMountPath, kEndpoint), pluginEndpoint, resourceName)
		if err != nil {
			glog.Fatal(err)
		}
		glog.Infoln("device-plugin registered with the kubelet")

		// This is checking if the plugin socket was deleted. If so,
		// stop the grpc server and start the whole thing again.
		for {
			if _, err := os.Lstat(pluginEndpointPath); err != nil {
				glog.Errorln(err)
				ngm.grpcServer.Stop()
				break
			}
			time.Sleep(1 * time.Second)
		}
		wg.Wait()
	}
}

func main() {
	flag.Parse()
	glog.Infoln("device-plugin started")
	ngm := NewNvidiaGPUManager()
	// Keep on trying until success. This is required
	// because Nvidia drivers may not be installed initially.
	for {
		err := ngm.Start()
		if err == nil {
			break
		}
		// Use non-default level to avoid log spam.
		glog.V(3).Infof("nvidiaGPUManager.Start() failed: %v", err)
		time.Sleep(5 * time.Second)
	}
	ngm.Serve(pluginMountPath, kubeletEndpoint, pluginEndpointPrefix)
}
<commit_msg>Stop the server before fatally exiting.<commit_after>// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"path"
	"regexp"
	"sync"
	"time"

	"github.com/golang/glog"
	"golang.org/x/net/context"
	"google.golang.org/grpc"

	pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1alpha"
)

const (
	// All NVIDIA GPUs cards should be mounted with nvidiactl and nvidia-uvm
	// If the driver installed correctly, these two devices will be there.
	nvidiaCtlDevice = "/dev/nvidiactl"
	nvidiaUVMDevice = "/dev/nvidia-uvm"
	// Optional device.
	nvidiaUVMToolsDevice = "/dev/nvidia-uvm-tools"
	devDirectory         = "/dev"
	nvidiaDeviceRE       = `^nvidia[0-9]*$`

	// Device plugin settings.
	pluginMountPath      = "/device-plugin"
	kubeletEndpoint      = "kubelet.sock"
	pluginEndpointPrefix = "nvidiaGPU"
	resourceName         = "nvidia.com/gpu"
	ContainerPathPrefix  = "/usr/local/nvidia"
	HostPathPrefix       = "/home/kubernetes/bin/nvidia"
)

// nvidiaGPUManager manages nvidia gpu devices.
type nvidiaGPUManager struct {
	defaultDevices []string
	devices        map[string]pluginapi.Device
	grpcServer     *grpc.Server
}

func NewNvidiaGPUManager() *nvidiaGPUManager {
	return &nvidiaGPUManager{
		devices: make(map[string]pluginapi.Device),
	}
}

// Discovers all NVIDIA GPU devices available on the local node by walking `/dev` directory.
func (ngm *nvidiaGPUManager) discoverGPUs() error {
	reg := regexp.MustCompile(nvidiaDeviceRE)
	files, err := ioutil.ReadDir(devDirectory)
	if err != nil {
		return err
	}
	for _, f := range files {
		if f.IsDir() {
			continue
		}
		if reg.MatchString(f.Name()) {
			glog.Infof("Found Nvidia GPU %q\n", f.Name())
			ngm.devices[f.Name()] = pluginapi.Device{f.Name(), pluginapi.Healthy}
		}
	}
	return nil
}

func (ngm *nvidiaGPUManager) GetDeviceState(DeviceName string) string {
	// TODO: calling Nvidia tools to figure out actual device state
	return pluginapi.Healthy
}

// Discovers Nvidia GPU devices and sets up device access environment.
func (ngm *nvidiaGPUManager) Start() error {
	if _, err := os.Stat(nvidiaCtlDevice); err != nil {
		return err
	}

	if _, err := os.Stat(nvidiaUVMDevice); err != nil {
		return err
	}

	ngm.defaultDevices = []string{nvidiaCtlDevice, nvidiaUVMDevice}

	if _, err := os.Stat(nvidiaUVMToolsDevice); err != nil {
		ngm.defaultDevices = append(ngm.defaultDevices, nvidiaUVMToolsDevice)
	}

	if err := ngm.discoverGPUs(); err != nil {
		return err
	}

	return nil
}

// Act as a grpc client and register with the kubelet.
func Register(kubeletEndpoint, pluginEndpoint, resourceName string) error {
	conn, err := grpc.Dial(kubeletEndpoint, grpc.WithInsecure(),
		grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) {
			return net.DialTimeout("unix", addr, timeout)
		}))
	if err != nil {
		return fmt.Errorf("device-plugin: cannot connect to kubelet service: %v", err)
	}
	defer conn.Close()
	client := pluginapi.NewRegistrationClient(conn)

	request := &pluginapi.RegisterRequest{
		Version:      pluginapi.Version,
		Endpoint:     pluginEndpoint,
		ResourceName: resourceName,
	}

	if _, err = client.Register(context.Background(), request); err != nil {
		return fmt.Errorf("device-plugin: cannot register to kubelet service: %v", err)
	}
	return nil
}

// Implements DevicePlugin service functions
func (ngm *nvidiaGPUManager) ListAndWatch(emtpy *pluginapi.Empty, stream pluginapi.DevicePlugin_ListAndWatchServer) error {
	glog.Infoln("device-plugin: ListAndWatch start")
	changed := true
	for {
		for id, dev := range ngm.devices {
			state := ngm.GetDeviceState(id)
			if dev.Health != state {
				changed = true
				dev.Health = state
				ngm.devices[id] = dev
			}
		}
		if changed {
			resp := new(pluginapi.ListAndWatchResponse)
			for _, dev := range ngm.devices {
				resp.Devices = append(resp.Devices, &pluginapi.Device{dev.ID, dev.Health})
			}
			glog.Infof("ListAndWatch: send devices %v\n", resp)
			if err := stream.Send(resp); err != nil {
				glog.Errorf("device-plugin: cannot update device states: %v\n", err)
				ngm.grpcServer.Stop()
				return err
			}
		}
		changed = false
		time.Sleep(5 * time.Second)
	}
}

func (ngm *nvidiaGPUManager) Allocate(ctx context.Context, rqt *pluginapi.AllocateRequest) (*pluginapi.AllocateResponse, error) {
	resp := new(pluginapi.AllocateResponse)
	// Add all requested devices to Allocate Response
	for _, id := range rqt.DevicesIDs {
		dev, ok := ngm.devices[id]
		if !ok {
			return nil, fmt.Errorf("invalid allocation request with non-existing device %s", id)
		}
		if dev.Health != pluginapi.Healthy {
			return nil, fmt.Errorf("invalid allocation request with unhealthy device %s", id)
		}
		resp.Devices = append(resp.Devices, &pluginapi.DeviceSpec{
			HostPath:      "/dev/" + id,
			ContainerPath: "/dev/" + id,
			Permissions:   "mrw",
		})
	}
	// Add all default devices to Allocate Response
	for _, d := range ngm.defaultDevices {
		resp.Devices = append(resp.Devices, &pluginapi.DeviceSpec{
			HostPath:      d,
			ContainerPath: d,
			Permissions:   "mrw",
		})
	}

	resp.Mounts = append(resp.Mounts, &pluginapi.Mount{
		ContainerPath: path.Join(ContainerPathPrefix, "lib64"),
		HostPath:      path.Join(HostPathPrefix, "lib"),
		ReadOnly:      true,
	})
	resp.Mounts = append(resp.Mounts, &pluginapi.Mount{
		ContainerPath: path.Join(ContainerPathPrefix, "bin"),
		HostPath:      path.Join(HostPathPrefix, "bin"),
		ReadOnly:      true,
	})
	return resp, nil
}

func (ngm *nvidiaGPUManager) Serve(pMountPath, kEndpoint, pEndpointPrefix string) {
	for {
		pluginEndpoint := fmt.Sprintf("%s-%d.sock", pEndpointPrefix, time.Now().Unix())
		pluginEndpointPath := path.Join(pMountPath, pluginEndpoint)
		glog.Infof("starting device-plugin server at: %s\n", pluginEndpointPath)
		lis, err := net.Listen("unix", pluginEndpointPath)
		if err != nil {
			glog.Fatalf("starting device-plugin server failed: %v", err)
		}
		ngm.grpcServer = grpc.NewServer()
		pluginapi.RegisterDevicePluginServer(ngm.grpcServer, ngm)

		var wg sync.WaitGroup
		wg.Add(1)
		// Starts device plugin service.
		go func() {
			defer wg.Done()
			// Blocking call to accept incoming connections.
			err := ngm.grpcServer.Serve(lis)
			glog.Errorf("device-plugin server stopped serving: %v", err)
		}()

		// Wait till the grpcServer is ready to serve services.
		for len(ngm.grpcServer.GetServiceInfo()) <= 0 {
			time.Sleep(1 * time.Second)
		}
		glog.Infoln("device-plugin server started serving")

		// Registers with Kubelet.
		err = Register(path.Join(pMountPath, kEndpoint), pluginEndpoint, resourceName)
		if err != nil {
			ngm.grpcServer.Stop()
			wg.Wait()
			glog.Fatal(err)
		}
		glog.Infoln("device-plugin registered with the kubelet")

		// This is checking if the plugin socket was deleted. If so,
		// stop the grpc server and start the whole thing again.
		for {
			if _, err := os.Lstat(pluginEndpointPath); err != nil {
				glog.Errorln(err)
				ngm.grpcServer.Stop()
				break
			}
			time.Sleep(1 * time.Second)
		}
		wg.Wait()
	}
}

func main() {
	flag.Parse()
	glog.Infoln("device-plugin started")
	ngm := NewNvidiaGPUManager()
	// Keep on trying until success. This is required
	// because Nvidia drivers may not be installed initially.
	for {
		err := ngm.Start()
		if err == nil {
			break
		}
		// Use non-default level to avoid log spam.
		glog.V(3).Infof("nvidiaGPUManager.Start() failed: %v", err)
		time.Sleep(5 * time.Second)
	}
	ngm.Serve(pluginMountPath, kubeletEndpoint, pluginEndpointPrefix)
}
<|endoftext|>


<commit_before>package integration

import (
	"os"
	"os/exec"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Upgrade", func() {
	BeforeEach(func() {
		dir := setupTestWorkingDirWithVersion("v1.2.2")
		os.Chdir(dir)
	})
	Describe("Upgrading a cluster using offline mode", func() {
		Context("Using a minikube layout", func() {
			Context("Using Ubuntu 16.04", func() {
				ItOnAWS("should be upgraded [slow] [upgrade]", func(aws infrastructureProvisioner) {
					WithMiniInfrastructure(Ubuntu1604LTS, aws, func(node NodeDeets, sshKey string) {
						// Install previous version cluster
						err := installKismaticMini(node, sshKey)
						Expect(err).ToNot(HaveOccurred())

						// Extract current version of kismatic
						pwd, err := os.Getwd()
						Expect(err).ToNot(HaveOccurred())
						err = extractCurrentKismatic(pwd)
						Expect(err).ToNot(HaveOccurred())

						// Perform upgrade
						cmd := exec.Command("./kismatic", "upgrade", "offline", "-f", "kismatic-testing.yaml")
						cmd.Stderr = os.Stderr
						cmd.Stdout = os.Stdout
						err = cmd.Run()
						Expect(err).ToNot(HaveOccurred())
					})
				})
			})

			Context("Using CentOS 7", func() {
				ItOnAWS("should be upgraded [slow] [upgrade]", func(aws infrastructureProvisioner) {
					WithMiniInfrastructure(CentOS7, aws, func(node NodeDeets, sshKey string) {
						// Install previous version cluster
						err := installKismaticMini(node, sshKey)
						Expect(err).ToNot(HaveOccurred())

						// Extract new version of kismatic
						pwd, err := os.Getwd()
						Expect(err).ToNot(HaveOccurred())
						err = extractCurrentKismatic(pwd)
						Expect(err).ToNot(HaveOccurred())

						// Perform upgrade
						cmd := exec.Command("./kismatic", "upgrade", "offline", "-f", "kismatic-testing.yaml")
						cmd.Stderr = os.Stderr
						cmd.Stdout = os.Stdout
						err = cmd.Run()
						Expect(err).ToNot(HaveOccurred())
					})
				})
			})
		})

		Context("Using a 1/2/1 cluster", func() {
			ItOnAWS("should still be a highly available cluster after upgrade", func(aws infrastructureProvisioner) {
				WithInfrastructureAndDNS(NodeCount{1, 2, 1, 0, 0}, CentOS7, aws, func(nodes provisionedNodes, sshKey string) {
					opts := installOptions{allowPackageInstallation: true}
					err := installKismatic(nodes, opts, sshKey)
					Expect(err).ToNot(HaveOccurred())

					// Extract new version of kismatic
					pwd, err := os.Getwd()
					Expect(err).ToNot(HaveOccurred())
					err = extractCurrentKismatic(pwd)
					Expect(err).ToNot(HaveOccurred())

					// Perform upgrade
					cmd := exec.Command("./kismatic", "upgrade", "offline", "-f", "kismatic-testing.yaml")
					cmd.Stderr = os.Stderr
					cmd.Stdout = os.Stdout
					err = cmd.Run()
					Expect(err).ToNot(HaveOccurred())

					By("Removing a Kubernetes master node")
					err = aws.TerminateNode(nodes.master[0])
					Expect(err).ToNot(HaveOccurred(), "could not remove node")

					By("Re-running Kuberang")
					err = runViaSSH([]string{"sudo kuberang"}, []NodeDeets{nodes.master[1]}, sshKey, 5*time.Minute)
					Expect(err).ToNot(HaveOccurred())
				})
			})
		})

		// This spec will be used for testing non-destructive kismatic features on
		// an upgraded cluster.
		// This spec is open to modification when new assertions have to be made.
		Context("Using a skunkworks cluster", func() {
			ItOnAWS("should result in an upgraded cluster [slow] [upgrade]", func(aws infrastructureProvisioner) {
				WithInfrastructureAndDNS(NodeCount{Etcd: 3, Master: 2, Worker: 3, Ingress: 2, Storage: 2}, CentOS7, aws, func(nodes provisionedNodes, sshKey string) {
					// reserve one of the workers for the add-worker test
					allWorkers := nodes.worker
					nodes.worker = allWorkers[0 : len(nodes.worker)-1]

					// Standup cluster with previous version
					opts := installOptions{allowPackageInstallation: true}
					err := installKismatic(nodes, opts, sshKey)
					FailIfError(err)

					pwd, err := os.Getwd()
					FailIfError(err)
					err = extractCurrentKismatic(pwd)
					FailIfError(err)

					cmd := exec.Command("./kismatic", "upgrade", "offline", "-f", "kismatic-testing.yaml")
					cmd.Stdout = os.Stdout
					cmd.Stderr = os.Stderr
					err = cmd.Run()
					FailIfError(err)

					assertClusterVersionIsCurrent()

					sub := SubDescribe("Using an upgraded cluster")
					defer sub.Check()

					sub.It("should allow adding a new storage volume", func() error {
						planFile, err := os.Open("kismatic-testing.yaml")
						if err != nil {
							return err
						}
						return createVolume(planFile, "test-vol", 1, 1, "")
					})

					sub.It("should allow adding a worker node", func() error {
						newWorker := allWorkers[len(allWorkers)-1]
						return addWorkerToCluster(newWorker)
					})

					sub.It("should have an accessible dashboard", func() error {
						return canAccessDashboard()
					})

					sub.It("should be able to deploy a workload with ingress", func() error {
						return verifyIngressNodes(nodes.master[0], nodes.ingress, sshKey)
					})

					sub.It("should not have kube-apiserver systemd service", func() error {
						return nil
					})
				})
			})
		})

		Context("Using a cluster that has no internet access", func() {
			ItOnAWS("should result in an upgraded cluster", func(aws infrastructureProvisioner) {
				distro := CentOS7
				WithInfrastructure(NodeCount{Etcd: 1, Master: 1, Worker: 1}, distro, aws, func(nodes provisionedNodes, sshKey string) {
					// Standup cluster with previous version
					opts := installOptions{allowPackageInstallation: true}
					err := installKismatic(nodes, opts, sshKey)
					FailIfError(err)

					pwd, err := os.Getwd()
					FailIfError(err)
					err = extractCurrentKismatic(pwd)
					FailIfError(err)
					assertClusterVersionIsCurrent()

					// Manually install the new packages
					InstallKismaticPackages(nodes, distro, sshKey, true)

					// Lock down internet access
					err = disableInternetAccess(nodes.allNodes(), sshKey)
					FailIfError(err)

					// Perform upgrade
					cmd := exec.Command("./kismatic", "upgrade", "offline", "-f", "kismatic-testing.yaml")
					cmd.Stderr = os.Stderr
					cmd.Stdout = os.Stdout
					err = cmd.Run()
					Expect(err).ToNot(HaveOccurred())
				})
			})
		})
	})
})
<commit_msg>Test offline upgrades for versions 1.0 and 1.1 (#381)<commit_after>package integration

import (
	"os"
	"os/exec"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Upgrade", func() {
	Describe("Upgrading a cluster using offline mode", func() {
		Describe("From KET version v1.2.2", func() {
			BeforeEach(func() {
				dir := setupTestWorkingDirWithVersion("v1.2.2")
				os.Chdir(dir)
			})
			Context("Using a minikube layout", func() {
				Context("Using Ubuntu 16.04", func() {
					ItOnAWS("should be upgraded [slow] [upgrade]", func(aws infrastructureProvisioner) {
						WithMiniInfrastructure(Ubuntu1604LTS, aws, func(node NodeDeets, sshKey string) {
							installAndUpgradeMinikube(node, sshKey)
						})
					})
				})

				Context("Using RedHat 7", func() {
					ItOnAWS("should be upgraded [slow] [upgrade]", func(aws infrastructureProvisioner) {
						WithMiniInfrastructure(RedHat7, aws, func(node NodeDeets, sshKey string) {
							installAndUpgradeMinikube(node, sshKey)
						})
					})
				})
			})

			// This spec will be used for testing non-destructive kismatic features on
			// an upgraded cluster.
			// This spec is open to modification when new assertions have to be made.
			Context("Using a skunkworks cluster", func() {
				ItOnAWS("should result in an upgraded cluster [slow] [upgrade]", func(aws infrastructureProvisioner) {
					WithInfrastructureAndDNS(NodeCount{Etcd: 3, Master: 2, Worker: 3, Ingress: 2, Storage: 2}, CentOS7, aws, func(nodes provisionedNodes, sshKey string) {
						// reserve one of the workers for the add-worker test
						allWorkers := nodes.worker
						nodes.worker = allWorkers[0 : len(nodes.worker)-1]

						// Standup cluster with previous version
						opts := installOptions{allowPackageInstallation: true}
						err := installKismatic(nodes, opts, sshKey)
						FailIfError(err)

						// Extract current version of kismatic
						extractCurrentKismaticInstaller()

						// Perform upgrade
						upgradeCluster()

						sub := SubDescribe("Using an upgraded cluster")
						defer sub.Check()

						sub.It("should allow adding a new storage volume", func() error {
							planFile, err := os.Open("kismatic-testing.yaml")
							if err != nil {
								return err
							}
							return createVolume(planFile, "test-vol", 1, 1, "")
						})

						sub.It("should allow adding a worker node", func() error {
							newWorker := allWorkers[len(allWorkers)-1]
							return addWorkerToCluster(newWorker)
						})

						sub.It("should have an accessible dashboard", func() error {
							return nil
						})

						sub.It("should be able to deploy a workload with ingress", func() error {
							return verifyIngressNodes(nodes.master[0], nodes.ingress, sshKey)
						})

						sub.It("should have an accessible dashboard", func() error {
							return canAccessDashboard()
						})
						sub.It("should not have kube-apiserver systemd service", func() error {
							return nil
						})
					})
				})
			})

			Context("Using a 1/2/1 cluster", func() {
				ItOnAWS("should still be a highly available cluster after upgrade", func(aws infrastructureProvisioner) {
					WithInfrastructureAndDNS(NodeCount{1, 2, 1, 0, 0}, CentOS7, aws, func(nodes provisionedNodes, sshKey string) {
						opts := installOptions{allowPackageInstallation: true}
						err := installKismatic(nodes, opts, sshKey)
						Expect(err).ToNot(HaveOccurred())

						// Extract current version of kismatic
						extractCurrentKismaticInstaller()

						// Perform upgrade
						upgradeCluster()

						By("Removing a Kubernetes master node")
						err = aws.TerminateNode(nodes.master[0])
						Expect(err).ToNot(HaveOccurred(), "could not remove node")

						By("Re-running Kuberang")
						err = runViaSSH([]string{"sudo kuberang"}, []NodeDeets{nodes.master[1]}, sshKey, 5*time.Minute)
						Expect(err).ToNot(HaveOccurred())
					})
				})
			})

			Context("Using a cluster that has no internet access", func() {
				ItOnAWS("should result in an upgraded cluster", func(aws infrastructureProvisioner) {
					distro := CentOS7
					WithInfrastructure(NodeCount{Etcd: 1, Master: 1, Worker: 1}, distro, aws, func(nodes provisionedNodes, sshKey string) {
						// Standup cluster with previous version
						opts := installOptions{allowPackageInstallation: true}
						err := installKismatic(nodes, opts, sshKey)
						FailIfError(err)

						// Extract current version of kismatic
						extractCurrentKismaticInstaller()

						// Manually install the new packages
						InstallKismaticPackages(nodes, distro, sshKey, true)

						// Lock down internet access
						err = disableInternetAccess(nodes.allNodes(), sshKey)
						FailIfError(err)

						// Perform upgrade
						upgradeCluster()
					})
				})
			})
		})

		Describe("From KET version v1.1.1", func() {
			BeforeEach(func() {
				dir := setupTestWorkingDirWithVersion("v1.1.1")
				os.Chdir(dir)
			})
			Context("Using a minikube layout", func() {
				Context("Using CentOS 7", func() {
					ItOnAWS("should be upgraded [slow] [upgrade]", func(aws infrastructureProvisioner) {
						WithMiniInfrastructure(CentOS7, aws, func(node NodeDeets, sshKey string) {
							installAndUpgradeMinikube(node, sshKey)
						})
					})
				})

				Context("Using RedHat 7", func() {
					ItOnAWS("should be upgraded [slow] [upgrade]", func(aws infrastructureProvisioner) {
						WithMiniInfrastructure(RedHat7, aws, func(node NodeDeets, sshKey string) {
							installAndUpgradeMinikube(node, sshKey)
						})
					})
				})
			})
		})

		Describe("From KET version v1.0.3", func() {
			BeforeEach(func() {
				dir := setupTestWorkingDirWithVersion("v1.0.3")
				os.Chdir(dir)
			})
			Context("Using a minikube layout", func() {
				Context("Using CentOS 7", func() {
					ItOnAWS("should be upgraded [slow] [upgrade]", func(aws infrastructureProvisioner) {
						WithMiniInfrastructure(CentOS7, aws, func(node NodeDeets, sshKey string) {
							installAndUpgradeMinikube(node, sshKey)
						})
					})
				})

				Context("Using Ubuntu 16.04", func() {
					ItOnAWS("should be upgraded [slow] [upgrade]", func(aws infrastructureProvisioner) {
						WithMiniInfrastructure(Ubuntu1604LTS, aws, func(node NodeDeets, sshKey string) {
							installAndUpgradeMinikube(node, sshKey)
						})
					})
				})
			})
		})
	})
})

func installAndUpgradeMinikube(node NodeDeets, sshKey string) {
	// Install previous version cluster
	err := installKismaticMini(node, sshKey)
	FailIfError(err)
	extractCurrentKismaticInstaller()
	upgradeCluster()
}

func extractCurrentKismaticInstaller() {
	// Extract current version of kismatic
	pwd, err := os.Getwd()
	FailIfError(err)
	err = extractCurrentKismatic(pwd)
	FailIfError(err)
}
func upgradeCluster() {
	// Perform upgrade
	cmd := exec.Command("./kismatic", "upgrade", "offline", "-f", "kismatic-testing.yaml")
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	err := cmd.Run()
	FailIfError(err)

	assertClusterVersionIsCurrent()
}
<|endoftext|>