<commit_before>package api

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"image"
	"image/draw"
	"image/png" // This needs to be imported because otherwhise image.Decode won't work with png images
	"os"
	"strings"
	"sync"

	"github.com/disintegration/imaging"
	webpbin "github.com/mjarkk/go-webpbin"
	"github.com/mjarkk/wotnlclans/db"
	"github.com/mjarkk/wotnlclans/other"
)

type imageAndIDType struct {
	ID    string
	Image image.Image
}

// GetIcons fetches all clan icons and creates a grid
func GetIcons() error {
	other.DevPrint("Getting clan icons...")
	clans, err := db.GetCurrentClansData()
	if err != nil {
		return err
	}
	// toSavePath := "./icons/clanIcons"
	// os.RemoveAll(toSavePath)
	// os.Mkdir(toSavePath, os.ModePerm)
	imgAndID := []imageAndIDType{}

	var waitForImgs sync.WaitGroup
	waitForImgs.Add(len(clans))

	for _, clan := range clans {
		go func(clan db.ClanStats) {
			iconToGet, ok := clan.Emblems["X195.Portal"]
			if !ok {
				iconToGet, ok = clan.Emblems["X256.Wowp"]
				if !ok {
					apiErr("GetIcons", errors.New("no X195.Portal and/or X256.Wowp"), "Check if clan has a clan icon")
					return
				}
			}
			if !strings.Contains(iconToGet, "http://") && !strings.Contains(iconToGet, "https://") {
				apiErr("GetIcons", errors.New("image is not a valid url"), "Check if the image url contains http(s)// : "+iconToGet)
				return
			}
			out, err := RawGet(iconToGet)
			if err != nil {
				apiErr("GetIcons", err, "Can't fetch image: "+iconToGet)
				return
			}

			// uriDots := strings.Split(iconToGet, ".")
			// fileExt := uriDots[len(uriDots)-1]
			// imgPath := toSavePath + "/" + clan.ID + "." + fileExt
			// err = ioutil.WriteFile(imgPath, out, 0644)
			// if err != nil {
			// 	return
			// }

			img, _, err := image.Decode(bytes.NewReader(out))
			if err != nil {
				apiErr("GetIcons", err, "Can't decode image: "+iconToGet)
				return
			}

			imgAndID = append(imgAndID, imageAndIDType{
				ID:    clan.ID,
				Image: img,
			})
			waitForImgs.Done()
		}(clan)
	}

	waitForImgs.Wait()

	if len(imgAndID) > 0 {
		imgSize := 60
		imgsInARow := 30

		ids := [][]string{[]string{}}
		outputImg := image.NewRGBA(image.Rectangle{image.Point{0, 0}, image.Point{0, 0}})

		for _, imgObj := range imgAndID {

			id := imgObj.ID
			if len(ids[len(ids)-1]) == imgsInARow {
				ids = append(ids, []string{})
			}
			from := struct {
				top  int
				side int
			}{
				top:  len(ids),
				side: len(ids[len(ids)-1]),
			}

			insertImg := imaging.Resize(imgObj.Image, imgSize, imgSize, imaging.Lanczos)
			offset := image.Point{imgSize * from.side, imgSize * (from.top - 1)}
			whereToInsert := image.Rectangle{offset, offset.Add(insertImg.Bounds().Size())}
			maxSize := image.Point{(len(ids[0]) + 1) * imgSize, len(ids) * imgSize}

			canvas := image.NewRGBA(image.Rectangle{image.Point{0, 0}, maxSize})
			draw.Draw(canvas, outputImg.Bounds(), outputImg, image.Point{0, 0}, draw.Src)
			draw.Draw(canvas, whereToInsert, insertImg, image.Point{0, 0}, draw.Src)
			outputImg = canvas

			ids[len(ids)-1] = append(ids[len(ids)-1], id)
		}

		file, err := os.Create("./icons/allIcons.png")
		if err != nil {
			apiErr("GetIcons", err, "Can't create allIcons.png")
			return err
		}
		png.Encode(file, outputImg)
		file.Close()

		err = webpbin.NewCWebP().
			Quality(50).
			InputFile("./icons/allIcons.png").
			OutputFile("./icons/allIcons.webp").
			Run()
		if err != nil {
			apiErr("GetIcons", err, "Can't create allIcons.webp")
			return err
		}

		file, err = os.Create("./icons/allIcons.json")
		if err != nil {
			apiErr("GetIcons", err, "Can't create allIcons.json")
			return err
		}
		writer := bufio.NewWriter(file)

		defer func() {
			writer.Flush()
			file.Close()
		}()

		jsonEncoder := json.NewEncoder(writer)
		err = jsonEncoder.Encode(ids)
		if err != nil {
			apiErr("GetIcons", err, "Can't encode config into json")
			return err
		}
	}

	return nil
}
<commit_msg>Changed the checking<commit_after>package api

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"image"
	"image/draw"
	"image/png" // This needs to be imported because otherwhise image.Decode won't work with png images
	"os"
	"strings"
	"sync"

	"github.com/disintegration/imaging"
	webpbin "github.com/mjarkk/go-webpbin"
	"github.com/mjarkk/wotnlclans/db"
	"github.com/mjarkk/wotnlclans/other"
)

type imageAndIDType struct {
	ID    string
	Image image.Image
}

// GetIcons fetches all clan icons and creates a grid
func GetIcons() error {
	other.DevPrint("Getting clan icons...")
	clans, err := db.GetCurrentClansData()
	if err != nil {
		return err
	}
	// toSavePath := "./icons/clanIcons"
	// os.RemoveAll(toSavePath)
	// os.Mkdir(toSavePath, os.ModePerm)
	imgAndID := []imageAndIDType{}

	var waitForImgs sync.WaitGroup
	waitForImgs.Add(len(clans))

	for _, clan := range clans {
		go func(clan db.ClanStats) {
			iconToGet, ok := clan.Emblems["X195.Portal"]
			if !ok || len(iconToGet) == 0 {
				iconToGet, ok = clan.Emblems["X256.Wowp"]
				if !ok || len(iconToGet) == 0 {
					apiErr("GetIcons", errors.New("no X195.Portal and/or X256.Wowp"), "Check if clan has a clan icon")
					return
				}
			}
			if !strings.Contains(iconToGet, "http://") && !strings.Contains(iconToGet, "https://") {
				apiErr("GetIcons", errors.New("image is not a valid url"), "Check if the image url contains http(s)// : "+iconToGet)
				return
			}
			out, err := RawGet(iconToGet)
			if err != nil {
				apiErr("GetIcons", err, "Can't fetch image: "+iconToGet)
				return
			}

			// uriDots := strings.Split(iconToGet, ".")
			// fileExt := uriDots[len(uriDots)-1]
			// imgPath := toSavePath + "/" + clan.ID + "." + fileExt
			// err = ioutil.WriteFile(imgPath, out, 0644)
			// if err != nil {
			// 	return
			// }

			img, _, err := image.Decode(bytes.NewReader(out))
			if err != nil {
				apiErr("GetIcons", err, "Can't decode image: "+iconToGet)
				return
			}

			imgAndID = append(imgAndID, imageAndIDType{
				ID:    clan.ID,
				Image: img,
			})
			waitForImgs.Done()
		}(clan)
	}

	waitForImgs.Wait()

	if len(imgAndID) > 0 {
		imgSize := 60
		imgsInARow := 30

		ids := [][]string{[]string{}}
		outputImg := image.NewRGBA(image.Rectangle{image.Point{0, 0}, image.Point{0, 0}})

		for _, imgObj := range imgAndID {

			id := imgObj.ID
			if len(ids[len(ids)-1]) == imgsInARow {
				ids = append(ids, []string{})
			}
			from := struct {
				top  int
				side int
			}{
				top:  len(ids),
				side: len(ids[len(ids)-1]),
			}

			insertImg := imaging.Resize(imgObj.Image, imgSize, imgSize, imaging.Lanczos)
			offset := image.Point{imgSize * from.side, imgSize * (from.top - 1)}
			whereToInsert := image.Rectangle{offset, offset.Add(insertImg.Bounds().Size())}
			maxSize := image.Point{(len(ids[0]) + 1) * imgSize, len(ids) * imgSize}

			canvas := image.NewRGBA(image.Rectangle{image.Point{0, 0}, maxSize})
			draw.Draw(canvas, outputImg.Bounds(), outputImg, image.Point{0, 0}, draw.Src)
			draw.Draw(canvas, whereToInsert, insertImg, image.Point{0, 0}, draw.Src)
			outputImg = canvas

			ids[len(ids)-1] = append(ids[len(ids)-1], id)
		}

		file, err := os.Create("./icons/allIcons.png")
		if err != nil {
			apiErr("GetIcons", err, "Can't create allIcons.png")
			return err
		}
		png.Encode(file, outputImg)
		file.Close()

		err = webpbin.NewCWebP().
			Quality(50).
			InputFile("./icons/allIcons.png").
			OutputFile("./icons/allIcons.webp").
			Run()
		if err != nil {
			apiErr("GetIcons", err, "Can't create allIcons.webp")
			return err
		}

		file, err = os.Create("./icons/allIcons.json")
		if err != nil {
			apiErr("GetIcons", err, "Can't create allIcons.json")
			return err
		}
		writer := bufio.NewWriter(file)

		defer func() {
			writer.Flush()
			file.Close()
		}()

		jsonEncoder := json.NewEncoder(writer)
		err = jsonEncoder.Encode(ids)
		if err != nil {
			apiErr("GetIcons", err, "Can't encode config into json")
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"strings"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"golang.org/x/crypto/ssh"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

// TODO: it would probably be slightly better to build up the objects
// in the code and then serialize to yaml.
var addon_controller_v1 = `
apiVersion: v1beta3
kind: ReplicationController
metadata:
  name: addon-test-v1
  namespace: default
  labels:
    k8s-app: addon-test
    version: v1
    kubernetes.io/cluster-service: "true"
spec:
  replicas: 2
  selector:
    k8s-app: addon-test
    version: v1
  template:
    metadata:
      labels:
        k8s-app: addon-test
        version: v1
        kubernetes.io/cluster-service: "true"
    spec:
      containers:
      - image: gcr.io/google_containers/serve_hostname:1.1
        name: addon-test
        ports:
        - containerPort: 9376
          protocol: TCP
`

var addon_controller_v2 = `
apiVersion: v1beta3
kind: ReplicationController
metadata:
  name: addon-test-v2
  namespace: default
  labels:
    k8s-app: addon-test
    version: v2
    kubernetes.io/cluster-service: "true"
spec:
  replicas: 2
  selector:
    k8s-app: addon-test
    version: v2
  template:
    metadata:
      labels:
        k8s-app: addon-test
        version: v2
        kubernetes.io/cluster-service: "true"
    spec:
      containers:
      - image: gcr.io/google_containers/serve_hostname:1.1
        name: addon-test
        ports:
        - containerPort: 9376
          protocol: TCP
`

var addon_service_v1 = `
apiVersion: v1beta3
kind: Service
metadata:
  name: addon-test
  namespace: default
  labels:
    k8s-app: addon-test
    kubernetes.io/cluster-service: "true"
    kubernetes.io/name: addon-test
spec:
  ports:
  - port: 9376
    protocol: TCP
    targetPort: 9376
  selector:
    k8s-app: addon-test
`

var addon_service_v2 = `
apiVersion: v1beta3
kind: Service
metadata:
  name: addon-test-updated
  namespace: default
  labels:
    k8s-app: addon-test
    kubernetes.io/cluster-service: "true"
    kubernetes.io/name: addon-test
    newLabel: newValue
spec:
  ports:
  - port: 9376
    protocol: TCP
    targetPort: 9376
  selector:
    k8s-app: addon-test
`

var invalid_addon_controller_v1 = `
apiVersion: v1beta3
kind: ReplicationController
metadata:
  name: invalid-addon-test-v1
  namespace: default
  labels:
    k8s-app: invalid-addon-test
    version: v1
spec:
  replicas: 2
  selector:
    k8s-app: invalid-addon-test
    version: v1
  template:
    metadata:
      labels:
        k8s-app: invalid-addon-test
        version: v1
        kubernetes.io/cluster-service: "true"
    spec:
      containers:
      - image: gcr.io/google_containers/serve_hostname:1.1
        name: invalid-addon-test
        ports:
        - containerPort: 9376
          protocol: TCP
`

var invalid_addon_service_v1 = `
apiVersion: v1beta3
kind: Service
metadata:
  name: ivalid-addon-test
  namespace: default
  labels:
    k8s-app: invalid-addon-test
    kubernetes.io/name: invalid-addon-test
spec:
  ports:
  - port: 9377
    protocol: TCP
    targetPort: 9376
  selector:
    k8s-app: invalid-addon-test
`

var addonTestPollInterval = 3 * time.Second
var addonTestPollTimeout = 3 * time.Minute
var addonNamespace = api.NamespaceDefault // addons are in the default namespace

type stringPair struct {
	data, fileName string
}

var _ = Describe("Addon update", func() {

	var dir string
	var sshClient *ssh.Client
	var c *client.Client
	var namespace *api.Namespace

	BeforeEach(func() {
		var err error
		c, err = loadClient()
		Expect(err).NotTo(HaveOccurred())

		sshClient, err = getSSHClient()
		Expect(err).NotTo(HaveOccurred())

		namespace, err = createTestingNS("addon-update-test", c)
		Expect(err).NotTo(HaveOccurred())

		// do not use "service" command because it clears the environment variables
		sshExecAndVerify(sshClient, "sudo TEST_ADDON_CHECK_INTERVAL_SEC=1 /etc/init.d/kube-addons restart")
	})

	AfterEach(func() {
		if sshClient != nil {
			// restart addon_update with the default options
			sshExec(sshClient, "sudo /etc/init.d/kube-addons restart")
			sshClient.Close()
		}
		if err := c.Namespaces().Delete(namespace.Name); err != nil {
			Failf("Couldn't delete ns %q: %s", namespace, err)
		}
		// Paranoia-- prevent reuse!
		namespace = nil
		c = nil
	})

	// WARNING: the test is not parallel-friendly!
	It("should propagate add-on file changes", func() {
		//these tests are long, so I squeezed several cases in one scenario
		Expect(sshClient).NotTo(BeNil())
		dir = namespace.Name // we use it only to give a unique string for each test execution

		// This test requires SSH, so the provider check should be identical to
		// those tests.
		if !providerIs("gce") {
			Logf(fmt.Sprintf("Skipping test, which is not implemented for %s", testContext.Provider))
			return
		}

		temporaryRemotePathPrefix := "addon-test-dir"
		temporaryRemotePath := temporaryRemotePathPrefix + "/" + dir                  // in home directory on kubernetes-master
		defer sshExec(sshClient, fmt.Sprintf("rm -rf %s", temporaryRemotePathPrefix)) // ignore the result in cleanup
		sshExecAndVerify(sshClient, fmt.Sprintf("mkdir -p %s", temporaryRemotePath))

		rcv1 := "addon-controller-v1.yaml"
		rcv2 := "addon-controller-v2.yaml"
		rcInvalid := "invalid-addon-controller-v1.yaml"

		svcv1 := "addon-service-v1.yaml"
		svcv2 := "addon-service-v2.yaml"
		svcInvalid := "invalid-addon-service-v1.yaml"

		var remoteFiles []stringPair = []stringPair{
			{addon_controller_v1, rcv1},
			{addon_controller_v2, rcv2},
			{addon_service_v1, svcv1},
			{addon_service_v2, svcv2},
			{invalid_addon_controller_v1, rcInvalid},
			{invalid_addon_service_v1, svcInvalid},
		}

		for _, p := range remoteFiles {
			err := writeRemoteFile(sshClient, p.data, temporaryRemotePath, p.fileName, 0644)
			Expect(err).NotTo(HaveOccurred())
		}

		// directory on kubernetes-master
		destinationDirPrefix := "/etc/kubernetes/addons/addon-test-dir"
		destinationDir := destinationDirPrefix + "/" + dir

		// cleanup from previous tests
		_, _, _, err := sshExec(sshClient, fmt.Sprintf("sudo rm -rf %s", destinationDirPrefix))
		Expect(err).NotTo(HaveOccurred())

		defer sshExec(sshClient, fmt.Sprintf("sudo rm -rf %s", destinationDirPrefix)) // ignore result in cleanup
		sshExecAndVerify(sshClient, fmt.Sprintf("sudo mkdir -p %s", destinationDir))

		By("copy invalid manifests to the destination dir (without kubernetes.io/cluster-service label)")
		sshExecAndVerify(sshClient, fmt.Sprintf("sudo cp %s/%s %s/%s", temporaryRemotePath, rcInvalid, destinationDir, rcInvalid))
		sshExecAndVerify(sshClient, fmt.Sprintf("sudo cp %s/%s %s/%s", temporaryRemotePath, svcInvalid, destinationDir, svcInvalid))
		// we will verify at the end of the test that the objects weren't created from the invalid manifests

		By("copy new manifests")
		sshExecAndVerify(sshClient, fmt.Sprintf("sudo cp %s/%s %s/%s", temporaryRemotePath, rcv1, destinationDir, rcv1))
		sshExecAndVerify(sshClient, fmt.Sprintf("sudo cp %s/%s %s/%s", temporaryRemotePath, svcv1, destinationDir, svcv1))

		waitForServiceInAddonTest(c, "addon-test", true)
		waitForReplicationControllerInAddonTest(c, "addon-test-v1", true)

		By("update manifests")
		sshExecAndVerify(sshClient, fmt.Sprintf("sudo cp %s/%s %s/%s", temporaryRemotePath, rcv2, destinationDir, rcv2))
		sshExecAndVerify(sshClient, fmt.Sprintf("sudo cp %s/%s %s/%s", temporaryRemotePath, svcv2, destinationDir, svcv2))
		sshExecAndVerify(sshClient, fmt.Sprintf("sudo rm %s/%s", destinationDir, rcv1))
		sshExecAndVerify(sshClient, fmt.Sprintf("sudo rm %s/%s", destinationDir, svcv1))
		/**
		 * Note that we have a small race condition here - the kube-addon-updater
		 * May notice that a new rc/service file appeared, while the old one will still be there.
		 * But it is ok - as long as we don't have rolling update, the result will be the same
		 */

		waitForServiceInAddonTest(c, "addon-test-updated", true)
		waitForReplicationControllerInAddonTest(c, "addon-test-v2", true)

		waitForServiceInAddonTest(c, "addon-test", false)
		waitForReplicationControllerInAddonTest(c, "addon-test-v1", false)

		By("remove manifests")
		sshExecAndVerify(sshClient, fmt.Sprintf("sudo rm %s/%s", destinationDir, rcv2))
		sshExecAndVerify(sshClient, fmt.Sprintf("sudo rm %s/%s", destinationDir, svcv2))

		waitForServiceInAddonTest(c, "addon-test-updated", false)
		waitForReplicationControllerInAddonTest(c, "invalid-addon-test-v1", false)

		By("verify invalid API addons weren't created")
		_, err = c.ReplicationControllers(addonNamespace).Get("invalid-addon-test-v1")
		Expect(err).To(HaveOccurred())
		_, err = c.Services(addonNamespace).Get("ivalid-addon-test")
		Expect(err).To(HaveOccurred())

		// invalid addons will be deleted by the deferred function
	})
})

func waitForServiceInAddonTest(c *client.Client, name string, exist bool) {
	expectNoError(waitForService(c, addonNamespace, name, exist, addonTestPollInterval, addonTestPollTimeout))
}

func waitForReplicationControllerInAddonTest(c *client.Client, name string, exist bool) {
	expectNoError(waitForReplicationController(c, addonNamespace, name, exist, addonTestPollInterval, addonTestPollTimeout))
}

// TODO marekbiskup 2015-06-11: merge the ssh code into pkg/util/ssh.go after
// kubernetes v1.0 is released. In particular the code of sshExec.
func getSSHClient() (*ssh.Client, error) {
	// Get a signer for the provider.
	signer, err := getSigner(testContext.Provider)
	if err != nil {
		return nil, fmt.Errorf("error getting signer for provider %s: '%v'", testContext.Provider, err)
	}

	config := &ssh.ClientConfig{
		User: os.Getenv("USER"),
		Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)},
	}

	host := getMasterHost() + ":22"
	client, err := ssh.Dial("tcp", host, config)
	if err != nil {
		return nil, fmt.Errorf("error getting SSH client to host %s: '%v'", host, err)
	}
	return client, err
}

func sshExecAndVerify(client *ssh.Client, cmd string) {
	_, _, rc, err := sshExec(client, cmd)
	Expect(err).NotTo(HaveOccurred())
	Expect(rc).To(Equal(0))
}

func sshExec(client *ssh.Client, cmd string) (string, string, int, error) {
	Logf(fmt.Sprintf("Executing '%s' on %v", cmd, client.RemoteAddr()))
	session, err := client.NewSession()
	if err != nil {
		return "", "", 0, fmt.Errorf("error creating session to host %s: '%v'", client.RemoteAddr(), err)
	}
	defer session.Close()

	// Run the command.
	code := 0
	var bout, berr bytes.Buffer

	session.Stdout, session.Stderr = &bout, &berr
	err = session.Run(cmd)
	if err != nil {
		// Check whether the command failed to run or didn't complete.
		if exiterr, ok := err.(*ssh.ExitError); ok {
			// If we got an ExitError and the exit code is nonzero, we'll
			// consider the SSH itself successful (just that the command run
			// errored on the host).
			if code = exiterr.ExitStatus(); code != 0 {
				err = nil
			}
		} else {
			// Some other kind of error happened (e.g. an IOError); consider the
			// SSH unsuccessful.
			err = fmt.Errorf("failed running `%s` on %s: '%v'", cmd, client.RemoteAddr(), err)
		}
	}
	return bout.String(), berr.String(), code, err
}

func writeRemoteFile(sshClient *ssh.Client, data, dir, fileName string, mode os.FileMode) error {
	Logf(fmt.Sprintf("Writing remote file '%s/%s' on %v", dir, fileName, sshClient.RemoteAddr()))
	session, err := sshClient.NewSession()
	if err != nil {
		return fmt.Errorf("error creating session to host %s: '%v'", sshClient.RemoteAddr(), err)
	}
	defer session.Close()

	fileSize := len(data)
	go func() {
		// ignore errors here. scp whould return errors if something goes wrong.
		pipe, _ := session.StdinPipe()
		defer pipe.Close()
		fmt.Fprintf(pipe, "C%#o %d %s\n", mode, fileSize, fileName)
		io.Copy(pipe, strings.NewReader(data))
		fmt.Fprint(pipe, "\x00")
	}()
	if err := session.Run(fmt.Sprintf("scp -t %s", dir)); err != nil {
		return err
	}
	return nil
}
<commit_msg>port Addon update test to v1 api<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"strings"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"golang.org/x/crypto/ssh"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

// TODO: it would probably be slightly better to build up the objects
// in the code and then serialize to yaml.
var addon_controller_v1 = `
apiVersion: v1
kind: ReplicationController
metadata:
  name: addon-test-v1
  namespace: default
  labels:
    k8s-app: addon-test
    version: v1
    kubernetes.io/cluster-service: "true"
spec:
  replicas: 2
  selector:
    k8s-app: addon-test
    version: v1
  template:
    metadata:
      labels:
        k8s-app: addon-test
        version: v1
        kubernetes.io/cluster-service: "true"
    spec:
      containers:
      - image: gcr.io/google_containers/serve_hostname:1.1
        name: addon-test
        ports:
        - containerPort: 9376
          protocol: TCP
`

var addon_controller_v2 = `
apiVersion: v1
kind: ReplicationController
metadata:
  name: addon-test-v2
  namespace: default
  labels:
    k8s-app: addon-test
    version: v2
    kubernetes.io/cluster-service: "true"
spec:
  replicas: 2
  selector:
    k8s-app: addon-test
    version: v2
  template:
    metadata:
      labels:
        k8s-app: addon-test
        version: v2
        kubernetes.io/cluster-service: "true"
    spec:
      containers:
      - image: gcr.io/google_containers/serve_hostname:1.1
        name: addon-test
        ports:
        - containerPort: 9376
          protocol: TCP
`

var addon_service_v1 = `
apiVersion: v1
kind: Service
metadata:
  name: addon-test
  namespace: default
  labels:
    k8s-app: addon-test
    kubernetes.io/cluster-service: "true"
    kubernetes.io/name: addon-test
spec:
  ports:
  - port: 9376
    protocol: TCP
    targetPort: 9376
  selector:
    k8s-app: addon-test
`

var addon_service_v2 = `
apiVersion: v1
kind: Service
metadata:
  name: addon-test-updated
  namespace: default
  labels:
    k8s-app: addon-test
    kubernetes.io/cluster-service: "true"
    kubernetes.io/name: addon-test
    newLabel: newValue
spec:
  ports:
  - port: 9376
    protocol: TCP
    targetPort: 9376
  selector:
    k8s-app: addon-test
`

var invalid_addon_controller_v1 = `
apiVersion: v1
kind: ReplicationController
metadata:
  name: invalid-addon-test-v1
  namespace: default
  labels:
    k8s-app: invalid-addon-test
    version: v1
spec:
  replicas: 2
  selector:
    k8s-app: invalid-addon-test
    version: v1
  template:
    metadata:
      labels:
        k8s-app: invalid-addon-test
        version: v1
        kubernetes.io/cluster-service: "true"
    spec:
      containers:
      - image: gcr.io/google_containers/serve_hostname:1.1
        name: invalid-addon-test
        ports:
        - containerPort: 9376
          protocol: TCP
`

var invalid_addon_service_v1 = `
apiVersion: v1
kind: Service
metadata:
  name: ivalid-addon-test
  namespace: default
  labels:
    k8s-app: invalid-addon-test
    kubernetes.io/name: invalid-addon-test
spec:
  ports:
  - port: 9377
    protocol: TCP
    targetPort: 9376
  selector:
    k8s-app: invalid-addon-test
`

var addonTestPollInterval = 3 * time.Second
var addonTestPollTimeout = 3 * time.Minute
var addonNamespace = api.NamespaceDefault // addons are in the default namespace

type stringPair struct {
	data, fileName string
}

var _ = Describe("Addon update", func() {

	var dir string
	var sshClient *ssh.Client
	var c *client.Client
	var namespace *api.Namespace

	BeforeEach(func() {
		var err error
		c, err = loadClient()
		Expect(err).NotTo(HaveOccurred())

		sshClient, err = getSSHClient()
		Expect(err).NotTo(HaveOccurred())

		namespace, err = createTestingNS("addon-update-test", c)
		Expect(err).NotTo(HaveOccurred())

		// do not use "service" command because it clears the environment variables
		sshExecAndVerify(sshClient, "sudo TEST_ADDON_CHECK_INTERVAL_SEC=1 /etc/init.d/kube-addons restart")
	})

	AfterEach(func() {
		if sshClient != nil {
			// restart addon_update with the default options
			sshExec(sshClient, "sudo /etc/init.d/kube-addons restart")
			sshClient.Close()
		}
		if err := c.Namespaces().Delete(namespace.Name); err != nil {
			Failf("Couldn't delete ns %q: %s", namespace, err)
		}
		// Paranoia-- prevent reuse!
		namespace = nil
		c = nil
	})

	// WARNING: the test is not parallel-friendly!
	It("should propagate add-on file changes", func() {
		//these tests are long, so I squeezed several cases in one scenario
		Expect(sshClient).NotTo(BeNil())
		dir = namespace.Name // we use it only to give a unique string for each test execution

		// This test requires SSH, so the provider check should be identical to
		// those tests.
		if !providerIs("gce") {
			Logf(fmt.Sprintf("Skipping test, which is not implemented for %s", testContext.Provider))
			return
		}

		temporaryRemotePathPrefix := "addon-test-dir"
		temporaryRemotePath := temporaryRemotePathPrefix + "/" + dir                  // in home directory on kubernetes-master
		defer sshExec(sshClient, fmt.Sprintf("rm -rf %s", temporaryRemotePathPrefix)) // ignore the result in cleanup
		sshExecAndVerify(sshClient, fmt.Sprintf("mkdir -p %s", temporaryRemotePath))

		rcv1 := "addon-controller-v1.yaml"
		rcv2 := "addon-controller-v2.yaml"
		rcInvalid := "invalid-addon-controller-v1.yaml"

		svcv1 := "addon-service-v1.yaml"
		svcv2 := "addon-service-v2.yaml"
		svcInvalid := "invalid-addon-service-v1.yaml"

		var remoteFiles []stringPair = []stringPair{
			{addon_controller_v1, rcv1},
			{addon_controller_v2, rcv2},
			{addon_service_v1, svcv1},
			{addon_service_v2, svcv2},
			{invalid_addon_controller_v1, rcInvalid},
			{invalid_addon_service_v1, svcInvalid},
		}

		for _, p := range remoteFiles {
			err := writeRemoteFile(sshClient, p.data, temporaryRemotePath, p.fileName, 0644)
			Expect(err).NotTo(HaveOccurred())
		}

		// directory on kubernetes-master
		destinationDirPrefix := "/etc/kubernetes/addons/addon-test-dir"
		destinationDir := destinationDirPrefix + "/" + dir

		// cleanup from previous tests
		_, _, _, err := sshExec(sshClient, fmt.Sprintf("sudo rm -rf %s", destinationDirPrefix))
		Expect(err).NotTo(HaveOccurred())

		defer sshExec(sshClient, fmt.Sprintf("sudo rm -rf %s", destinationDirPrefix)) // ignore result in cleanup
		sshExecAndVerify(sshClient, fmt.Sprintf("sudo mkdir -p %s", destinationDir))

		By("copy invalid manifests to the destination dir (without kubernetes.io/cluster-service label)")
		sshExecAndVerify(sshClient, fmt.Sprintf("sudo cp %s/%s %s/%s", temporaryRemotePath, rcInvalid, destinationDir, rcInvalid))
		sshExecAndVerify(sshClient, fmt.Sprintf("sudo cp %s/%s %s/%s", temporaryRemotePath, svcInvalid, destinationDir, svcInvalid))
		// we will verify at the end of the test that the objects weren't created from the invalid manifests

		By("copy new manifests")
		sshExecAndVerify(sshClient, fmt.Sprintf("sudo cp %s/%s %s/%s", temporaryRemotePath, rcv1, destinationDir, rcv1))
		sshExecAndVerify(sshClient, fmt.Sprintf("sudo cp %s/%s %s/%s", temporaryRemotePath, svcv1, destinationDir, svcv1))

		waitForServiceInAddonTest(c, "addon-test", true)
		waitForReplicationControllerInAddonTest(c, "addon-test-v1", true)

		By("update manifests")
		sshExecAndVerify(sshClient, fmt.Sprintf("sudo cp %s/%s %s/%s", temporaryRemotePath, rcv2, destinationDir, rcv2))
		sshExecAndVerify(sshClient, fmt.Sprintf("sudo cp %s/%s %s/%s", temporaryRemotePath, svcv2, destinationDir, svcv2))
		sshExecAndVerify(sshClient, fmt.Sprintf("sudo rm %s/%s", destinationDir, rcv1))
		sshExecAndVerify(sshClient, fmt.Sprintf("sudo rm %s/%s", destinationDir, svcv1))
		/**
		 * Note that we have a small race condition here - the kube-addon-updater
		 * May notice that a new rc/service file appeared, while the old one will still be there.
		 * But it is ok - as long as we don't have rolling update, the result will be the same
		 */

		waitForServiceInAddonTest(c, "addon-test-updated", true)
		waitForReplicationControllerInAddonTest(c, "addon-test-v2", true)

		waitForServiceInAddonTest(c, "addon-test", false)
		waitForReplicationControllerInAddonTest(c, "addon-test-v1", false)

		By("remove manifests")
		sshExecAndVerify(sshClient, fmt.Sprintf("sudo rm %s/%s", destinationDir, rcv2))
		sshExecAndVerify(sshClient, fmt.Sprintf("sudo rm %s/%s", destinationDir, svcv2))

		waitForServiceInAddonTest(c, "addon-test-updated", false)
		waitForReplicationControllerInAddonTest(c, "invalid-addon-test-v1", false)

		By("verify invalid API addons weren't created")
		_, err = c.ReplicationControllers(addonNamespace).Get("invalid-addon-test-v1")
		Expect(err).To(HaveOccurred())
		_, err = c.Services(addonNamespace).Get("ivalid-addon-test")
		Expect(err).To(HaveOccurred())

		// invalid addons will be deleted by the deferred function
	})
})

func waitForServiceInAddonTest(c *client.Client, name string, exist bool) {
	expectNoError(waitForService(c, addonNamespace, name, exist, addonTestPollInterval, addonTestPollTimeout))
}

func waitForReplicationControllerInAddonTest(c *client.Client, name string, exist bool) {
	expectNoError(waitForReplicationController(c, addonNamespace, name, exist, addonTestPollInterval, addonTestPollTimeout))
}

// TODO marekbiskup 2015-06-11: merge the ssh code into pkg/util/ssh.go after
// kubernetes v1.0 is released. In particular the code of sshExec.
func getSSHClient() (*ssh.Client, error) {
	// Get a signer for the provider.
	signer, err := getSigner(testContext.Provider)
	if err != nil {
		return nil, fmt.Errorf("error getting signer for provider %s: '%v'", testContext.Provider, err)
	}

	config := &ssh.ClientConfig{
		User: os.Getenv("USER"),
		Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)},
	}

	host := getMasterHost() + ":22"
	client, err := ssh.Dial("tcp", host, config)
	if err != nil {
		return nil, fmt.Errorf("error getting SSH client to host %s: '%v'", host, err)
	}
	return client, err
}

func sshExecAndVerify(client *ssh.Client, cmd string) {
	_, _, rc, err := sshExec(client, cmd)
	Expect(err).NotTo(HaveOccurred())
	Expect(rc).To(Equal(0))
}

func sshExec(client *ssh.Client, cmd string) (string, string, int, error) {
	Logf(fmt.Sprintf("Executing '%s' on %v", cmd, client.RemoteAddr()))
	session, err := client.NewSession()
	if err != nil {
		return "", "", 0, fmt.Errorf("error creating session to host %s: '%v'", client.RemoteAddr(), err)
	}
	defer session.Close()

	// Run the command.
	code := 0
	var bout, berr bytes.Buffer

	session.Stdout, session.Stderr = &bout, &berr
	err = session.Run(cmd)
	if err != nil {
		// Check whether the command failed to run or didn't complete.
		if exiterr, ok := err.(*ssh.ExitError); ok {
			// If we got an ExitError and the exit code is nonzero, we'll
			// consider the SSH itself successful (just that the command run
			// errored on the host).
			if code = exiterr.ExitStatus(); code != 0 {
				err = nil
			}
		} else {
			// Some other kind of error happened (e.g. an IOError); consider the
			// SSH unsuccessful.
			err = fmt.Errorf("failed running `%s` on %s: '%v'", cmd, client.RemoteAddr(), err)
		}
	}
	return bout.String(), berr.String(), code, err
}

func writeRemoteFile(sshClient *ssh.Client, data, dir, fileName string, mode os.FileMode) error {
	Logf(fmt.Sprintf("Writing remote file '%s/%s' on %v", dir, fileName, sshClient.RemoteAddr()))
	session, err := sshClient.NewSession()
	if err != nil {
		return fmt.Errorf("error creating session to host %s: '%v'", sshClient.RemoteAddr(), err)
	}
	defer session.Close()

	fileSize := len(data)
	go func() {
		// ignore errors here. scp whould return errors if something goes wrong.
		pipe, _ := session.StdinPipe()
		defer pipe.Close()
		fmt.Fprintf(pipe, "C%#o %d %s\n", mode, fileSize, fileName)
		io.Copy(pipe, strings.NewReader(data))
		fmt.Fprint(pipe, "\x00")
	}()
	if err := session.Run(fmt.Sprintf("scp -t %s", dir)); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package e2e

import (
	"fmt"
	"testing"

	"github.com/stretchr/testify/assert"

	. "github.com/argoproj/argo-cd/v2/test/e2e/fixture"
	. "github.com/argoproj/argo-cd/v2/util/errors"
)

func TestClusterList(t *testing.T) {
	output := FailOnErr(RunCli("cluster", "list")).(string)
	assert.Equal(t, fmt.Sprintf(`SERVER                          NAME        VERSION  STATUS      MESSAGE
https://kubernetes.default.svc  in-cluster  %v     Successful  `, GetVersions().ServerVersion), output)
}

func TestClusterGet(t *testing.T) {
	output := FailOnErr(RunCli("cluster", "get", "https://kubernetes.default.svc")).(string)

	assert.Contains(t, output, fmt.Sprintf(`
name: in-cluster
server: https://kubernetes.default.svc
serverVersion: "%v"`, GetVersions().ServerVersion))

	assert.Contains(t, output, `config:
  tlsClientConfig:
    insecure: false`)

	assert.Contains(t, output, `status: Successful`)
}
<commit_msg>chore: Skip cluster e2e tests if already run (#6705)<commit_after>package e2e

import (
	"fmt"
	"testing"

	"github.com/stretchr/testify/assert"

	. "github.com/argoproj/argo-cd/v2/test/e2e/fixture"
	. "github.com/argoproj/argo-cd/v2/util/errors"
)

func TestClusterList(t *testing.T) {
	SkipIfAlreadyRun(t)
	defer RecordTestRun(t)
	output := FailOnErr(RunCli("cluster", "list")).(string)
	assert.Equal(t, fmt.Sprintf(`SERVER                          NAME        VERSION  STATUS      MESSAGE
https://kubernetes.default.svc  in-cluster  %v     Successful  `, GetVersions().ServerVersion), output)
}

func TestClusterGet(t *testing.T) {
	SkipIfAlreadyRun(t)
	defer RecordTestRun(t)
	output := FailOnErr(RunCli("cluster", "get", "https://kubernetes.default.svc")).(string)

	assert.Contains(t, output, fmt.Sprintf(`
name: in-cluster
server: https://kubernetes.default.svc
serverVersion: "%v"`, GetVersions().ServerVersion))

	assert.Contains(t, output, `config:
  tlsClientConfig:
    insecure: false`)

	assert.Contains(t, output, `status: Successful`)
}
<|endoftext|>


<commit_before>// errchk $G -e $D/$F.go

// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

func main() {
	var i int = 100
	i = i << -3 // ERROR "overflows"
}

/*
ixedbugs/bug016.go:7: overflow converting constant to <uint32>UINT32
fixedbugs/bug016.go:7: illegal types for operand: AS
	(<int32>INT32)
*/
<commit_msg>test: match gccgo error messages for bug016.go.<commit_after>// errchk $G -e $D/$F.go

// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

func main() {
	var i int = 100
	i = i << -3 // ERROR "overflows|negative"
}

/*
ixedbugs/bug016.go:7: overflow converting constant to <uint32>UINT32
fixedbugs/bug016.go:7: illegal types for operand: AS
	(<int32>INT32)
*/
<|endoftext|>


<commit_before>// errchk $G -e $D/$F.go

// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Test cases for revised conversion rules.

package main

func main() {
	type NewInt int
	i0 := 0
	var i1 int = 1
	var i2 NewInt = 1
	i0 = i0
	i0 = i1
	i0 = int(i2)
	i1 = i0
	i1 = i1
	i1 = int(i2)
	i2 = NewInt(i0)
	i2 = NewInt(i1)
	i2 = i2

	type A1 [3]int
	type A2 [3]NewInt
	var a0 [3]int
	var a1 A1
	var a2 A2
	a0 = a0
	a0 = a1
	a0 = [3]int(a2) // ERROR "cannot"
	a1 = a0
	a1 = a1
	a1 = A1(a2) // ERROR "cannot"
	a2 = A2(a0) // ERROR "cannot"
	a2 = A2(a1) // ERROR "cannot"
	a2 = a2

	type S1 struct {
		x int
	}
	type S2 struct {
		x NewInt
	}
	var s0 struct {
		x int
	}
	var s1 S1
	var s2 S2
	s0 = s0
	s0 = s1
	s0 = struct {
		x int
	}(s2) // ERROR "cannot"
	s1 = s0
	s1 = s1
	s1 = S1(s2) // ERROR "cannot"
	s2 = S2(s0) // ERROR "cannot"
	s2 = S2(s1) // ERROR "cannot"
	s2 = s2

	type P1 *int
	type P2 *NewInt
	var p0 *int
	var p1 P1
	var p2 P2
	p0 = p0
	p0 = p1
	p0 = (*int)(p2) // ERROR "cannot"
	p1 = p0
	p1 = p1
	p1 = P1(p2) // ERROR "cannot"
	p2 = P2(p0) // ERROR "cannot"
	p2 = P2(p1) // ERROR "cannot"
	p2 = p2

	type Q1 *struct {
		x int
	}
	type Q2 *S1
	var q0 *struct {
		x int
	}
	var q1 Q1
	var q2 Q2
	var ps1 *S1
	q0 = q0
	q0 = q1
	q0 = (*struct {
		x int
	})(ps1) // legal because of special conversion exception for pointers
	q0 = (*struct {
		x int
	})(q2) // ERROR "cannot"
	q1 = q0
	q1 = q1
	q1 = Q1(q2)    // ERROR "cannot"
	q2 = (*S1)(q0) // legal because of special conversion exception for pointers
	q2 = Q2(q1)    // ERROR "cannot"
	q2 = q2

	type F1 func(x NewInt) int
	type F2 func(x int) NewInt
	var f0 func(x NewInt) int
	var f1 F1
	var f2 F2
	f0 = f0
	f0 = f1
	f0 = func(x NewInt) int(f2) // ERROR "cannot"
	f1 = f0
	f1 = f1
	f1 = F1(f2) // ERROR "cannot"
	f2 = F2(f0) // ERROR "cannot"
	f2 = F2(f1) // ERROR "cannot"
	f2 = f2

	type X1 interface {
		f() int
	}
	type X2 interface {
		f() NewInt
	}
	var x0 interface {
		f() int
	}
	var x1 X1
	var x2 X2
	x0 = x0
	x0 = x1
	x0 = interface {
		f() int
	}(x2) // ERROR "cannot|need type assertion"
	x1 = x0
	x1 = x1
	x1 = X1(x2) // ERROR "cannot|need type assertion"
	x2 = X2(x0) // ERROR "cannot|need type assertion"
	x2 = X2(x1) // ERROR "cannot|need type assertion"
	x2 = x2

	type L1 []int
	type L2 []NewInt
	var l0 []int
	var l1 L1
	var l2 L2
	l0 = l0
	l0 = l1
	l0 = []int(l2) // ERROR "cannot"
	l1 = l0
	l1 = l1
	l1 = L1(l2) // ERROR "cannot"
	l2 = L2(l0) // ERROR "cannot"
	l2 = L2(l1) // ERROR "cannot"
	l2 = l2

	type M1 map[string]int
	type M2 map[string]NewInt
	var m0 []int
	var m1 L1
	var m2 L2
	m0 = m0
	m0 = m1
	m0 = []int(m2) // ERROR "cannot"
	m1 = m0
	m1 = m1
	m1 = L1(m2) // ERROR "cannot"
	m2 = L2(m0) // ERROR "cannot"
	m2 = L2(m1) // ERROR "cannot"
	m2 = m2

	type C1 chan int
	type C2 chan NewInt
	var c0 chan int
	var c1 C1
	var c2 C2
	c0 = c0
	c0 = c1
	c0 = chan int(c2) // ERROR "cannot"
	c1 = c0
	c1 = c1
	c1 = C1(c2) // ERROR "cannot"
	c2 = C2(c0) // ERROR "cannot"
	c2 = C2(c1) // ERROR "cannot"
	c2 = c2

	// internal compiler error (6g and gccgo)
	type T interface{}
	var _ T = 17 // assignment compatible
	_ = T(17)    // internal compiler error even though assignment compatible
}
<commit_msg>test: Match gccgo error messages.<commit_after>// errchk $G -e $D/$F.go

// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Test cases for revised conversion rules.

package main

func main() {
	type NewInt int
	i0 := 0
	var i1 int = 1
	var i2 NewInt = 1
	i0 = i0
	i0 = i1
	i0 = int(i2)
	i1 = i0
	i1 = i1
	i1 = int(i2)
	i2 = NewInt(i0)
	i2 = NewInt(i1)
	i2 = i2

	type A1 [3]int
	type A2 [3]NewInt
	var a0 [3]int
	var a1 A1
	var a2 A2
	a0 = a0
	a0 = a1
	a0 = [3]int(a2) // ERROR "cannot|invalid"
	a1 = a0
	a1 = a1
	a1 = A1(a2) // ERROR "cannot|invalid"
	a2 = A2(a0) // ERROR "cannot|invalid"
	a2 = A2(a1) // ERROR "cannot|invalid"
	a2 = a2

	type S1 struct {
		x int
	}
	type S2 struct {
		x NewInt
	}
	var s0 struct {
		x int
	}
	var s1 S1
	var s2 S2
	s0 = s0
	s0 = s1
	s0 = struct {
		x int
	}(s2) // ERROR "cannot|invalid"
	s1 = s0
	s1 = s1
	s1 = S1(s2) // ERROR "cannot|invalid"
	s2 = S2(s0) // ERROR "cannot|invalid"
	s2 = S2(s1) // ERROR "cannot|invalid"
	s2 = s2

	type P1 *int
	type P2 *NewInt
	var p0 *int
	var p1 P1
	var p2 P2
	p0 = p0
	p0 = p1
	p0 = (*int)(p2) // ERROR "cannot|invalid"
	p1 = p0
	p1 = p1
	p1 = P1(p2) // ERROR "cannot|invalid"
	p2 = P2(p0) // ERROR "cannot|invalid"
	p2 = P2(p1) // ERROR "cannot|invalid"
	p2 = p2

	type Q1 *struct {
		x int
	}
	type Q2 *S1
	var q0 *struct {
		x int
	}
	var q1 Q1
	var q2 Q2
	var ps1 *S1
	q0 = q0
	q0 = q1
	q0 = (*struct {
		x int
	})(ps1) // legal because of special conversion exception for pointers
	q0 = (*struct {
		x int
	})(q2) // ERROR "cannot|invalid"
	q1 = q0
	q1 = q1
	q1 = Q1(q2)    // ERROR "cannot|invalid"
	q2 = (*S1)(q0) // legal because of special conversion exception for pointers
	q2 = Q2(q1)    // ERROR "cannot|invalid"
	q2 = q2

	type F1 func(x NewInt) int
	type F2 func(x int) NewInt
	var f0 func(x NewInt) int
	var f1 F1
	var f2 F2
	f0 = f0
	f0 = f1
	f0 = func(x NewInt) int(f2) // ERROR "cannot|invalid"
	f1 = f0
	f1 = f1
	f1 = F1(f2) // ERROR "cannot|invalid"
	f2 = F2(f0) // ERROR "cannot|invalid"
	f2 = F2(f1) // ERROR "cannot|invalid"
	f2 = f2

	type X1 interface {
		f() int
	}
	type X2 interface {
		f() NewInt
	}
	var x0 interface {
		f() int
	}
	var x1 X1
	var x2 X2
	x0 = x0
	x0 = x1
	x0 = interface {
		f() int
	}(x2) // ERROR "cannot|need type assertion|incompatible"
	x1 = x0
	x1 = x1
	x1 = X1(x2) // ERROR "cannot|need type assertion|incompatible"
	x2 = X2(x0) // ERROR "cannot|need type assertion|incompatible"
	x2 = X2(x1) // ERROR "cannot|need type assertion|incompatible"
	x2 = x2

	type L1 []int
	type L2 []NewInt
	var l0 []int
	var l1 L1
	var l2 L2
	l0 = l0
	l0 = l1
	l0 = []int(l2) // ERROR "cannot|invalid"
	l1 = l0
	l1 = l1
	l1 = L1(l2) // ERROR "cannot|invalid"
	l2 = L2(l0) // ERROR "cannot|invalid"
	l2 = L2(l1) // ERROR "cannot|invalid"
	l2 = l2

	type M1 map[string]int
	type M2 map[string]NewInt
	var m0 []int
	var m1 L1
	var m2 L2
	m0 = m0
	m0 = m1
	m0 = []int(m2) // ERROR "cannot|invalid"
	m1 = m0
	m1 = m1
	m1 = L1(m2) // ERROR "cannot|invalid"
	m2 = L2(m0) // ERROR "cannot|invalid"
	m2 = L2(m1) // ERROR "cannot|invalid"
	m2 = m2

	type C1 chan int
	type C2 chan NewInt
	var c0 chan int
	var c1 C1
	var c2 C2
	c0 = c0
	c0 = c1
	c0 = chan int(c2) // ERROR "cannot|invalid"
	c1 = c0
	c1 = c1
	c1 = C1(c2) // ERROR "cannot|invalid"
	c2 = C2(c0) // ERROR "cannot|invalid"
	c2 = C2(c1) // ERROR "cannot|invalid"
	c2 = c2

	// internal compiler error (6g and gccgo)
	type T interface{}
	var _ T = 17 // assignment compatible
	_ = T(17)    // internal compiler error even though assignment compatible
}
<|endoftext|>


<commit_before><commit_msg>veyron/services/mgmt/device/impl/impl_test.go: test helper refactoring<commit_after><|endoftext|>


<commit_before>package api

import (
	"net/http"

	"github.com/labstack/echo"
)

type Prefix struct {
	Background string   `json:"background"`
	Color      string   `json:"color"`
	Words      []string `json:"words"`
	TimedEvent bool     `json:"timedEvent"`
	Default    bool     `json:"default"`
}

var DefaultPrefixes = []Prefix{
	Prefix{
		Background: "4C6C9B",
		Color:      "FFFFFF",
		Words:      []string{"HW", "Read", "Reading"},
		Default:    true,
	},
	Prefix{
		Background: "9ACD32",
		Color:      "FFFFFF",
		Words:      []string{"Project"},
		Default:    true,
	},
	Prefix{
		Background: "C3A528",
		Color:      "FFFFFF",
		Words:      []string{"Report", "Essay", "Paper", "Write"},
		Default:    true,
	},
	Prefix{
		Background: "FFA500",
		Color:      "FFFFFF",
		Words:      []string{"Quiz", "PopQuiz"},
		Default:    true,
	},
	Prefix{
		Background: "DC143C",
		Color:      "FFFFFF",
		Words:      []string{"Test", "Final", "Exam", "Midterm"},
		Default:    true,
	},
	Prefix{
		Background: "2AC0F1",
		Color:      "FFFFFF",
		Words:      []string{"ICA"},
		Default:    true,
	},
	Prefix{
		Background: "2AF15E",
		Color:      "FFFFFF",
		Words:      []string{"Lab", "Study", "Memorize"},
		TimedEvent: true,
		Default:    true,
	},
	Prefix{
		Background: "003DAD",
		Color:      "FFFFFF",
		Words:      []string{"DocID"},
		Default:    true,
	},
	Prefix{
		Background: "000000",
		Color:      "00FF00",
		Words:      []string{"Trojun", "Hex"},
		Default:    true,
	},
	Prefix{
		Background: "5000BC",
		Color:      "FFFFFF",
		Words:      []string{"OptionalHW", "Challenge"},
		Default:    true,
	},
	Prefix{
		Background: "000099",
		Color:      "FFFFFF",
		Words:      []string{"Presentation", "Prez"},
		Default:    true,
	},
	Prefix{
		Background: "123456",
		Color:      "FFFFFF",
		Words:      []string{"BuildSession", "Build"},
		TimedEvent: true,
		Default:    true,
	},
	Prefix{
		Background: "5A1B87",
		Color:      "FFFFFF",
		Words:      []string{"Meeting", "Meet"},
		TimedEvent: true,
		Default:    true,
	},
	Prefix{
		Background: "01b501",
		Color:      "FFFFFF",
		Words:      []string{"Begin", "Start"},
		TimedEvent: true,
		Default:    true,
	},
}

type PrefixesResponse struct {
	Status             string   `json:"status"`
	Prefixes           []Prefix `json:"prefixes"`
	FallbackBackground string   `json:"fallbackBackground"`
	FallbackColor      string   `json:"fallbackColor"`
}

func InitPrefixesAPI(e *echo.Echo) {
	e.GET("/prefixes/getList", func(c echo.Context) error {
		if GetSessionUserID(&c) == -1 {
			return c.JSON(http.StatusUnauthorized, ErrorResponse{"error", "logged_out"})
		}

		return c.JSON(http.StatusOK, PrefixesResponse{"ok", DefaultPrefixes, "FFD3BD", "000000"})
	})
}
<commit_msg>Add GradedHW and GradedHomework<commit_after>package api

import (
	"net/http"

	"github.com/labstack/echo"
)

type Prefix struct {
	Background string   `json:"background"`
	Color      string   `json:"color"`
	Words      []string `json:"words"`
	TimedEvent bool     `json:"timedEvent"`
	Default    bool     `json:"default"`
}

var DefaultPrefixes = []Prefix{
	Prefix{
		Background: "4C6C9B",
		Color:      "FFFFFF",
		Words:      []string{"HW", "Read", "Reading"},
		Default:    true,
	},
	Prefix{
		Background: "9ACD32",
		Color:      "FFFFFF",
		Words:      []string{"Project"},
		Default:    true,
	},
	Prefix{
		Background: "C3A528",
		Color:      "FFFFFF",
		Words:      []string{"Report", "Essay", "Paper", "Write"},
		Default:    true,
	},
	Prefix{
		Background: "FFA500",
		Color:      "FFFFFF",
		Words:      []string{"Quiz", "PopQuiz", "GradedHW", "GradedHomework"},
		Default:    true,
	},
	Prefix{
		Background: "DC143C",
		Color:      "FFFFFF",
		Words:      []string{"Test", "Final", "Exam", "Midterm"},
		Default:    true,
	},
	Prefix{
		Background: "2AC0F1",
		Color:      "FFFFFF",
		Words:      []string{"ICA"},
		Default:    true,
	},
	Prefix{
		Background: "2AF15E",
		Color:      "FFFFFF",
		Words:      []string{"Lab", "Study", "Memorize"},
		TimedEvent: true,
		Default:    true,
	},
	Prefix{
		Background: "003DAD",
		Color:      "FFFFFF",
		Words:      []string{"DocID"},
		Default:    true,
	},
	Prefix{
		Background: "000000",
		Color:      "00FF00",
		Words:      []string{"Trojun", "Hex"},
		Default:    true,
	},
	Prefix{
		Background: "5000BC",
		Color:      "FFFFFF",
		Words:      []string{"OptionalHW", "Challenge"},
		Default:    true,
	},
	Prefix{
		Background: "000099",
		Color:      "FFFFFF",
		Words:      []string{"Presentation", "Prez"},
		Default:    true,
	},
	Prefix{
		Background: "123456",
		Color:      "FFFFFF",
		Words:      []string{"BuildSession", "Build"},
		TimedEvent: true,
		Default:    true,
	},
	Prefix{
		Background: "5A1B87",
		Color:      "FFFFFF",
		Words:      []string{"Meeting", "Meet"},
		TimedEvent: true,
		Default:    true,
	},
	Prefix{
		Background: "01b501",
		Color:      "FFFFFF",
		Words:      []string{"Begin", "Start"},
		TimedEvent: true,
		Default:    true,
	},
}

type PrefixesResponse struct {
	Status             string   `json:"status"`
	Prefixes           []Prefix `json:"prefixes"`
	FallbackBackground string   `json:"fallbackBackground"`
	FallbackColor      string   `json:"fallbackColor"`
}

func InitPrefixesAPI(e *echo.Echo) {
	e.GET("/prefixes/getList", func(c echo.Context) error {
		if GetSessionUserID(&c) == -1 {
			return c.JSON(http.StatusUnauthorized, ErrorResponse{"error", "logged_out"})
		}

		return c.JSON(http.StatusOK, PrefixesResponse{"ok", DefaultPrefixes, "FFD3BD", "000000"})
	})
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"github.com/rkoesters/xdg/basedir"
	"github.com/rkoesters/xkcd"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
)

func getComicPath(n int) string {
	return filepath.Join(basedir.CacheHome, "xkcd-gui", strconv.Itoa(n))
}

func getComicInfoPath(n int) string {
	return filepath.Join(getComicPath(n), "info")
}

func getComicImagePath(n int) string {
	return filepath.Join(getComicPath(n), "image")
}

func getComicInfo(n int) (*xkcd.Comic, error) {
	infoPath := getComicInfoPath(n)

	// First, check if we have the file.
	_, err := os.Stat(infoPath)
	if os.IsNotExist(err) {
		err = downloadComicInfo(n)
		if err != nil {
			return nil, err
		}
	} else if err != nil {
		return nil, err
	}

	f, err := os.Open(infoPath)
	if err != nil {
		return nil, err
	}
	c, err := xkcd.New(f)
	if err != nil {
		return nil, err
	}
	return c, nil
}

func downloadComicInfo(n int) error {
	err := os.MkdirAll(getComicPath(n), 0777)
	if err != nil {
		return err
	}

	comic, err := xkcd.Get(n)
	if err != nil {
		return err
	}

	f, err := os.Create(getComicInfoPath(n))
	if err != nil {
		return err
	}

	e := json.NewEncoder(f)
	err = e.Encode(comic)
	if err != nil {
		return err
	}
	return nil
}

func getComicImage(n int) (string, error) {
	imagePath := getComicImagePath(n)

	_, err := os.Stat(imagePath)
	if os.IsNotExist(err) {
		err = downloadComicImage(n)
		if err != nil {
			return "", err
		}
	} else if err != nil {
		return "", err
	}
	return imagePath, nil
}

func downloadComicImage(n int) error {
	c, err := getComicInfo(n)
	if err != nil {
		return err
	}

	f, err := os.Create(getComicImagePath(n))
	if err != nil {
		return err
	}

	resp, err := http.Get(c.Img)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	_, err = io.Copy(f, resp.Body)
	if err != nil {
		return err
	}
	return nil
}
<commit_msg>Move cache from xkcd-gui to xkcd-gtk.<commit_after>package main

import (
	"encoding/json"
	"github.com/rkoesters/xdg/basedir"
	"github.com/rkoesters/xkcd"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
)

func getComicPath(n int) string {
	return filepath.Join(basedir.CacheHome, "xkcd-gtk", strconv.Itoa(n))
}

func getComicInfoPath(n int) string {
	return filepath.Join(getComicPath(n), "info")
}

func getComicImagePath(n int) string {
	return filepath.Join(getComicPath(n), "image")
}

func getComicInfo(n int) (*xkcd.Comic, error) {
	infoPath := getComicInfoPath(n)

	// First, check if we have the file.
	_, err := os.Stat(infoPath)
	if os.IsNotExist(err) {
		err = downloadComicInfo(n)
		if err != nil {
			return nil, err
		}
	} else if err != nil {
		return nil, err
	}

	f, err := os.Open(infoPath)
	if err != nil {
		return nil, err
	}
	c, err := xkcd.New(f)
	if err != nil {
		return nil, err
	}
	return c, nil
}

func downloadComicInfo(n int) error {
	err := os.MkdirAll(getComicPath(n), 0777)
	if err != nil {
		return err
	}

	comic, err := xkcd.Get(n)
	if err != nil {
		return err
	}

	f, err := os.Create(getComicInfoPath(n))
	if err != nil {
		return err
	}

	e := json.NewEncoder(f)
	err = e.Encode(comic)
	if err != nil {
		return err
	}
	return nil
}

func getComicImage(n int) (string, error) {
	imagePath := getComicImagePath(n)

	_, err := os.Stat(imagePath)
	if os.IsNotExist(err) {
		err = downloadComicImage(n)
		if err != nil {
			return "", err
		}
	} else if err != nil {
		return "", err
	}
	return imagePath, nil
}

func downloadComicImage(n int) error {
	c, err := getComicInfo(n)
	if err != nil {
		return err
	}

	f, err := os.Create(getComicImagePath(n))
	if err != nil {
		return err
	}

	resp, err := http.Get(c.Img)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	_, err = io.Copy(f, resp.Body)
	if err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package libvirt

import (
	"sync"

	"github.com/dorzheh/deployer/deployer"
	ssh "github.com/dorzheh/infra/comm/common"
	"github.com/dorzheh/mxj"
)

type Driver struct {
	sync.Mutex
	run func(string) (string, error)
}

func NewDriver(config *ssh.Config) *Driver {
	d := new(Driver)
	d.run = deployer.RunFunc(config)
	return d
}

// Define is responsible for creating a new domain acording to provided XML template
func (d *Driver) DefineDomain(domainConfig string) error {
	d.Lock()
	defer d.Unlock()
	if _, err := d.run("virsh define " + domainConfig); err != nil {
		return err
	}
	return nil
}

func (d *Driver) StartDomain(name string) error {
	d.Lock()
	defer d.Unlock()
	if _, err := d.run("virsh start " + name); err != nil {
		return err
	}
	return nil
}

func (d *Driver) DestroyDomain(name string) error {
	d.Lock()
	defer d.Unlock()
	if _, err := d.run("virsh destroy " + name); err != nil {
		return err
	}
	return nil
}

func (d *Driver) UndefineDomain(name string) error {
	d.Lock()
	defer d.Unlock()
	if _, err := d.run("virsh undefine " + name); err != nil {
		return err
	}
	return nil
}

// Define is responsible for creating a new domain acording to provided XML template
func (d *Driver) SetAutostart(name string) error {
	d.Lock()
	defer d.Unlock()
	if _, err := d.run("virsh autostart " + name); err != nil {
		return err
	}
	return nil
}

func (d *Driver) DomainExists(name string) bool {
	d.Lock()
	defer d.Unlock()
	if _, err := d.run("virsh dominfo " + name); err != nil {
		return false
	}
	return true
}

func (d *Driver) Emulator() (string, error) {
	d.Lock()
	defer d.Unlock()
	out, err := d.run("virsh capabilities")
	if err != nil {
		return "", err
	}

	m, err := mxj.NewMapXml([]byte(out))
	if err != nil {
		return "", err
	}

	v, _ := m.ValuesForPath("capabilities.guest.arch", "-name:x86_64")
	return v[0].(map[string]interface{})["emulator"].(string), nil
}
<commit_msg>Update driver.go<commit_after>package libvirt

import (
	"sync"

	"github.com/dorzheh/deployer/deployer"
	ssh "github.com/dorzheh/infra/comm/common"
	"github.com/dorzheh/mxj"
)

type Driver struct {
	sync.Mutex
	run func(string) (string, error)
}

func NewDriver(config *ssh.Config) *Driver {
	d := new(Driver)
	d.run = deployer.RunFunc(config)
	return d
}

func (d *Driver) DefineDomain(domainConfig string) error {
	d.Lock()
	defer d.Unlock()
	if _, err := d.run("virsh define " + domainConfig); err != nil {
		return err
	}
	return nil
}

func (d *Driver) StartDomain(name string) error {
	d.Lock()
	defer d.Unlock()
	if _, err := d.run("virsh start " + name); err != nil {
		return err
	}
	return nil
}

func (d *Driver) DestroyDomain(name string) error {
	d.Lock()
	defer d.Unlock()
	if _, err := d.run("virsh destroy " + name); err != nil {
		return err
	}
	return nil
}

func (d *Driver) UndefineDomain(name string) error {
	d.Lock()
	defer d.Unlock()
	if _, err := d.run("virsh undefine " + name); err != nil {
		return err
	}
	return nil
}

// Define is responsible for creating a new domain acording to provided XML template
func (d *Driver) SetAutostart(name string) error {
	d.Lock()
	defer d.Unlock()
	if _, err := d.run("virsh autostart " + name); err != nil {
		return err
	}
	return nil
}

func (d *Driver) DomainExists(name string) bool {
	d.Lock()
	defer d.Unlock()
	if _, err := d.run("virsh dominfo " + name); err != nil {
		return false
	}
	return true
}

func (d *Driver) Emulator() (string, error) {
	d.Lock()
	defer d.Unlock()
	out, err := d.run("virsh capabilities")
	if err != nil {
		return "", err
	}

	m, err := mxj.NewMapXml([]byte(out))
	if err != nil {
		return "", err
	}

	v, _ := m.ValuesForPath("capabilities.guest.arch", "-name:x86_64")
	return v[0].(map[string]interface{})["emulator"].(string), nil
}
<|endoftext|>


<commit_before>package queues
/*
 *  Filename:    priority.go
 *  Package:     queues
 *  Author:      Bryan Matsuo <bmatsuo@soe.ucsc.edu>
 *  Created:     Wed Jul  6 22:18:57 PDT 2011
 *  Description:
 */
import (
    "sort"
    "fmt"
    "container/heap"
    "container/vector"
)

type PrioritizedTask interface {
    Task
    Key() float64
    SetKey(float64)
}
type PTask struct {
    F func(int64)
    P float64
}
func (pt *PTask) Type() string {
    return "PTask"
}
func (pt *PTask) SetFunc(f func(int64)) {
    pt.F = f
}
func (pt *PTask) Func() func(int64) {
    return pt.F
}
func (pt *PTask) Key() float64 {
    return pt.P
}
func (pt *PTask) SetKey(k float64) {
    pt.P = k
}

type pQueue struct {
    elements []RegisteredTask
}
func newPQueue() *pQueue {
    var h = new(pQueue)
    h.elements = make([]RegisteredTask, 0, 5)
    return h
}
func (h *pQueue) GetPTask(i int) PrioritizedTask {
    if n := len(h.elements) ; i < 0 || i >= n {
        panic("badindex")
    }
    return h.elements[i].Task().(PrioritizedTask)
}
func (h *pQueue) Len() int {
    return len(h.elements)
}
func (h *pQueue) Less(i, j int) bool {
    return h.GetPTask(i).Key() < h.GetPTask(j).Key()
}
func (h *pQueue) Swap(i, j int) {
    if n := len(h.elements) ; i < 0 || i >=n || j < 0 || j >= n {
        panic("badindex")
    }
    var tmp = h.elements[i]
    h.elements[i] = h.elements[j]
    h.elements[j] = tmp
}
func (h *pQueue) Push(x interface{}) {
    switch x.(RegisteredTask).Task().(type) {
    case PrioritizedTask:
        h.elements = append(h.elements, x.(RegisteredTask))
    default:
        panic("badtype")
    }
}
func (h *pQueue) Pop() interface{} {
    if len(h.elements) <= 0 {
        panic("empty")
    }
    var head = h.elements[0]
    h.elements = h.elements[1:]
    return head
}
func (h *pQueue) FindId(id int64) (int, RegisteredTask) {
    for i, elm := range h.elements {
        if elm.Id() == id {
            return i, elm
        }
    }
    return -1, nil
}

type PriorityQueue struct {
    h  *pQueue
}

func NewPriorityQueue() *PriorityQueue {
    var pq = new(PriorityQueue)
    pq.h = newPQueue()
    // No need to call heap.Init(pq.h) on an empty heap.
    return pq
}

func (pq *PriorityQueue) Len() int {
    return pq.h.Len()
}
func (pq *PriorityQueue) Dequeue() RegisteredTask {
    if pq.Len() <= 0 {
        panic("empty")
    }
    return heap.Pop(pq.h).(RegisteredTask)
}
func (pq *PriorityQueue) Enqueue(task RegisteredTask) {
    switch task.Task().(type) {
    case PrioritizedTask:
        heap.Push(pq.h, task)
    default:
        panic(fmt.Sprintf("nokey %s", task.Task().Type()))
    }
}
func (pq *PriorityQueue) SetKey(id int64, k float64) {
    var i, task = pq.h.FindId(id)
    if i < 0 {
        return
    }
    heap.Remove(pq.h, i)
    task.Task().(PrioritizedTask).SetKey(k)
    heap.Push(pq.h, task)
}

//  A priority queue based on the "container/vector" package.
//  Ideally, an array-based priority queue implementation should have
//  fast dequeues and slow enqueues. I fear the vector.Vector class
//  gives slow equeues and slow dequeues.
type VectorPriorityQueue struct {
    v *vector.Vector
}

func NewVectorPriorityQueue() *VectorPriorityQueue {
    var vpq = new(VectorPriorityQueue)
    vpq.v = new(vector.Vector)
    return vpq
}

func (vpq *VectorPriorityQueue) Len() int {
    return vpq.v.Len()
}
type etypeStopIter struct {
}
func (e etypeStopIter) String() string {
    return "STOPITER"
}
func (vpq *VectorPriorityQueue) Enqueue(task RegisteredTask) {
    switch task.Task().(type) {
    case PrioritizedTask:
        break
    default:
        panic(fmt.Sprintf("nokey %s", task.Task().Type()))
    }
    var i int
    defer func() {
        if r := recover(); r != nil {
            switch r.(type) {
            case etypeStopIter:
                break
            default:
                panic(r)
            }
        }
        vpq.v.Insert(i, task)
    } ()
    vpq.v.Do(func (telm interface{}) {
        if task.Task().(PrioritizedTask).Key() > telm.(RegisteredTask).Task().(PrioritizedTask).Key() {
            i++
        } else {
            panic(etypeStopIter{})
        }
    })
}
func (vpq *VectorPriorityQueue) Dequeue() RegisteredTask {
    var head = vpq.v.At(0).(RegisteredTask)
    vpq.v.Delete(0)
    return head
}
func (vpq *VectorPriorityQueue) SetKey(id int64, k float64) {
    var i int
    defer func() {
        if r := recover(); r != nil {
            switch r.(type) {
            case etypeStopIter:
                var rtask = vpq.v.At(i).(RegisteredTask)
                vpq.v.Delete(i)
                rtask.Task().(PrioritizedTask).SetKey(k)
                vpq.Enqueue(rtask)
            default:
                panic(r)
            }
        }
    } ()
    vpq.v.Do(func (telm interface{}) {
        if telm.(RegisteredTask).Id() != id {
            i++
        } else {
            panic(etypeStopIter{})
        }
    })
}

type ArrayPriorityQueue struct {
    v          []RegisteredTask
    head, tail int
}

func NewArrayPriorityQueue() *ArrayPriorityQueue {
    var apq = new(ArrayPriorityQueue)
    apq.v = make([]RegisteredTask, 10)
    return apq
}

func (apq *ArrayPriorityQueue) Len() int {
    return apq.tail - apq.head
}

func (apq *ArrayPriorityQueue) Enqueue(task RegisteredTask) {
    var key = task.Task().(PrioritizedTask).Key()
    var n = apq.Len()
    var insertoffset = sort.Search(
            n,
            func(i int)bool{
                if n == 0 || i >= n {
                    return true
                }
                return apq.v[apq.head+i].Task().(PrioritizedTask).Key() >= key } )
    if apq.tail != len(apq.v) {
        for j := apq.tail ; j > apq.head+insertoffset ; j-- {
            apq.v[j] = apq.v[j-1]
        }
        apq.v[apq.head+insertoffset] = task
        apq.tail++
        return
    }
    var newv = apq.v
    if apq.head < len(apq.v)/2 {
        newv = make([]RegisteredTask, 2* len(apq.v))
    }
    copy(newv, apq.v[apq.head:apq.head+insertoffset])
    newv[insertoffset] = task
    copy(newv, apq.v[apq.head+insertoffset:apq.tail])
    for i := apq.head ; i < apq.tail ; i++ {
        apq.v[i] = nil
    }
    apq.v = newv
    apq.head = 0
    apq.tail = n+1
}

func (apq *ArrayPriorityQueue) Dequeue() RegisteredTask {
    if apq.Len() == 0 {
        panic("empty")
    }
    var task = apq.v[apq.head]
    apq.v[apq.head] = nil
    apq.head++
    return task
}

func (apq *ArrayPriorityQueue) SetKey(id int64, k float64) {
}
<commit_msg>Fix nil dereference error.<commit_after>package queues
/*
 *  Filename:    priority.go
 *  Package:     queues
 *  Author:      Bryan Matsuo <bmatsuo@soe.ucsc.edu>
 *  Created:     Wed Jul  6 22:18:57 PDT 2011
 *  Description:
 */
import (
    "sort"
    "fmt"
    "container/heap"
    "container/vector"
)

type PrioritizedTask interface {
    Task
    Key() float64
    SetKey(float64)
}
type PTask struct {
    F func(int64)
    P float64
}
func (pt *PTask) Type() string {
    return "PTask"
}
func (pt *PTask) SetFunc(f func(int64)) {
    pt.F = f
}
func (pt *PTask) Func() func(int64) {
    return pt.F
}
func (pt *PTask) Key() float64 {
    return pt.P
}
func (pt *PTask) SetKey(k float64) {
    pt.P = k
}

type pQueue struct {
    elements []RegisteredTask
}
func newPQueue() *pQueue {
    var h = new(pQueue)
    h.elements = make([]RegisteredTask, 0, 5)
    return h
}
func (h *pQueue) GetPTask(i int) PrioritizedTask {
    if n := len(h.elements) ; i < 0 || i >= n {
        panic("badindex")
    }
    return h.elements[i].Task().(PrioritizedTask)
}
func (h *pQueue) Len() int {
    return len(h.elements)
}
func (h *pQueue) Less(i, j int) bool {
    return h.GetPTask(i).Key() < h.GetPTask(j).Key()
}
func (h *pQueue) Swap(i, j int) {
    if n := len(h.elements) ; i < 0 || i >=n || j < 0 || j >= n {
        panic("badindex")
    }
    var tmp = h.elements[i]
    h.elements[i] = h.elements[j]
    h.elements[j] = tmp
}
func (h *pQueue) Push(x interface{}) {
    switch x.(RegisteredTask).Task().(type) {
    case PrioritizedTask:
        h.elements = append(h.elements, x.(RegisteredTask))
    default:
        panic("badtype")
    }
}
func (h *pQueue) Pop() interface{} {
    if len(h.elements) <= 0 {
        panic("empty")
    }
    var head = h.elements[0]
    h.elements = h.elements[1:]
    return head
}
func (h *pQueue) FindId(id int64) (int, RegisteredTask) {
    for i, elm := range h.elements {
        if elm.Id() == id {
            return i, elm
        }
    }
    return -1, nil
}

type PriorityQueue struct {
    h  *pQueue
}

func NewPriorityQueue() *PriorityQueue {
    var pq = new(PriorityQueue)
    pq.h = newPQueue()
    // No need to call heap.Init(pq.h) on an empty heap.
    return pq
}

func (pq *PriorityQueue) Len() int {
    return pq.h.Len()
}
func (pq *PriorityQueue) Dequeue() RegisteredTask {
    if pq.Len() <= 0 {
        panic("empty")
    }
    return heap.Pop(pq.h).(RegisteredTask)
}
func (pq *PriorityQueue) Enqueue(task RegisteredTask) {
    switch task.Task().(type) {
    case PrioritizedTask:
        heap.Push(pq.h, task)
    default:
        panic(fmt.Sprintf("nokey %s", task.Task().Type()))
    }
}
func (pq *PriorityQueue) SetKey(id int64, k float64) {
    var i, task = pq.h.FindId(id)
    if i < 0 {
        return
    }
    heap.Remove(pq.h, i)
    task.Task().(PrioritizedTask).SetKey(k)
    heap.Push(pq.h, task)
}

//  A priority queue based on the "container/vector" package.
//  Ideally, an array-based priority queue implementation should have
//  fast dequeues and slow enqueues. I fear the vector.Vector class
//  gives slow equeues and slow dequeues.
type VectorPriorityQueue struct {
    v *vector.Vector
}

func NewVectorPriorityQueue() *VectorPriorityQueue {
    var vpq = new(VectorPriorityQueue)
    vpq.v = new(vector.Vector)
    return vpq
}

func (vpq *VectorPriorityQueue) Len() int {
    return vpq.v.Len()
}
type etypeStopIter struct {
}
func (e etypeStopIter) String() string {
    return "STOPITER"
}
func (vpq *VectorPriorityQueue) Enqueue(task RegisteredTask) {
    switch task.Task().(type) {
    case PrioritizedTask:
        break
    default:
        panic(fmt.Sprintf("nokey %s", task.Task().Type()))
    }
    var i int
    defer func() {
        if r := recover(); r != nil {
            switch r.(type) {
            case etypeStopIter:
                break
            default:
                panic(r)
            }
        }
        vpq.v.Insert(i, task)
    } ()
    vpq.v.Do(func (telm interface{}) {
        if task.Task().(PrioritizedTask).Key() > telm.(RegisteredTask).Task().(PrioritizedTask).Key() {
            i++
        } else {
            panic(etypeStopIter{})
        }
    })
}
func (vpq *VectorPriorityQueue) Dequeue() RegisteredTask {
    var head = vpq.v.At(0).(RegisteredTask)
    vpq.v.Delete(0)
    return head
}
func (vpq *VectorPriorityQueue) SetKey(id int64, k float64) {
    var i int
    defer func() {
        if r := recover(); r != nil {
            switch r.(type) {
            case etypeStopIter:
                var rtask = vpq.v.At(i).(RegisteredTask)
                vpq.v.Delete(i)
                rtask.Task().(PrioritizedTask).SetKey(k)
                vpq.Enqueue(rtask)
            default:
                panic(r)
            }
        }
    } ()
    vpq.v.Do(func (telm interface{}) {
        if telm.(RegisteredTask).Id() != id {
            i++
        } else {
            panic(etypeStopIter{})
        }
    })
}

type ArrayPriorityQueue struct {
    v          []RegisteredTask
    head, tail int
}

func NewArrayPriorityQueue() *ArrayPriorityQueue {
    var apq = new(ArrayPriorityQueue)
    apq.v = make([]RegisteredTask, 10)
    return apq
}

func (apq *ArrayPriorityQueue) Len() int {
    return apq.tail - apq.head
}

func (apq *ArrayPriorityQueue) Enqueue(task RegisteredTask) {
    var key = task.Task().(PrioritizedTask).Key()
    var n = apq.Len()
    var insertoffset = sort.Search(
            n,
            func(i int)bool{
                if n == 0 || i >= n {
                    return true
                }
                return apq.v[apq.head+i].Task().(PrioritizedTask).Key() >= key } )
    if apq.tail != len(apq.v) {
        for j := apq.tail ; j > apq.head+insertoffset ; j-- {
            apq.v[j] = apq.v[j-1]
        }
        apq.v[apq.head+insertoffset] = task
        apq.tail++
        return
    }
    var newv = apq.v
    if apq.head <= len(apq.v)/2 {
        newv = make([]RegisteredTask, 2* len(apq.v))
    }
    copy(newv, apq.v[apq.head:apq.head+insertoffset])
    newv[insertoffset] = task
    copy(newv[insertoffset+1:], apq.v[apq.head+insertoffset:apq.tail])
    for i := apq.head ; i < apq.tail ; i++ {
        apq.v[i] = nil
    }
    apq.v = newv
    apq.head = 0
    apq.tail = n+1
}

func (apq *ArrayPriorityQueue) Dequeue() RegisteredTask {
    if apq.Len() == 0 {
        panic("empty")
    }
    var task = apq.v[apq.head]
    apq.v[apq.head] = nil
    apq.head++
    return task
}

func (apq *ArrayPriorityQueue) SetKey(id int64, k float64) {
}
<|endoftext|>


<commit_before>// Comparing the speeds of the golang native regex library and rubex.
// The numbers show a dramatic difference, with rubex being nearly 400
// times slower than the native go libraries.  Unfortunately for us,
// the native go libraries have a different regex behavior than rubex,
// so we'll have to hack at it a bit to fit our needs if we decide to use it.
// (which we should, I mean, come on, 400 times faster?  That's mad wins.)

package main

import re "rubex"
import "time"
import "regexp"
import "runtime"
import "os"
import "strconv"

var re1 []*regexp.Regexp
var re2 []*re.Regexp
const NUM = 100
var STR = "abcdabc"

func init() {
	re1 = make([]*regexp.Regexp, NUM)
	re2 = make([]*re.Regexp, NUM)
	for i := 0; i < NUM; i ++ {
		re1[i] = regexp.MustCompile("[a-c]*$")
		re2[i] = re.MustCompile("[a-c]*$")
	}
	for i := 0; i < 10; i ++ {
		STR += STR
	}
	println("len:", len(STR))
}

func rubex_page(index int) {
	for i := 0; i < 100; i++ {
		r := re2[index]
		r.MatchString(STR)
	}
}

func regexp_page(index int) {
	for i := 0; i < 100; i++ {
		r := re1[index]
		r.MatchString(STR)
	}
}

func render_pages(name string, fp  func(int), num_routines, num_renders int) {
	for i := 0; i < num_routines; i++ {
		go func () {
			t := time.Now()
			for j := 0; j < num_renders; j++ {
				fp(i)
			}
			println(name + "-average: ",  time.Since(t).Nanoseconds()/int64(num_renders*1000000), "ms")
		}()
	}
}

func main() {
	cpu, _ := strconv.Atoi(os.Args[1])
	lib := os.Args[2]
	println("using CPUs:", cpu)
	runtime.GOMAXPROCS(cpu)
	num_routines := 90
	num_renders := 20

	if lib == "rubex" {
		render_pages("rubex", rubex_page, num_routines, num_renders)
	} else {
		render_pages("regexp", regexp_page, num_routines, num_renders)
	}

	d, _ := time.ParseDuration("5s")
	for i := 0; i < 100; i ++ {
		println("goroutine:", runtime.NumGoroutine())
		time.Sleep(d)

	}
	println ("Done")
}

<commit_msg>more updates<commit_after>// Comparing the speeds of the golang native regex library and rubex.
// The numbers show a dramatic difference, with rubex being nearly 400
// times slower than the native go libraries.  Unfortunately for us,
// the native go libraries have a different regex behavior than rubex,
// so we'll have to hack at it a bit to fit our needs if we decide to use it.
// (which we should, I mean, come on, 400 times faster?  That's mad wins.)

package main

import re "rubex"
import "time"
import "regexp"
import "runtime"
import "os"
import "strconv"

var re1 []Matcher
var re2 []Matcher
const NUM = 100
const NNN = 1000
var STR = "abcdabc"

type Matcher interface {
	MatchString(string) bool
}

type Task struct {
	str string
	m Matcher
}

var TaskChann chan *Task

func init() {
	re1 = make([]Matcher, NUM)
	re2 = make([]Matcher, NUM)
	for i := 0; i < NUM; i ++ {
		re1[i] = regexp.MustCompile("[a-c]*$")
		re2[i] = re.MustCompile("[a-c]*$")
	}
	TaskChann = make(chan *Task, 100)
	for i := 0; i < 10; i ++ {
		STR += STR
	}
	println("len:", len(STR))
}

func render_pages(name string, marray []Matcher, num_routines, num_renders int) {
	for i := 0; i < num_routines; i++ {
		m := marray[i]
		go func () {
			runtime.LockOSThread()
			for j := 0; j < num_renders; j++ {
				t := time.Now()
				for i := 0; i < NNN; i++ {
        			        m.MatchString(STR)
			        }
				println(name + "-average: ",  time.Since(t).Nanoseconds()/int64(1000*NNN), "us")
			}
		}()
	}
}

func render_pages2(name string, marray []Matcher, num_routines, num_renders int) {
	go func() {
		for i := 0; i < 100000; i ++ {
			t := &Task{str: STR, m: marray[0]}
			TaskChann <- t
		}
	}()
	for i := 0; i < num_routines; i++ {
		m := marray[i]
		go func () {
			runtime.LockOSThread()
			for j := 0; j < num_renders; j++ {
				t := time.Now()
				for i := 0; i < NNN; i++ {
					task := <- TaskChann
         				m.MatchString(task.str)
        			}
				println(name + "-average: ",  time.Since(t).Nanoseconds()/int64(1000*NNN), "us")
			}
		}()
	}
}

func main() {
	cpu, _ := strconv.Atoi(os.Args[1])
	lib := os.Args[2]
	println("using CPUs:", cpu)
	runtime.GOMAXPROCS(cpu)
	num_routines := 2
	num_renders := 20

	if lib == "rubex" {
		render_pages2("rubex", re2, num_routines, num_renders)
	} else {
		render_pages2("regexp", re1, num_routines, num_renders)
	}

	d, _ := time.ParseDuration("5s")
	for i := 0; i < 100; i ++ {
		println("goroutine:", runtime.NumGoroutine())
		time.Sleep(d)

	}
	println ("Done")
}

<|endoftext|>


<commit_before>package model

// transfer2go data model module
// Copyright (c) 2017 - Valentin Kuznetsov <vkuznet@gmail.com>

import (
	"database/sql"
	"fmt"
	"io/ioutil"
	"log"
	"os/exec"
	"strings"

	"github.com/vkuznet/transfer2go/utils"

	// loads sqlite3 database layer
	_ "github.com/mattn/go-sqlite3"
)

// main DB record we work with
type Record map[string]interface{}

// DB is global pointer to sql database object, it is initialized once when server starts
var DB *sql.DB
var DBTYPE string
var DBSQL Record

func check(msg string, err error) {
	if err != nil {
		log.Fatalf("ERROR %s, %v\n", msg, err)
	}
}

// helper function to load DBS SQL statements
func LoadSQL(owner string) Record {
	dbsql := make(Record)
	// query statement
	tmplData := make(Record)
	tmplData["Owner"] = owner
	sdir := fmt.Sprintf("%s/sql", utils.STATICDIR)
	for _, f := range utils.ListFiles(sdir) {
		k := strings.Split(f, ".")[0]
		dbsql[k] = utils.ParseTmpl(sdir, f, tmplData)
	}
	return dbsql
}

// helper function to get SQL statement from DBSQL dict for a given key
func getSQL(key string) string {
	// use generic query API to fetch the results from DB
	stm, ok := DBSQL[key]
	if !ok {
		msg := fmt.Sprintf("Unable to load %s SQL", key)
		log.Fatal(msg)
	}
	return stm.(string)
}

// helper function to assign placeholder for SQL WHERE clause, it depends on database type
func placeholder(pholder string) string {
	if DBTYPE == "ora" || DBTYPE == "oci8" {
		return fmt.Sprintf(":%s", pholder)
	} else if DBTYPE == "PostgreSQL" {
		return fmt.Sprintf("$%s", pholder)
	} else {
		return "?"
	}
}

// CatalogEntry represents an entry in TFC
type CatalogEntry struct {
	Lfn     string `json:"lfn"`     // lfn stands for Logical File Name
	Pfn     string `json:"pfn"`     // pfn stands for Physical File Name
	Dataset string `json:"dataset"` // dataset represents collection of blocks
	Block   string `json:"block"`   // block idetify single block within a dataset
	Bytes   int64  `json:"bytes"`   // size of the files in bytes
	Hash    string `json:"hash"`    // hash represents checksum of the pfn
}

// Catalog represents Trivial File Catalog (TFC) of the model
type Catalog struct {
	Type     string `json:"type"`     // catalog type, e.g. filesystem, sqlite3, etc.
	Uri      string `json:"uri"`      // catalog uri, e.g. file.db
	Login    string `json:"login"`    // database login
	Password string `json:"password"` // database password
	Owner    string `json:"owner"`    // used by ORACLE DB, defines owner of the database
}

// Dump method returns TFC dump in CSV format
func (c *Catalog) Dump() []byte {
	if c.Type == "sqlite3" {
		//         cmd := fmt.Sprintf("sqlite3 %s .dump", c.Uri)
		out, err := exec.Command("sqlite3", c.Uri, ".dump").Output()
		if err != nil {
			log.Println("ERROR c.Dump", err)
		}
		return out
	}
	log.Println("Catalog Dump method is not implemented yet for", c.Type)
	return nil

}

// Add method adds entry to a catalog
func (c *Catalog) Add(entry CatalogEntry) error {

	// add entry to the catalog
	tx, e := DB.Begin()
	check("Unable to setup transaction", e)

	var stm string
	var did, bid int

	// insert dataset into dataset tables
	stm = fmt.Sprintf("INSERT INTO DATASETS(dataset) VALUES(?)")
	_, e = DB.Exec(stm, entry.Dataset)
	if e != nil {
		if !strings.Contains(e.Error(), "UNIQUE") {
			check("Unable to insert into datasets table", e)
		}
	}

	// get dataset id
	stm = fmt.Sprintf("SELECT id FROM DATASETS WHERE dataset=?")
	rows, err := DB.Query(stm, entry.Dataset)
	check("Unable to perform DB.Query over datasets table", err)
	defer rows.Close()
	for rows.Next() {
		err = rows.Scan(&did)
		check("Unable to scan rows for datasetid", err)
	}

	// insert block into block table
	stm = fmt.Sprintf("INSERT INTO BLOCKS(block) VALUES(?)")
	_, e = DB.Exec(stm, entry.Block)
	if e != nil {
		if !strings.Contains(e.Error(), "UNIQUE") {
			check("Unable to insert into blocks table", e)
		}
	}

	// get block id
	stm = fmt.Sprintf("SELECT id FROM BLOCKS WHERE block=?")
	rows, err = DB.Query(stm, entry.Block)
	check("Unable to DB.Query over blocks table", err)
	for rows.Next() {
		err = rows.Scan(&bid)
		check("Unable to scan rows for datasetid", err)
	}

	// insert entry into files table
	stm = fmt.Sprintf("INSERT INTO FILES(lfn, pfn, blockid, datasetid, bytes, hash) VALUES(?,?,?,?,?,?)")
	_, err = DB.Exec(stm, entry.Lfn, entry.Pfn, bid, did, entry.Bytes, entry.Hash)
	if e != nil {
		if !strings.Contains(e.Error(), "UNIQUE") {
			check(fmt.Sprintf("Unable to DB.Exec(%s)", stm), err)
		}
	}

	tx.Commit()

	if utils.VERBOSE > 0 {
		log.Println("Committed to Catalog", entry, "datasetid", did, "blockid", bid)
	}

	return nil
}

// Files method of catalog returns list of files known in catalog
func (c *Catalog) Files(pattern string) []string {
	var files []string
	if c.Type == "filesystem" {
		filesInfo, err := ioutil.ReadDir(c.Uri)
		if err != nil {
			log.Println("ERROR: unable to list files in catalog", c.Uri, err)
			return []string{}
		}
		for _, f := range filesInfo {
			if pattern != "" {
				if strings.Contains(f.Name(), pattern) {
					files = append(files, fmt.Sprintf("%s/%s", c.Uri, f.Name()))
				}
			} else {
				files = append(files, fmt.Sprintf("%s/%s", c.Uri, f.Name()))
			}
		}
		return files
	} else {
		// TODO: make generic database statements via templates for given database type
		//         cols := []string{"dataset", "blockid", "lfn", "pfn", "bytes", "hash"}
		//         stm := fmt.Sprintf("SELECT %s FROM FILES AS F JOIN BLOCKS AS B ON F.BLOCKID=B.ID JOIN DATASETS AS D ON F.DATASETID = D.ID WHERE F.LFN=?", strings.Join(cols, ","))
		stm := getSQL("files_blocks_datasets") + fmt.Sprintf(" WHERE F.LFN=%s", placeholder("lfn"))
		if utils.VERBOSE > 0 {
			log.Println("Files query", stm, pattern)
		}
		vals := []interface{}{new(sql.NullString), new(sql.NullString), new(sql.NullString), new(sql.NullString), new(sql.NullInt64), new(sql.NullString)}

		// fetch data from DB
		rows, err := DB.Query(stm, pattern)
		if err != nil {
			log.Printf("ERROR DB.Query, query='%s' error=%v\n", stm, err)
			return files
		}
		defer rows.Close()
		for rows.Next() {
			rec := CatalogEntry{}
			err := rows.Scan(&rec.Dataset, &rec.Block, &rec.Lfn, &rec.Pfn, &rec.Bytes, &rec.Hash)
			if err != nil {
				msg := fmt.Sprintf("ERROR rows.Scan, vals='%v', error=%v", vals, err)
				log.Fatal(msg)
			}
			files = append(files, rec.Lfn)
		}
		return files
	}
	return files
}

// FileInfo provides information about given file name in Catalog
func (c *Catalog) FileInfo(fileEntry string) CatalogEntry {
	if c.Type == "filesystem" {
		fname := fileEntry
		data, err := ioutil.ReadFile(fname)
		if err != nil {
			log.Println("ERROR, unable to read a file", fname, err)
		}
		hash, b := utils.Hash(data)
		// TODO: I need to know how to generate dataset and block names in this case
		entry := CatalogEntry{Lfn: fname, Pfn: fname, Hash: hash, Bytes: b, Dataset: "/a/b/c", Block: "123"}
		return entry
	} else {
		// TODO: make generic SQL statement via templates
		//         cols := []string{"lfn", "pfn", "bytes", "hash"}
		//         stm := fmt.Sprintf("SELECT %s FROM FILES AS F WHERE LFN = ?", strings.Join(cols, ","))
		stm := getSQL("files") + fmt.Sprintf(" WHERE F.LFN=%s", placeholder("lfn"))
		if utils.VERBOSE > 0 {
			log.Println("FileInfo query", stm, fileEntry)
		}

		// fetch data from DB
		rows, err := DB.Query(stm, fileEntry)
		if err != nil {
			log.Printf("ERROR DB.Query, query='%s' error=%v\n", stm, err)
			return CatalogEntry{}
		}
		defer rows.Close()
		for rows.Next() {
			rec := CatalogEntry{}
			err := rows.Scan(&rec.Lfn, &rec.Pfn, &rec.Bytes, &rec.Hash)
			if err != nil {
				log.Println("ERROR rows.Scan", err)
				return CatalogEntry{}
			}
			return rec
		}
	}
	return CatalogEntry{}
}

// TFC stands for Trivial File Catalog
var TFC Catalog
<commit_msg>fixes based on golint suggestions<commit_after>package model

// transfer2go data model module
// Copyright (c) 2017 - Valentin Kuznetsov <vkuznet@gmail.com>

import (
	"database/sql"
	"fmt"
	"io/ioutil"
	"log"
	"os/exec"
	"strings"

	"github.com/vkuznet/transfer2go/utils"

	// loads sqlite3 database layer
	_ "github.com/mattn/go-sqlite3"
)

// Record represent main DB record we work with
type Record map[string]interface{}

// DB is global pointer to sql database object, it is initialized once when server starts
var DB *sql.DB

// DBTYPE holds database type, e.g. sqlite3
var DBTYPE string

// DBSQL represent common record we get from DB SQL statement
var DBSQL Record

func check(msg string, err error) {
	if err != nil {
		log.Fatalf("ERROR %s, %v\n", msg, err)
	}
}

// LoadSQL is a helper function to load DBS SQL statements
func LoadSQL(owner string) Record {
	dbsql := make(Record)
	// query statement
	tmplData := make(Record)
	tmplData["Owner"] = owner
	sdir := fmt.Sprintf("%s/sql", utils.STATICDIR)
	for _, f := range utils.ListFiles(sdir) {
		k := strings.Split(f, ".")[0]
		dbsql[k] = utils.ParseTmpl(sdir, f, tmplData)
	}
	return dbsql
}

// helper function to get SQL statement from DBSQL dict for a given key
func getSQL(key string) string {
	// use generic query API to fetch the results from DB
	stm, ok := DBSQL[key]
	if !ok {
		msg := fmt.Sprintf("Unable to load %s SQL", key)
		log.Fatal(msg)
	}
	return stm.(string)
}

// helper function to assign placeholder for SQL WHERE clause, it depends on database type
func placeholder(pholder string) string {
	if DBTYPE == "ora" || DBTYPE == "oci8" {
		return fmt.Sprintf(":%s", pholder)
	} else if DBTYPE == "PostgreSQL" {
		return fmt.Sprintf("$%s", pholder)
	} else {
		return "?"
	}
}

// CatalogEntry represents an entry in TFC
type CatalogEntry struct {
	Lfn     string `json:"lfn"`     // lfn stands for Logical File Name
	Pfn     string `json:"pfn"`     // pfn stands for Physical File Name
	Dataset string `json:"dataset"` // dataset represents collection of blocks
	Block   string `json:"block"`   // block idetify single block within a dataset
	Bytes   int64  `json:"bytes"`   // size of the files in bytes
	Hash    string `json:"hash"`    // hash represents checksum of the pfn
}

// Catalog represents Trivial File Catalog (TFC) of the model
type Catalog struct {
	Type     string `json:"type"`     // catalog type, e.g. filesystem, sqlite3, etc.
	Uri      string `json:"uri"`      // catalog uri, e.g. file.db
	Login    string `json:"login"`    // database login
	Password string `json:"password"` // database password
	Owner    string `json:"owner"`    // used by ORACLE DB, defines owner of the database
}

// Dump method returns TFC dump in CSV format
func (c *Catalog) Dump() []byte {
	if c.Type == "sqlite3" {
		//         cmd := fmt.Sprintf("sqlite3 %s .dump", c.Uri)
		out, err := exec.Command("sqlite3", c.Uri, ".dump").Output()
		if err != nil {
			log.Println("ERROR c.Dump", err)
		}
		return out
	}
	log.Println("Catalog Dump method is not implemented yet for", c.Type)
	return nil

}

// Add method adds entry to a catalog
func (c *Catalog) Add(entry CatalogEntry) error {

	// add entry to the catalog
	tx, e := DB.Begin()
	check("Unable to setup transaction", e)

	var stm string
	var did, bid int

	// insert dataset into dataset tables
	stm = fmt.Sprintf("INSERT INTO DATASETS(dataset) VALUES(?)")
	_, e = DB.Exec(stm, entry.Dataset)
	if e != nil {
		if !strings.Contains(e.Error(), "UNIQUE") {
			check("Unable to insert into datasets table", e)
		}
	}

	// get dataset id
	stm = fmt.Sprintf("SELECT id FROM DATASETS WHERE dataset=?")
	rows, err := DB.Query(stm, entry.Dataset)
	check("Unable to perform DB.Query over datasets table", err)
	defer rows.Close()
	for rows.Next() {
		err = rows.Scan(&did)
		check("Unable to scan rows for datasetid", err)
	}

	// insert block into block table
	stm = fmt.Sprintf("INSERT INTO BLOCKS(block) VALUES(?)")
	_, e = DB.Exec(stm, entry.Block)
	if e != nil {
		if !strings.Contains(e.Error(), "UNIQUE") {
			check("Unable to insert into blocks table", e)
		}
	}

	// get block id
	stm = fmt.Sprintf("SELECT id FROM BLOCKS WHERE block=?")
	rows, err = DB.Query(stm, entry.Block)
	check("Unable to DB.Query over blocks table", err)
	for rows.Next() {
		err = rows.Scan(&bid)
		check("Unable to scan rows for datasetid", err)
	}

	// insert entry into files table
	stm = fmt.Sprintf("INSERT INTO FILES(lfn, pfn, blockid, datasetid, bytes, hash) VALUES(?,?,?,?,?,?)")
	_, err = DB.Exec(stm, entry.Lfn, entry.Pfn, bid, did, entry.Bytes, entry.Hash)
	if e != nil {
		if !strings.Contains(e.Error(), "UNIQUE") {
			check(fmt.Sprintf("Unable to DB.Exec(%s)", stm), err)
		}
	}

	tx.Commit()

	if utils.VERBOSE > 0 {
		log.Println("Committed to Catalog", entry, "datasetid", did, "blockid", bid)
	}

	return nil
}

// Files method of catalog returns list of files known in catalog
func (c *Catalog) Files(pattern string) []string {
	var files []string
	if c.Type == "filesystem" {
		filesInfo, err := ioutil.ReadDir(c.Uri)
		if err != nil {
			log.Println("ERROR: unable to list files in catalog", c.Uri, err)
			return []string{}
		}
		for _, f := range filesInfo {
			if pattern != "" {
				if strings.Contains(f.Name(), pattern) {
					files = append(files, fmt.Sprintf("%s/%s", c.Uri, f.Name()))
				}
			} else {
				files = append(files, fmt.Sprintf("%s/%s", c.Uri, f.Name()))
			}
		}
		return files
	}
	// TODO: make generic database statements via templates for given database type
	//         cols := []string{"dataset", "blockid", "lfn", "pfn", "bytes", "hash"}
	//         stm := fmt.Sprintf("SELECT %s FROM FILES AS F JOIN BLOCKS AS B ON F.BLOCKID=B.ID JOIN DATASETS AS D ON F.DATASETID = D.ID WHERE F.LFN=?", strings.Join(cols, ","))
	stm := getSQL("files_blocks_datasets") + fmt.Sprintf(" WHERE F.LFN=%s", placeholder("lfn"))
	if utils.VERBOSE > 0 {
		log.Println("Files query", stm, pattern)
	}
	vals := []interface{}{new(sql.NullString), new(sql.NullString), new(sql.NullString), new(sql.NullString), new(sql.NullInt64), new(sql.NullString)}

	// fetch data from DB
	rows, err := DB.Query(stm, pattern)
	if err != nil {
		log.Printf("ERROR DB.Query, query='%s' error=%v\n", stm, err)
		return files
	}
	defer rows.Close()
	for rows.Next() {
		rec := CatalogEntry{}
		err := rows.Scan(&rec.Dataset, &rec.Block, &rec.Lfn, &rec.Pfn, &rec.Bytes, &rec.Hash)
		if err != nil {
			msg := fmt.Sprintf("ERROR rows.Scan, vals='%v', error=%v", vals, err)
			log.Fatal(msg)
		}
		files = append(files, rec.Lfn)
	}
	return files
}

// FileInfo provides information about given file name in Catalog
func (c *Catalog) FileInfo(fileEntry string) CatalogEntry {
	if c.Type == "filesystem" {
		fname := fileEntry
		data, err := ioutil.ReadFile(fname)
		if err != nil {
			log.Println("ERROR, unable to read a file", fname, err)
		}
		hash, b := utils.Hash(data)
		// TODO: I need to know how to generate dataset and block names in this case
		entry := CatalogEntry{Lfn: fname, Pfn: fname, Hash: hash, Bytes: b, Dataset: "/a/b/c", Block: "123"}
		return entry
	}
	// TODO: make generic SQL statement via templates
	//         cols := []string{"lfn", "pfn", "bytes", "hash"}
	//         stm := fmt.Sprintf("SELECT %s FROM FILES AS F WHERE LFN = ?", strings.Join(cols, ","))
	stm := getSQL("files") + fmt.Sprintf(" WHERE F.LFN=%s", placeholder("lfn"))
	if utils.VERBOSE > 0 {
		log.Println("FileInfo query", stm, fileEntry)
	}

	// fetch data from DB
	rows, err := DB.Query(stm, fileEntry)
	if err != nil {
		log.Printf("ERROR DB.Query, query='%s' error=%v\n", stm, err)
		return CatalogEntry{}
	}
	defer rows.Close()
	for rows.Next() {
		rec := CatalogEntry{}
		err := rows.Scan(&rec.Lfn, &rec.Pfn, &rec.Bytes, &rec.Hash)
		if err != nil {
			log.Println("ERROR rows.Scan", err)
			return CatalogEntry{}
		}
		return rec
	}
	return CatalogEntry{}
}

// TFC stands for Trivial File Catalog
var TFC Catalog
<|endoftext|>