<commit_before>package input

import (
	"fmt"
	"time"

	"github.com/funkygao/dbus/engine"
	"github.com/funkygao/dbus/pkg/myslave"
	conf "github.com/funkygao/jsconf"
	log "github.com/funkygao/log4go"
)

type MysqlbinlogInput struct {
	stopChan chan struct{}

	slave *myslave.MySlave
}

func (this *MysqlbinlogInput) Init(config *conf.Conf) {
	this.stopChan = make(chan struct{})
	this.slave = myslave.New().LoadConfig(config)

	// so that KafkaOutput can reuse
	key := fmt.Sprintf("myslave.%s", config.String("name", ""))
	engine.Globals().Register(key, this.slave)
}

func (this *MysqlbinlogInput) Stop() {
	log.Trace("stopping...")
	close(this.stopChan)
	this.slave.StopReplication()
}

func (this *MysqlbinlogInput) Run(r engine.InputRunner, h engine.PluginHelper) error {
	backoff := time.Second * 5
	for {
	RESTART_REPLICATION:

		log.Info("starting replication")

		ready := make(chan struct{})
		go this.slave.StartReplication(ready)
		select {
		case <-ready:
		case <-this.stopChan:
			log.Trace("yes sir!")
			return nil
		}

		rows := this.slave.EventStream()
		errors := this.slave.Errors()
		for {
			select {
			case <-this.stopChan:
				log.Trace("yes sir!")
				return nil

			case err := <-errors:
				// e,g.
				// ERROR 1236 (HY000): Could not find first log file name in binary log index file
				// ERROR 1236 (HY000): Could not open log file
				log.Error("backoff %s: %v", backoff, err)
				this.slave.StopReplication()

				select {
				case <-time.After(backoff):
				case <-this.stopChan:
					log.Trace("yes sir!")
					return nil
				}
				goto RESTART_REPLICATION

			case pack, ok := <-r.InChan():
				if !ok {
					log.Trace("yes sir!")
					return nil
				}

				select {
				case err := <-errors:
					// TODO is this neccessary?
					log.Error("backoff %s: %v", backoff, err)
					this.slave.StopReplication()

					select {
					case <-time.After(backoff):
					case <-this.stopChan:
						log.Trace("yes sir!")
						return nil
					}
					goto RESTART_REPLICATION

				case row, ok := <-rows:
					if !ok {
						log.Info("event stream closed")
						return nil
					}

					pack.Payload = row
					r.Inject(pack)

				case <-this.stopChan:
					log.Trace("yes sir!")
					return nil
				}
			}
		}
	}

	return nil
}

func init() {
	engine.RegisterPlugin("MysqlbinlogInput", func() engine.Plugin {
		return new(MysqlbinlogInput)
	})
}
<commit_msg>add identification for logger<commit_after>package input

import (
	"fmt"
	"time"

	"github.com/funkygao/dbus/engine"
	"github.com/funkygao/dbus/pkg/myslave"
	conf "github.com/funkygao/jsconf"
	log "github.com/funkygao/log4go"
)

type MysqlbinlogInput struct {
	stopChan chan struct{}

	slave *myslave.MySlave
}

func (this *MysqlbinlogInput) Init(config *conf.Conf) {
	this.stopChan = make(chan struct{})
	this.slave = myslave.New().LoadConfig(config)

	// so that KafkaOutput can reuse
	key := fmt.Sprintf("myslave.%s", config.String("name", ""))
	engine.Globals().Register(key, this.slave)
}

func (this *MysqlbinlogInput) Stop() {
	log.Trace("stopping...")
	close(this.stopChan)
	this.slave.StopReplication()
}

func (this *MysqlbinlogInput) Run(r engine.InputRunner, h engine.PluginHelper) error {
	name := r.Name()
	backoff := time.Second * 5

	for {
	RESTART_REPLICATION:

		log.Info("[%s] starting replication", name)

		ready := make(chan struct{})
		go this.slave.StartReplication(ready)
		select {
		case <-ready:
		case <-this.stopChan:
			log.Trace("[%s] yes sir!", name)
			return nil
		}

		rows := this.slave.EventStream()
		errors := this.slave.Errors()
		for {
			select {
			case <-this.stopChan:
				log.Trace("[%s] yes sir!", name)
				return nil

			case err := <-errors:
				// e,g.
				// ERROR 1236 (HY000): Could not find first log file name in binary log index file
				// ERROR 1236 (HY000): Could not open log file
				log.Error("[%s] backoff %s: %v", name, backoff, err)
				this.slave.StopReplication()

				select {
				case <-time.After(backoff):
				case <-this.stopChan:
					log.Trace("[%s] yes sir!", name)
					return nil
				}
				goto RESTART_REPLICATION

			case pack, ok := <-r.InChan():
				if !ok {
					log.Trace("[%s] yes sir!", name)
					return nil
				}

				select {
				case err := <-errors:
					// TODO is this neccessary?
					log.Error("[%s] backoff %s: %v", name, backoff, err)
					this.slave.StopReplication()

					select {
					case <-time.After(backoff):
					case <-this.stopChan:
						log.Trace("[%s] yes sir!", name)
						return nil
					}
					goto RESTART_REPLICATION

				case row, ok := <-rows:
					if !ok {
						log.Info("[%s] event stream closed", name)
						return nil
					}

					pack.Payload = row
					r.Inject(pack)

				case <-this.stopChan:
					log.Trace("[%s] yes sir!", name)
					return nil
				}
			}
		}
	}

	return nil
}

func init() {
	engine.RegisterPlugin("MysqlbinlogInput", func() engine.Plugin {
		return new(MysqlbinlogInput)
	})
}
<|endoftext|>


<commit_before>// Copyright 2022 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0

package krusty_test

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"sigs.k8s.io/kustomize/api/krusty"
	"sigs.k8s.io/kustomize/api/loader"
	"sigs.k8s.io/kustomize/api/resmap"
	"sigs.k8s.io/kustomize/kyaml/filesys"
	"sigs.k8s.io/kustomize/kyaml/yaml"
)

func TestRemoteLoad_LocalProtocol(t *testing.T) {
	type testRepos struct {
		root          string
		simple        string
		noSuffix      string
		multiBaseDev  string
		withSubmodule string
	}

	// creates git repos under a root temporary directory with the following structure
	// root/
	//   simple.git/			- base with just a pod
	//   nosuffix/				- same as simple.git/ without the .git suffix
	//   multibase.git/			- base with a dev overlay
	//   with-submodule.git/	- includes `simple` as a submodule
	//     submodule/			- the submodule referencing `simple
	createGitRepos := func(t *testing.T) testRepos {
		t.Helper()

		bash := func(script string) {
			cmd := exec.Command("sh", "-c", script)
			o, err := cmd.CombinedOutput()
			if err != nil {
				t.Fatalf("error running %v\nerr: %v\n%s", script, err, string(o))
			}
		}
		root := t.TempDir()
		bash(fmt.Sprintf(`
set -eux

export ROOT="%s"
export GIT_AUTHOR_EMAIL=nobody@kustomize.io
export GIT_AUTHOR_NAME=Nobody
export GIT_COMMITTER_EMAIL=nobody@kustomize.io
export GIT_COMMITTER_NAME=Nobody

cp -r testdata/remoteload/simple $ROOT/simple.git
(
	cd $ROOT/simple.git
	git init --initial-branch=main
	git add .
	git commit -m "import"
	git checkout -b change-image
	cat >>kustomization.yaml <<EOF

images:
- name: nginx
  newName: nginx
  newTag: "2"
EOF
	git commit -am "image change"
	git checkout main
)
cp -r $ROOT/simple.git $ROOT/nosuffix
cp -r testdata/remoteload/multibase $ROOT/multibase.git
(
	cd $ROOT/multibase.git
	git init --initial-branch=main
	git add .
	git commit -m "import"
)
(
	mkdir $ROOT/with-submodule.git
	cd $ROOT/with-submodule.git
	git init --initial-branch=main
	git submodule add $ROOT/simple.git submodule
	git commit -m "import"
)
`, root))
		return testRepos{
			root: root,
			// The strings below aren't currently used, and more serve as documentation.
			simple:        "simple.git",
			noSuffix:      "nosuffix",
			multiBaseDev:  "multibase.git",
			withSubmodule: "with-submodule.git",
		}
	}

	const simpleBuild = `apiVersion: v1
kind: Pod
metadata:
  labels:
    app: myapp
  name: myapp-pod
spec:
  containers:
  - image: nginx:1.7.9
    name: nginx
`
	var simpleBuildWithNginx2 = strings.ReplaceAll(simpleBuild, "nginx:1.7.9", "nginx:2")
	var multibaseDevExampleBuild = strings.ReplaceAll(simpleBuild, "myapp-pod", "dev-myapp-pod")

	repos := createGitRepos(t)
	tests := []struct {
		name          string
		kustomization string
		expected      string
		err           string
		skip          bool
	}{
		{
			name: "simple",
			kustomization: `
resources:
- file://$ROOT/simple.git
`,
			expected: simpleBuild,
		},
		{
			name: "without git suffix",
			kustomization: `
resources:
- file://$ROOT/nosuffix
`,
			expected: simpleBuild,
		},
		{
			name: "has path",
			kustomization: `
resources:
- file://$ROOT/multibase.git/dev
`,
			expected: multibaseDevExampleBuild,
		},
		{
			name: "has path without git suffix",
			kustomization: `
resources:
- file://$ROOT/multibase//dev
`,
			expected: multibaseDevExampleBuild,
		},
		{
			name: "has ref",
			kustomization: `
resources:
- "file://$ROOT/simple.git?ref=change-image"
`,

			expected: simpleBuildWithNginx2,
		},
		{
			// Version is the same as ref
			name: "has version",
			kustomization: `
resources:
- file://$ROOT/simple.git?version=change-image
`,
			expected: simpleBuildWithNginx2,
		},
		{
			name: "has submodule",
			kustomization: `
resources:
- file://$ROOT/with-submodule.git/submodule
`,
			expected: simpleBuild,
		},
		{
			name: "has timeout",
			kustomization: `
resources:
- file://$ROOT/simple.git?timeout=500
`,
			expected: simpleBuild,
		},
		{
			name: "triple slash absolute path",
			kustomization: `
resources:
- file:///$ROOT/simple.git
`,
			expected: simpleBuild,
		},
		{
			name: "has submodule but not initialized",
			kustomization: `
resources:
- file://$ROOT/with-submodule.git/submodule?submodules=0
`,
			err: "unable to find one of 'kustomization.yaml', 'kustomization.yml' or 'Kustomization' in directory",
		},
		{
			name: "has origin annotation",
			skip: true, // The annotated path should be "pod.yaml" but is "notCloned/pod.yaml"
			kustomization: `
resources:
- file://$ROOT/simple.git
buildMetadata: [originAnnotations]
`,
			expected: `apiVersion: v1
kind: Pod
metadata:
  annotations:
    config.kubernetes.io/origin: |
      path: pod.yaml
      repo: file://$ROOT/simple.git
  labels:
    app: myapp
  name: myapp-pod
spec:
  containers:
  - image: nginx:1.7.9
    name: nginx
`,
		},
		{
			name: "has ref path timeout and origin annotation",
			kustomization: `
resources:
- file://$ROOT/multibase.git/dev?version=main&timeout=500
buildMetadata: [originAnnotations]
`,
			expected: `apiVersion: v1
kind: Pod
metadata:
  annotations:
    config.kubernetes.io/origin: |
      path: base/pod.yaml
      repo: file://$ROOT/multibase.git
      ref: main
  labels:
    app: myapp
  name: dev-myapp-pod
spec:
  containers:
  - image: nginx:1.7.9
    name: nginx
`,
		},
		{
			name: "repo does not exist ",
			kustomization: `
resources:
- file:///not/a/real/repo
`,
			err: "fatal: '/not/a/real/repo' does not appear to be a git repository",
		},
	}

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			if test.skip {
				t.SkipNow()
			}

			kust := strings.ReplaceAll(test.kustomization, "$ROOT", repos.root)
			fSys, tmpDir := createKustDir(t, kust)

			b := krusty.MakeKustomizer(krusty.MakeDefaultOptions())
			m, err := b.Run(
				fSys,
				tmpDir.String())

			if test.err != "" {
				require.Error(t, err)
				require.Contains(t, err.Error(), test.err)
			} else {
				require.NoError(t, err)
				checkYaml(t, m, strings.ReplaceAll(test.expected, "$ROOT", repos.root))
			}
		})
	}
}

func TestRemoteLoad_RemoteProtocols(t *testing.T) {
	// Slow remote tests with long timeouts.
	// TODO: If these end up flaking, they should retry. If not, remove this TODO.
	tests := []struct {
		name          string
		kustomization string
		err           string
		errT          error
		beforeTest    func(t *testing.T)
	}{
		{
			name: "https",
			kustomization: `
resources:
- https://github.com/kubernetes-sigs/kustomize//examples/multibases/dev/?submodules=0&ref=kustomize%2Fv4.5.7&timeout=300
`,
		},
		{
			name: "git double-colon https",
			kustomization: `
resources:
- git::https://github.com/kubernetes-sigs/kustomize//examples/multibases/dev/?submodules=0&ref=kustomize%2Fv4.5.7&timeout=300
`,
		},
		{
			name: "https raw remote file",
			kustomization: `
resources:
- https://raw.githubusercontent.com/kubernetes-sigs/kustomize/v3.1.0/examples/multibases/base/pod.yaml?timeout=300
namePrefix: dev-
`,
		},
		{
			name: "https without scheme",
			kustomization: `
resources:
- github.com/kubernetes-sigs/kustomize/examples/multibases/dev?submodules=0&ref=kustomize%2Fv4.5.7&timeout=300
`,
		},
		{
			name:       "ssh",
			beforeTest: configureGitSSHCommand,
			kustomization: `
resources:
- git@github.com/kubernetes-sigs/kustomize/examples/multibases/dev?submodules=0&ref=kustomize%2Fv4.5.7&timeout=300
`,
		},
		{
			name:       "ssh with colon",
			beforeTest: configureGitSSHCommand,
			kustomization: `
resources:
- git@github.com:kubernetes-sigs/kustomize/examples/multibases/dev?submodules=0&ref=kustomize%2Fv4.5.7&timeout=300
`,
		},
		{
			name:       "ssh scheme",
			beforeTest: configureGitSSHCommand,
			kustomization: `
resources:
- ssh://git@github.com/kubernetes-sigs/kustomize/examples/multibases/dev?submodules=0&ref=kustomize%2Fv4.5.7&timeout=300
`,
		},
		{
			name: "http error",
			kustomization: `
resources:
- https://github.com/thisisa404.yaml
`,
			err:  "accumulating resources: accumulating resources from 'https://github.com/thisisa404.yaml': HTTP Error: status code 404 (Not Found)",
			errT: loader.ErrHTTP,
		},
	}

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			if test.beforeTest != nil {
				test.beforeTest(t)
			}
			fSys, tmpDir := createKustDir(t, test.kustomization)

			b := krusty.MakeKustomizer(krusty.MakeDefaultOptions())
			m, err := b.Run(
				fSys,
				tmpDir.String())

			if test.err != "" {
				require.Error(t, err)
				assert.Contains(t, err.Error(), test.err)
				if test.errT != nil {
					assert.ErrorIs(t, err, test.errT)
				}
			} else {
				require.NoError(t, err)
				const multibaseDevExampleBuild = `apiVersion: v1
kind: Pod
metadata:
  labels:
    app: myapp
  name: dev-myapp-pod
spec:
  containers:
  - image: nginx:1.7.9
    name: nginx
`
				checkYaml(t, m, multibaseDevExampleBuild)
			}
		})
	}
}

func configureGitSSHCommand(t *testing.T) {
	t.Helper()

	// This contains a read-only Deploy Key for the kustomize repo.
	node, err := yaml.ReadFile("testdata/repo_read_only_ssh_key.yaml")
	require.NoError(t, err)
	keyB64, err := node.GetString("key")
	require.NoError(t, err)
	key, err := base64.StdEncoding.DecodeString(keyB64)
	require.NoError(t, err)

	// Write the key to a temp file and use it in SSH
	f, err := os.CreateTemp("", "kustomize_ssh")
	require.NoError(t, err)
	_, err = io.Copy(f, bytes.NewReader(key))
	require.NoError(t, err)
	cmd := fmt.Sprintf("ssh -i %s -o IdentitiesOnly=yes", f.Name())
	const SSHCommandKey = "GIT_SSH_COMMAND"
	t.Setenv(SSHCommandKey, cmd)
	t.Cleanup(func() {
		_ = os.Remove(f.Name())
	})
}

func createKustDir(t *testing.T, content string) (filesys.FileSystem, filesys.ConfirmedDir) {
	t.Helper()

	fSys := filesys.MakeFsOnDisk()
	tmpDir, err := filesys.NewTmpConfirmedDir()
	require.NoError(t, err)
	require.NoError(t, fSys.WriteFile(filepath.Join(tmpDir.String(), "kustomization.yaml"), []byte(content)))
	return fSys, tmpDir
}

func checkYaml(t *testing.T, actual resmap.ResMap, expected string) {
	t.Helper()

	yml, err := actual.AsYaml()
	assert.NoError(t, err)
	assert.Equal(t, expected, string(yml))
}
<commit_msg>fix `TestRemoteLoad_LocalProtocol` (#4844)<commit_after>// Copyright 2022 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0

package krusty_test

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"sigs.k8s.io/kustomize/api/krusty"
	"sigs.k8s.io/kustomize/api/loader"
	"sigs.k8s.io/kustomize/api/resmap"
	"sigs.k8s.io/kustomize/kyaml/filesys"
	"sigs.k8s.io/kustomize/kyaml/yaml"
)

func TestRemoteLoad_LocalProtocol(t *testing.T) {
	type testRepos struct {
		root          string
		simple        string
		noSuffix      string
		multiBaseDev  string
		withSubmodule string
	}

	// creates git repos under a root temporary directory with the following structure
	// root/
	//   simple.git/			- base with just a pod
	//   nosuffix/				- same as simple.git/ without the .git suffix
	//   multibase.git/			- base with a dev overlay
	//   with-submodule.git/	- includes `simple` as a submodule
	//     submodule/			- the submodule referencing `simple
	createGitRepos := func(t *testing.T) testRepos {
		t.Helper()

		bash := func(script string) {
			cmd := exec.Command("sh", "-c", script)
			o, err := cmd.CombinedOutput()
			if err != nil {
				t.Fatalf("error running %v\nerr: %v\n%s", script, err, string(o))
			}
		}
		root := t.TempDir()
		bash(fmt.Sprintf(`
set -eux

export ROOT="%s"
export GIT_AUTHOR_EMAIL=nobody@kustomize.io
export GIT_AUTHOR_NAME=Nobody
export GIT_COMMITTER_EMAIL=nobody@kustomize.io
export GIT_COMMITTER_NAME=Nobody

cp -r testdata/remoteload/simple $ROOT/simple.git
(
	cd $ROOT/simple.git
	git config --global protocol.file.allow always
	git init --initial-branch=main
	git add .
	git commit -m "import"
	git checkout -b change-image
	cat >>kustomization.yaml <<EOF

images:
- name: nginx
  newName: nginx
  newTag: "2"
EOF
	git commit -am "image change"
	git checkout main
)
cp -r $ROOT/simple.git $ROOT/nosuffix
cp -r testdata/remoteload/multibase $ROOT/multibase.git
(
	cd $ROOT/multibase.git
	git init --initial-branch=main
	git add .
	git commit -m "import"
)
(
	mkdir $ROOT/with-submodule.git
	cd $ROOT/with-submodule.git
	git init --initial-branch=main
	git submodule add $ROOT/simple.git submodule
	git commit -m "import"
)
`, root))
		return testRepos{
			root: root,
			// The strings below aren't currently used, and more serve as documentation.
			simple:        "simple.git",
			noSuffix:      "nosuffix",
			multiBaseDev:  "multibase.git",
			withSubmodule: "with-submodule.git",
		}
	}

	const simpleBuild = `apiVersion: v1
kind: Pod
metadata:
  labels:
    app: myapp
  name: myapp-pod
spec:
  containers:
  - image: nginx:1.7.9
    name: nginx
`
	var simpleBuildWithNginx2 = strings.ReplaceAll(simpleBuild, "nginx:1.7.9", "nginx:2")
	var multibaseDevExampleBuild = strings.ReplaceAll(simpleBuild, "myapp-pod", "dev-myapp-pod")

	repos := createGitRepos(t)
	tests := []struct {
		name          string
		kustomization string
		expected      string
		err           string
		skip          bool
	}{
		{
			name: "simple",
			kustomization: `
resources:
- file://$ROOT/simple.git
`,
			expected: simpleBuild,
		},
		{
			name: "without git suffix",
			kustomization: `
resources:
- file://$ROOT/nosuffix
`,
			expected: simpleBuild,
		},
		{
			name: "has path",
			kustomization: `
resources:
- file://$ROOT/multibase.git/dev
`,
			expected: multibaseDevExampleBuild,
		},
		{
			name: "has path without git suffix",
			kustomization: `
resources:
- file://$ROOT/multibase//dev
`,
			expected: multibaseDevExampleBuild,
		},
		{
			name: "has ref",
			kustomization: `
resources:
- "file://$ROOT/simple.git?ref=change-image"
`,

			expected: simpleBuildWithNginx2,
		},
		{
			// Version is the same as ref
			name: "has version",
			kustomization: `
resources:
- file://$ROOT/simple.git?version=change-image
`,
			expected: simpleBuildWithNginx2,
		},
		{
			name: "has submodule",
			kustomization: `
resources:
- file://$ROOT/with-submodule.git/submodule
`,
			expected: simpleBuild,
		},
		{
			name: "has timeout",
			kustomization: `
resources:
- file://$ROOT/simple.git?timeout=500
`,
			expected: simpleBuild,
		},
		{
			name: "triple slash absolute path",
			kustomization: `
resources:
- file:///$ROOT/simple.git
`,
			expected: simpleBuild,
		},
		{
			name: "has submodule but not initialized",
			kustomization: `
resources:
- file://$ROOT/with-submodule.git/submodule?submodules=0
`,
			err: "unable to find one of 'kustomization.yaml', 'kustomization.yml' or 'Kustomization' in directory",
		},
		{
			name: "has origin annotation",
			skip: true, // The annotated path should be "pod.yaml" but is "notCloned/pod.yaml"
			kustomization: `
resources:
- file://$ROOT/simple.git
buildMetadata: [originAnnotations]
`,
			expected: `apiVersion: v1
kind: Pod
metadata:
  annotations:
    config.kubernetes.io/origin: |
      path: pod.yaml
      repo: file://$ROOT/simple.git
  labels:
    app: myapp
  name: myapp-pod
spec:
  containers:
  - image: nginx:1.7.9
    name: nginx
`,
		},
		{
			name: "has ref path timeout and origin annotation",
			kustomization: `
resources:
- file://$ROOT/multibase.git/dev?version=main&timeout=500
buildMetadata: [originAnnotations]
`,
			expected: `apiVersion: v1
kind: Pod
metadata:
  annotations:
    config.kubernetes.io/origin: |
      path: base/pod.yaml
      repo: file://$ROOT/multibase.git
      ref: main
  labels:
    app: myapp
  name: dev-myapp-pod
spec:
  containers:
  - image: nginx:1.7.9
    name: nginx
`,
		},
		{
			name: "repo does not exist ",
			kustomization: `
resources:
- file:///not/a/real/repo
`,
			err: "fatal: '/not/a/real/repo' does not appear to be a git repository",
		},
	}

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			if test.skip {
				t.SkipNow()
			}

			kust := strings.ReplaceAll(test.kustomization, "$ROOT", repos.root)
			fSys, tmpDir := createKustDir(t, kust)

			b := krusty.MakeKustomizer(krusty.MakeDefaultOptions())
			m, err := b.Run(
				fSys,
				tmpDir.String())

			if test.err != "" {
				require.Error(t, err)
				require.Contains(t, err.Error(), test.err)
			} else {
				require.NoError(t, err)
				checkYaml(t, m, strings.ReplaceAll(test.expected, "$ROOT", repos.root))
			}
		})
	}
}

func TestRemoteLoad_RemoteProtocols(t *testing.T) {
	// Slow remote tests with long timeouts.
	// TODO: If these end up flaking, they should retry. If not, remove this TODO.
	tests := []struct {
		name          string
		kustomization string
		err           string
		errT          error
		beforeTest    func(t *testing.T)
	}{
		{
			name: "https",
			kustomization: `
resources:
- https://github.com/kubernetes-sigs/kustomize//examples/multibases/dev/?submodules=0&ref=kustomize%2Fv4.5.7&timeout=300
`,
		},
		{
			name: "git double-colon https",
			kustomization: `
resources:
- git::https://github.com/kubernetes-sigs/kustomize//examples/multibases/dev/?submodules=0&ref=kustomize%2Fv4.5.7&timeout=300
`,
		},
		{
			name: "https raw remote file",
			kustomization: `
resources:
- https://raw.githubusercontent.com/kubernetes-sigs/kustomize/v3.1.0/examples/multibases/base/pod.yaml?timeout=300
namePrefix: dev-
`,
		},
		{
			name: "https without scheme",
			kustomization: `
resources:
- github.com/kubernetes-sigs/kustomize/examples/multibases/dev?submodules=0&ref=kustomize%2Fv4.5.7&timeout=300
`,
		},
		{
			name:       "ssh",
			beforeTest: configureGitSSHCommand,
			kustomization: `
resources:
- git@github.com/kubernetes-sigs/kustomize/examples/multibases/dev?submodules=0&ref=kustomize%2Fv4.5.7&timeout=300
`,
		},
		{
			name:       "ssh with colon",
			beforeTest: configureGitSSHCommand,
			kustomization: `
resources:
- git@github.com:kubernetes-sigs/kustomize/examples/multibases/dev?submodules=0&ref=kustomize%2Fv4.5.7&timeout=300
`,
		},
		{
			name:       "ssh scheme",
			beforeTest: configureGitSSHCommand,
			kustomization: `
resources:
- ssh://git@github.com/kubernetes-sigs/kustomize/examples/multibases/dev?submodules=0&ref=kustomize%2Fv4.5.7&timeout=300
`,
		},
		{
			name: "http error",
			kustomization: `
resources:
- https://github.com/thisisa404.yaml
`,
			err:  "accumulating resources: accumulating resources from 'https://github.com/thisisa404.yaml': HTTP Error: status code 404 (Not Found)",
			errT: loader.ErrHTTP,
		},
	}

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			if test.beforeTest != nil {
				test.beforeTest(t)
			}
			fSys, tmpDir := createKustDir(t, test.kustomization)

			b := krusty.MakeKustomizer(krusty.MakeDefaultOptions())
			m, err := b.Run(
				fSys,
				tmpDir.String())

			if test.err != "" {
				require.Error(t, err)
				assert.Contains(t, err.Error(), test.err)
				if test.errT != nil {
					assert.ErrorIs(t, err, test.errT)
				}
			} else {
				require.NoError(t, err)
				const multibaseDevExampleBuild = `apiVersion: v1
kind: Pod
metadata:
  labels:
    app: myapp
  name: dev-myapp-pod
spec:
  containers:
  - image: nginx:1.7.9
    name: nginx
`
				checkYaml(t, m, multibaseDevExampleBuild)
			}
		})
	}
}

func configureGitSSHCommand(t *testing.T) {
	t.Helper()

	// This contains a read-only Deploy Key for the kustomize repo.
	node, err := yaml.ReadFile("testdata/repo_read_only_ssh_key.yaml")
	require.NoError(t, err)
	keyB64, err := node.GetString("key")
	require.NoError(t, err)
	key, err := base64.StdEncoding.DecodeString(keyB64)
	require.NoError(t, err)

	// Write the key to a temp file and use it in SSH
	f, err := os.CreateTemp("", "kustomize_ssh")
	require.NoError(t, err)
	_, err = io.Copy(f, bytes.NewReader(key))
	require.NoError(t, err)
	cmd := fmt.Sprintf("ssh -i %s -o IdentitiesOnly=yes", f.Name())
	const SSHCommandKey = "GIT_SSH_COMMAND"
	t.Setenv(SSHCommandKey, cmd)
	t.Cleanup(func() {
		_ = os.Remove(f.Name())
	})
}

func createKustDir(t *testing.T, content string) (filesys.FileSystem, filesys.ConfirmedDir) {
	t.Helper()

	fSys := filesys.MakeFsOnDisk()
	tmpDir, err := filesys.NewTmpConfirmedDir()
	require.NoError(t, err)
	require.NoError(t, fSys.WriteFile(filepath.Join(tmpDir.String(), "kustomization.yaml"), []byte(content)))
	return fSys, tmpDir
}

func checkYaml(t *testing.T, actual resmap.ResMap, expected string) {
	t.Helper()

	yml, err := actual.AsYaml()
	assert.NoError(t, err)
	assert.Equal(t, expected, string(yml))
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package model

import (
	"fmt"

	"k8s.io/kops/nodeup/pkg/distros"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"

	"github.com/golang/glog"
)

// KubectlBuilder install kubectl
type KubectlBuilder struct {
	*NodeupModelContext
}

var _ fi.ModelBuilder = &KubectlBuilder{}

// Build is responsible for managing the kubectl on the nodes
func (b *KubectlBuilder) Build(c *fi.ModelBuilderContext) error {
	if !b.IsMaster {
		return nil
	}

	{
		// TODO: Extract to common function?
		assetName := "kubectl"
		assetPath := ""
		asset, err := b.Assets.Find(assetName, assetPath)
		if err != nil {
			return fmt.Errorf("error trying to locate asset %q: %v", assetName, err)
		}
		if asset == nil {
			return fmt.Errorf("unable to locate asset %q", assetName)
		}

		c.AddTask(&nodetasks.File{
			Path:     b.KubectlPath() + "/" + assetName,
			Contents: asset,
			Type:     nodetasks.FileType_File,
			Mode:     s("0755"),
		})
	}

	{
		kubeconfig, err := b.BuildPKIKubeconfig("kubecfg")
		if err != nil {
			return err
		}

		c.AddTask(&nodetasks.File{
			Path:     "/var/lib/kubectl/kubeconfig",
			Contents: fi.NewStringResource(kubeconfig),
			Type:     nodetasks.FileType_File,
			Mode:     s("0400"),
		})

		adminUser, adminGroup, err := b.findKubeconfigUser()
		if err != nil {
			return err
		}

		if adminUser != nil && adminUser.Home != "" {
			c.AddTask(&nodetasks.File{
				Path:  adminUser.Home + "/.kube/",
				Type:  nodetasks.FileType_Directory,
				Mode:  s("0700"),
				Owner: s(adminUser.Name),
				Group: s(adminGroup.Name),
			})

			c.AddTask(&nodetasks.File{
				Path:     adminUser.Home + "/.kube/config",
				Contents: fi.NewStringResource(kubeconfig),
				Type:     nodetasks.FileType_File,
				Mode:     s("0400"),
				Owner:    s(adminUser.Name),
				Group:    s(adminGroup.Name),
			})
		}
	}

	return nil
}

// findKubeconfigUser finds the default user for whom we should create a kubeconfig
func (b *KubectlBuilder) findKubeconfigUser() (*fi.User, *fi.Group, error) {
	var users []string
	switch b.Distribution {
	case distros.DistributionJessie, distros.DistributionDebian9:
		users = []string{"admin", "root"}
	default:
		glog.Warningf("Unknown distro; won't write kubeconfig to homedir %s", b.Distribution)
		return nil, nil, nil
	}

	for _, s := range users {
		user, err := fi.LookupUser(s)
		if err != nil {
			glog.Warningf("error looking up user %q: %v", s, err)
			continue
		}
		if user == nil {
			continue
		}
		group, err := fi.LookupGroupById(user.Gid)
		if err != nil {
			glog.Warningf("unable to find group %d for user %q", user.Gid, s)
			continue
		}
		if group == nil {
			continue
		}
		return user, group, nil
	}

	return nil, nil, nil
}
<commit_msg>Install kubelet config for default centos user<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package model

import (
	"fmt"

	"k8s.io/kops/nodeup/pkg/distros"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"

	"github.com/golang/glog"
)

// KubectlBuilder install kubectl
type KubectlBuilder struct {
	*NodeupModelContext
}

var _ fi.ModelBuilder = &KubectlBuilder{}

// Build is responsible for managing the kubectl on the nodes
func (b *KubectlBuilder) Build(c *fi.ModelBuilderContext) error {
	if !b.IsMaster {
		return nil
	}

	{
		// TODO: Extract to common function?
		assetName := "kubectl"
		assetPath := ""
		asset, err := b.Assets.Find(assetName, assetPath)
		if err != nil {
			return fmt.Errorf("error trying to locate asset %q: %v", assetName, err)
		}
		if asset == nil {
			return fmt.Errorf("unable to locate asset %q", assetName)
		}

		c.AddTask(&nodetasks.File{
			Path:     b.KubectlPath() + "/" + assetName,
			Contents: asset,
			Type:     nodetasks.FileType_File,
			Mode:     s("0755"),
		})
	}

	{
		kubeconfig, err := b.BuildPKIKubeconfig("kubecfg")
		if err != nil {
			return err
		}

		c.AddTask(&nodetasks.File{
			Path:     "/var/lib/kubectl/kubeconfig",
			Contents: fi.NewStringResource(kubeconfig),
			Type:     nodetasks.FileType_File,
			Mode:     s("0400"),
		})

		adminUser, adminGroup, err := b.findKubeconfigUser()
		if err != nil {
			return err
		}

		if adminUser != nil && adminUser.Home != "" {
			c.AddTask(&nodetasks.File{
				Path:  adminUser.Home + "/.kube/",
				Type:  nodetasks.FileType_Directory,
				Mode:  s("0700"),
				Owner: s(adminUser.Name),
				Group: s(adminGroup.Name),
			})

			c.AddTask(&nodetasks.File{
				Path:     adminUser.Home + "/.kube/config",
				Contents: fi.NewStringResource(kubeconfig),
				Type:     nodetasks.FileType_File,
				Mode:     s("0400"),
				Owner:    s(adminUser.Name),
				Group:    s(adminGroup.Name),
			})
		}
	}

	return nil
}

// findKubeconfigUser finds the default user for whom we should create a kubeconfig
func (b *KubectlBuilder) findKubeconfigUser() (*fi.User, *fi.Group, error) {
	var users []string
	switch b.Distribution {
	case distros.DistributionJessie, distros.DistributionDebian9:
		users = []string{"admin", "root"}
	case distros.DistributionCentos7:
		users = []string{"centos"}
	default:
		glog.Warningf("Unknown distro; won't write kubeconfig to homedir %s", b.Distribution)
		return nil, nil, nil
	}

	for _, s := range users {
		user, err := fi.LookupUser(s)
		if err != nil {
			glog.Warningf("error looking up user %q: %v", s, err)
			continue
		}
		if user == nil {
			continue
		}
		group, err := fi.LookupGroupById(user.Gid)
		if err != nil {
			glog.Warningf("unable to find group %d for user %q", user.Gid, s)
			continue
		}
		if group == nil {
			continue
		}
		return user, group, nil
	}

	return nil, nil, nil
}
<|endoftext|>


<commit_before>package parsesearch

import (
	"container/list"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"reflect"
	"sync"
	"sync/atomic"

	"github.com/blevesearch/bleve"
	"github.com/davecgh/go-spew/spew"
	"github.com/tmc/parse"
)

// Indexer manages the search index for a Parse app.
type Indexer struct {
	index bleve.Index

	// Parse app keys/appID
	webhookKey string
	masterKey  string
	appID      string
	status     *indexStatusList
}

// NewIndexer prepares a new Indexer given the necessary Parse App credentials.
func NewIndexer(webhookKey, masterKey, appID string) (*Indexer, error) {
	path := "contents.bleve"
	i, err := bleve.Open(path)
	if err == bleve.ErrorIndexPathDoesNotExist {
		im := bleve.NewIndexMapping()
		i, err = bleve.New(path, im)
	} else if err != nil {
		return nil, err
	}
	return &Indexer{
		index:      i,
		webhookKey: webhookKey,
		masterKey:  masterKey,
		appID:      appID,
		status:     &indexStatusList{},
	}, nil
}

// Index is an http.HandlerFunc that accepts a parse afterSave webhook request.
//
// It adds or updates the provided objet in the search index.
func (i *Indexer) Index(w http.ResponseWriter, r *http.Request) {
	req, err := webhookRequest(r, i.webhookKey)
	if err != nil {
		writeErr(w, err)
		return
	}
	//TODO(tmc): guard these casts
	obj := req.Object.(map[string]interface{})
	err = i.index.Index(obj["objectId"].(string), obj)
	if err != nil {
		writeErr(w, err)
		return
	}
	json.NewEncoder(os.Stdout).Encode(req)

	err = json.NewEncoder(w).Encode(Response{
		Success: true,
	})
	if err != nil {
		log.Println("error writing response:", err)
	}
}

// Unindex is an http.HandlerFunc that accepts a parse afterDelete webhook request.
//
// It removes the provided object from the index.
func (i *Indexer) Unindex(w http.ResponseWriter, r *http.Request) {
	req, err := webhookRequest(r, i.webhookKey)
	if err != nil {
		writeErr(w, err)
		return
	}
	//TODO(tmc): guard these casts
	obj := req.Object.(map[string]interface{})
	err = i.index.Delete(obj["objectId"].(string))
	if err != nil {
		writeErr(w, err)
		return
	}
	json.NewEncoder(os.Stdout).Encode(req)

	err = json.NewEncoder(w).Encode(Response{
		Success: true,
	})
	if err != nil {
		log.Println("error writing response:", err)
	}
}

// Search is an http.HandlerFunc that accepts a Parse Cloud Code Webhook request.
//
// The expected query parameter is 'q'
func (i *Indexer) Search(w http.ResponseWriter, r *http.Request) {
	req, err := webhookRequest(r, i.webhookKey)
	if err != nil {
		writeErr(w, err)
		return
	}
	rawq := req.Params["q"]
	if rawq == nil {
		writeErr(w, fmt.Errorf("no term provided"))
		return
	}
	q, ok := rawq.(string)
	if q == "" || !ok {
		writeErr(w, fmt.Errorf("no term provided"))
		return
	}
	query := bleve.NewQueryStringQuery(q)
	search := bleve.NewSearchRequest(query)
	searchResults, err := i.index.Search(search)
	if err != nil {
		writeErr(w, err)
		return
	}
	spew.Dump(searchResults)
	ids := []string{}
	for _, h := range searchResults.Hits {
		ids = append(ids, h.ID)
	}
	err = json.NewEncoder(w).Encode(Response{
		Success: ids,
	})
	if err != nil {
		log.Println("error encoding response:", err)
	}
}

// Reindex fetches all objects for a class and indexes them.
// Could be long-running.
func (i *Indexer) Reindex(className string) error {
	//TODO(tmc): add counters/progress reporting
	client, err := parse.NewClient(i.appID, "")
	if err != nil {
		return err
	}
	client.TraceOn(log.New(os.Stderr, "[parse api] ", log.LstdFlags))
	client = client.WithMasterKey(i.masterKey)
	iter, err := client.NewQueryIter(className, "{}")
	if err != nil {
		return err
	}
	status := &indexStatus{class: className}

	i.status.register(status)
	defer i.status.unregister(status)

	for o := iter.Next(); o != nil; o = iter.Next() {
		obj := o.(map[string]interface{})
		if err := i.index.Index(obj["objectId"].(string), obj); err != nil {
			log.Println("index error:", err)
		}

		log.Println("index incremented")
		status.incr()
	}
	return iter.Err()
}

// IndexStatus lists the status of all current Reindexing processes.
func (i *Indexer) IndexStatus(w http.ResponseWriter, r *http.Request) {
	i.status.writeIndexStatus(w)
}

type indexStatus struct {
	class string
	count uint32
}

func (i *indexStatus) incr() {
	atomic.AddUint32(&i.count, 1)
}

func (i *indexStatus) String() string {
	return fmt.Sprintf("indexed %d objects for class %s", i.count, i.class)
}

type indexStatusList struct {
	inProgress *list.List
	lock       sync.Mutex
	once       sync.Once
}

func (i *indexStatusList) register(status *indexStatus) {
	i.once.Do(func() {
		i.inProgress = list.New()
	})
	i.lock.Lock()
	defer i.lock.Unlock()
	i.inProgress.PushBack(status)
}

func (i *indexStatusList) unregister(status *indexStatus) {
	i.lock.Lock()
	defer i.lock.Unlock()
	var found *list.Element
	for e := i.inProgress.Front(); e != nil; e = e.Next() {
		if reflect.DeepEqual(e.Value, status) {
			found = e
			break
		}
	}
	if found != nil {
		i.inProgress.Remove(found)
	}
}

func (i *indexStatusList) writeIndexStatus(w io.Writer) {
	i.lock.Lock()
	defer i.lock.Unlock()
	for e := i.inProgress.Front(); e != nil; e = e.Next() {
		m := e.Value.(*indexStatus).String()
		fmt.Fprintf(w, "%s\n", m)
	}
}
<commit_msg>Use new QueryIter api<commit_after>package parsesearch

import (
	"container/list"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"reflect"
	"sync"
	"sync/atomic"

	"github.com/blevesearch/bleve"
	"github.com/davecgh/go-spew/spew"
	"github.com/tmc/parse"
)

// Indexer manages the search index for a Parse app.
type Indexer struct {
	index bleve.Index

	// Parse app keys/appID
	webhookKey string
	masterKey  string
	appID      string
	status     *indexStatusList
}

// NewIndexer prepares a new Indexer given the necessary Parse App credentials.
func NewIndexer(webhookKey, masterKey, appID string) (*Indexer, error) {
	path := "contents.bleve"
	i, err := bleve.Open(path)
	if err == bleve.ErrorIndexPathDoesNotExist {
		im := bleve.NewIndexMapping()
		i, err = bleve.New(path, im)
	} else if err != nil {
		return nil, err
	}
	return &Indexer{
		index:      i,
		webhookKey: webhookKey,
		masterKey:  masterKey,
		appID:      appID,
		status:     &indexStatusList{},
	}, nil
}

// Index is an http.HandlerFunc that accepts a parse afterSave webhook request.
//
// It adds or updates the provided objet in the search index.
func (i *Indexer) Index(w http.ResponseWriter, r *http.Request) {
	req, err := webhookRequest(r, i.webhookKey)
	if err != nil {
		writeErr(w, err)
		return
	}
	//TODO(tmc): guard these casts
	obj := req.Object.(map[string]interface{})
	err = i.index.Index(obj["objectId"].(string), obj)
	if err != nil {
		writeErr(w, err)
		return
	}
	json.NewEncoder(os.Stdout).Encode(req)

	err = json.NewEncoder(w).Encode(Response{
		Success: true,
	})
	if err != nil {
		log.Println("error writing response:", err)
	}
}

// Unindex is an http.HandlerFunc that accepts a parse afterDelete webhook request.
//
// It removes the provided object from the index.
func (i *Indexer) Unindex(w http.ResponseWriter, r *http.Request) {
	req, err := webhookRequest(r, i.webhookKey)
	if err != nil {
		writeErr(w, err)
		return
	}
	//TODO(tmc): guard these casts
	obj := req.Object.(map[string]interface{})
	err = i.index.Delete(obj["objectId"].(string))
	if err != nil {
		writeErr(w, err)
		return
	}
	json.NewEncoder(os.Stdout).Encode(req)

	err = json.NewEncoder(w).Encode(Response{
		Success: true,
	})
	if err != nil {
		log.Println("error writing response:", err)
	}
}

// Search is an http.HandlerFunc that accepts a Parse Cloud Code Webhook request.
//
// The expected query parameter is 'q'
func (i *Indexer) Search(w http.ResponseWriter, r *http.Request) {
	req, err := webhookRequest(r, i.webhookKey)
	if err != nil {
		writeErr(w, err)
		return
	}
	rawq := req.Params["q"]
	if rawq == nil {
		writeErr(w, fmt.Errorf("no term provided"))
		return
	}
	q, ok := rawq.(string)
	if q == "" || !ok {
		writeErr(w, fmt.Errorf("no term provided"))
		return
	}
	query := bleve.NewQueryStringQuery(q)
	search := bleve.NewSearchRequest(query)
	searchResults, err := i.index.Search(search)
	if err != nil {
		writeErr(w, err)
		return
	}
	spew.Dump(searchResults)
	ids := []string{}
	for _, h := range searchResults.Hits {
		ids = append(ids, h.ID)
	}
	err = json.NewEncoder(w).Encode(Response{
		Success: ids,
	})
	if err != nil {
		log.Println("error encoding response:", err)
	}
}

// Reindex fetches all objects for a class and indexes them.
// Could be long-running.
func (i *Indexer) Reindex(className string) error {
	//TODO(tmc): add counters/progress reporting
	client, err := parse.NewClient(i.appID, "")
	if err != nil {
		return err
	}
	client.TraceOn(log.New(os.Stderr, "[parse api] ", log.LstdFlags))
	client = client.WithMasterKey(i.masterKey)

	dest := []map[string]interface{}{}
	iter, err := client.NewQueryClassIter(className, "{}", dest)
	if err != nil {
		return err
	}

	status := &indexStatus{class: className}

	i.status.register(status)
	defer i.status.unregister(status)

	for iter.Next() {
		for _, obj := range dest {
			if err := i.index.Index(obj["objectId"].(string), obj); err != nil {
				log.Println("index error:", err)
			}

			log.Println("index incremented")
			status.incr()
		}
	}
	return iter.Err()
}

// IndexStatus lists the status of all current Reindexing processes.
func (i *Indexer) IndexStatus(w http.ResponseWriter, r *http.Request) {
	i.status.writeIndexStatus(w)
}

type indexStatus struct {
	class string
	count uint32
}

func (i *indexStatus) incr() {
	atomic.AddUint32(&i.count, 1)
}

func (i *indexStatus) String() string {
	return fmt.Sprintf("indexed %d objects for class %s", i.count, i.class)
}

type indexStatusList struct {
	inProgress *list.List
	lock       sync.Mutex
	once       sync.Once
}

func (i *indexStatusList) register(status *indexStatus) {
	i.once.Do(func() {
		i.inProgress = list.New()
	})
	i.lock.Lock()
	defer i.lock.Unlock()
	i.inProgress.PushBack(status)
}

func (i *indexStatusList) unregister(status *indexStatus) {
	i.lock.Lock()
	defer i.lock.Unlock()
	var found *list.Element
	for e := i.inProgress.Front(); e != nil; e = e.Next() {
		if reflect.DeepEqual(e.Value, status) {
			found = e
			break
		}
	}
	if found != nil {
		i.inProgress.Remove(found)
	}
}

func (i *indexStatusList) writeIndexStatus(w io.Writer) {
	i.lock.Lock()
	defer i.lock.Unlock()
	for e := i.inProgress.Front(); e != nil; e = e.Next() {
		m := e.Value.(*indexStatus).String()
		fmt.Fprintf(w, "%s\n", m)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"os"
	"path"

	"github.com/vito/gocart/command_runner"
	"github.com/vito/gocart/dependency"
	"github.com/vito/gocart/dependency_fetcher"
	"github.com/vito/gocart/locker"
)

func install(root string, recursive bool, aggregate bool, trickleDown bool, depth int) {
	if _, err := os.Stat(path.Join(root, CartridgeFile)); err != nil {
		println("no Cartridge file!")
		os.Exit(1)
		return
	}

	requestedDependencies := loadFile(path.Join(root, CartridgeFile))
	lockedDependencies := loadFile(path.Join(root, CartridgeLockFile))

	dependencies := locker.GenerateLock(requestedDependencies, lockedDependencies)

	newLockedDependencies := installDependencies(dependencies, recursive, trickleDown, depth)

	file, err := os.Create(path.Join(root, "Cartridge.lock"))
	if err != nil {
		fatal(err.Error())
	}
	defer file.Close()

	var dependenciesToBeWritten []dependency.Dependency

	if aggregate {
		for _, dep := range FetchedDependencies {
			dependenciesToBeWritten = append(dependenciesToBeWritten, dep)
		}
	} else {
		dependenciesToBeWritten = newLockedDependencies
	}

	err = updateLockFile(file, dependenciesToBeWritten)
	if err != nil {
		fatal(err.Error())
	}

	if depth == 0 {
		fmt.Println("\x1b[32mOK\x1b[0m")
	}
}

func installDependencies(
	dependencies []dependency.Dependency,
	recursive bool,
	trickleDown bool,
	depth int,
) []dependency.Dependency {
	runner := command_runner.New()

	fetcher, err := dependency_fetcher.New(runner)
	if err != nil {
		fatal(err.Error())
	}

	maxWidth := 0

	for _, dep := range dependencies {
		if len(dep.Path) > maxWidth {
			maxWidth = len(dep.Path)
		}
	}

	lockedDependencies := []dependency.Dependency{}

	for _, dep := range dependencies {
		fmt.Println(bold(dep.Path) + padding(maxWidth-len(dep.Path)+2) + cyan(dep.Version))

		lockedDependency := processDependency(fetcher, dep)

		if depth == 0 && trickleDown {
			TrickleDownDependencies[lockedDependency.Path] = lockedDependency
		}

		lockedDependencies = append(lockedDependencies, lockedDependency)
	}

	if recursive {
		installNextLevel(lockedDependencies, depth+1)
	}

	return lockedDependencies
}

func processDependency(
	fetcher *dependency_fetcher.DependencyFetcher,
	dep dependency.Dependency,
) dependency.Dependency {
	dep = trickledDown(dep)

	if findCurrentVersion(dep) == dep.Version {
		return dep
	}

	lockedDependency, err := fetcher.Fetch(dep)
	if err != nil {
		fatal(err.Error())
	}

	checkForConflicts(lockedDependency)

	FetchedDependencies[lockedDependency.Path] = lockedDependency

	return lockedDependency
}

func updateLockFile(writer io.Writer, dependencies []dependency.Dependency) error {
	for _, dependency := range dependencies {
		_, err := writer.Write([]byte(fmt.Sprintf("%s\t%s\n", dependency.Path, dependency.Version)))
		if err != nil {
			return err
		}
	}

	return nil
}

func installNextLevel(deps []dependency.Dependency, newDepth int) {
	for _, dependency := range deps {
		dependencyPath := dependency.FullPath(GOPATH)

		if _, err := os.Stat(path.Join(dependencyPath, "Cartridge")); err == nil {
			fmt.Println("\nfetching dependencies for", dependency.Path)
			install(dependencyPath, true, false, false, newDepth)
		}
	}
}

func trickledDown(dep dependency.Dependency) dependency.Dependency {
	trickled, found := TrickleDownDependencies[dep.Path]
	if found {
		return trickled
	}

	return dep
}

func checkForConflicts(lockedDependency dependency.Dependency) {
	currentVersion, found := FetchedDependencies[lockedDependency.Path]
	if found && currentVersion.Version != lockedDependency.Version {
		fatal("version conflict for " + currentVersion.Path)
	}
}
<commit_msg>use green helper<commit_after>package main

import (
	"fmt"
	"io"
	"os"
	"path"

	"github.com/vito/gocart/command_runner"
	"github.com/vito/gocart/dependency"
	"github.com/vito/gocart/dependency_fetcher"
	"github.com/vito/gocart/locker"
)

func install(root string, recursive bool, aggregate bool, trickleDown bool, depth int) {
	if _, err := os.Stat(path.Join(root, CartridgeFile)); err != nil {
		println("no Cartridge file!")
		os.Exit(1)
		return
	}

	requestedDependencies := loadFile(path.Join(root, CartridgeFile))
	lockedDependencies := loadFile(path.Join(root, CartridgeLockFile))

	dependencies := locker.GenerateLock(requestedDependencies, lockedDependencies)

	newLockedDependencies := installDependencies(dependencies, recursive, trickleDown, depth)

	file, err := os.Create(path.Join(root, "Cartridge.lock"))
	if err != nil {
		fatal(err.Error())
	}
	defer file.Close()

	var dependenciesToBeWritten []dependency.Dependency

	if aggregate {
		for _, dep := range FetchedDependencies {
			dependenciesToBeWritten = append(dependenciesToBeWritten, dep)
		}
	} else {
		dependenciesToBeWritten = newLockedDependencies
	}

	err = updateLockFile(file, dependenciesToBeWritten)
	if err != nil {
		fatal(err.Error())
	}

	if depth == 0 {
		fmt.Println(green("OK"))
	}
}

func installDependencies(
	dependencies []dependency.Dependency,
	recursive bool,
	trickleDown bool,
	depth int,
) []dependency.Dependency {
	runner := command_runner.New()

	fetcher, err := dependency_fetcher.New(runner)
	if err != nil {
		fatal(err.Error())
	}

	maxWidth := 0

	for _, dep := range dependencies {
		if len(dep.Path) > maxWidth {
			maxWidth = len(dep.Path)
		}
	}

	lockedDependencies := []dependency.Dependency{}

	for _, dep := range dependencies {
		fmt.Println(bold(dep.Path) + padding(maxWidth-len(dep.Path)+2) + cyan(dep.Version))

		lockedDependency := processDependency(fetcher, dep)

		if depth == 0 && trickleDown {
			TrickleDownDependencies[lockedDependency.Path] = lockedDependency
		}

		lockedDependencies = append(lockedDependencies, lockedDependency)
	}

	if recursive {
		installNextLevel(lockedDependencies, depth+1)
	}

	return lockedDependencies
}

func processDependency(
	fetcher *dependency_fetcher.DependencyFetcher,
	dep dependency.Dependency,
) dependency.Dependency {
	dep = trickledDown(dep)

	if findCurrentVersion(dep) == dep.Version {
		return dep
	}

	lockedDependency, err := fetcher.Fetch(dep)
	if err != nil {
		fatal(err.Error())
	}

	checkForConflicts(lockedDependency)

	FetchedDependencies[lockedDependency.Path] = lockedDependency

	return lockedDependency
}

func updateLockFile(writer io.Writer, dependencies []dependency.Dependency) error {
	for _, dependency := range dependencies {
		_, err := writer.Write([]byte(fmt.Sprintf("%s\t%s\n", dependency.Path, dependency.Version)))
		if err != nil {
			return err
		}
	}

	return nil
}

func installNextLevel(deps []dependency.Dependency, newDepth int) {
	for _, dependency := range deps {
		dependencyPath := dependency.FullPath(GOPATH)

		if _, err := os.Stat(path.Join(dependencyPath, "Cartridge")); err == nil {
			fmt.Println("\nfetching dependencies for", dependency.Path)
			install(dependencyPath, true, false, false, newDepth)
		}
	}
}

func trickledDown(dep dependency.Dependency) dependency.Dependency {
	trickled, found := TrickleDownDependencies[dep.Path]
	if found {
		return trickled
	}

	return dep
}

func checkForConflicts(lockedDependency dependency.Dependency) {
	currentVersion, found := FetchedDependencies[lockedDependency.Path]
	if found && currentVersion.Version != lockedDependency.Version {
		fatal("version conflict for " + currentVersion.Path)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"os/user"
	"path"
	"strings"

	"github.com/davidrjonas/ssh-iam-bridge/string_array"
)

func check(err error) {
	if err == nil {
		return
	}

	if exerr, ok := err.(*exec.ExitError); ok {
		os.Stderr.Write(exerr.Stderr)
	}

	panic(err)
}

func backupFile(filename string) {
	fmt.Println("Backing up", filename, "to", filename+".orig")
	err := exec.Command("cp", "-f", filename, filename+".orig").Run()
	check(err)
}

func install(selfPath, username string) {
	cmd_name := installAuthorizedKeysCommandScript(selfPath)
	installUser(username)
	installToSshd(cmd_name, username)
	installToPam(selfPath)
	installToCron(selfPath)
}

// ssh is picky about AuthorizedKeysCommand, see man sshd_config
func installAuthorizedKeysCommandScript(selfPath string) string {
	cmd_name := "/usr/sbin/ssh-iam-bridge-public-keys"
	fmt.Println("Writing AuthorizedKeysCommand script", cmd_name)

	script := fmt.Sprintf("#!/bin/sh\nexec %s authorized_keys \"$@\"\n", selfPath)

	check(ioutil.WriteFile(cmd_name, []byte(script), 0755))

	return cmd_name
}

func installUser(username string) {
	_, err := user.Lookup(username)

	if err == nil {
		// User already exists
		return
	}

	if _, ok := err.(user.UnknownUserError); !ok {
		panic(err)
	}

	fmt.Println("Creating SSH authorized keys lookup user", username)

	args := []string{
		"--system",
		"--shell", "/usr/sbin/nologin",
		"--comment", "SSH authorized keys lookup",
		username,
	}

	_, err = exec.Command("useradd", args...).Output()
	check(err)
}

func installToSshd(cmd, username string) {

	filename := "/etc/ssh/sshd_config"

	// TODO: Ensure 'PasswordAuthentication no' and 'UsePAM yes'

	lines_to_add := []string{
		"AuthorizedKeysCommand " + cmd + "\n",
		"AuthorizedKeysCommandUser " + username + "\n",
		"ChallengeResponseAuthentication yes\n",
		"AuthenticationMethods publickey keyboard-interactive:pam,publickey\n",
	}

	lines := string_array.ReadFile(filename)

	if string_array.ContainsAll(lines, lines_to_add) {
		return
	}

	fmt.Println("Updating", filename)

	// Comment out specific lines
	lines_to_comment := []string{
		"AuthorizedKeysCommand ",
		"AuthorizedKeysCommandUser ",
		"ChallengeResponseAuthentication ",
		"AuthenticationMethods ",
	}

	for idx, line := range lines {
		for _, check := range lines_to_comment {
			if strings.HasPrefix(line, check) {
				lines[idx] = "# " + line
			}
		}
	}

	backupFile(filename)

	check(string_array.WriteFile(filename, lines, lines_to_add))

	err := exec.Command("sshd", "-t").Run()

	if err != nil {
		if exerr, ok := err.(*exec.ExitError); ok {
			os.Stderr.Write(exerr.Stderr)
			os.Exit(1)
		}

		panic(err)
	}
}

func installToPam(selfPath string) {

	filename := "/etc/pam.d/sshd"
	fmt.Println("Updating", filename)

	pam_exec := "auth requisite pam_exec.so stdout quiet " + selfPath + " pam_create_user\n"

	lines := string_array.ReadFile(filename)

	for _, line := range lines {
		if line == pam_exec {
			return
		}
	}

	backupFile(filename)
	check(string_array.WriteFile(filename, []string{"# Next line added by " + selfPath + "\n", pam_exec}, lines))
}

func installToCron(selfPath string) {

	filename := "/etc/cron.d/" + path.Base(selfPath)

	fmt.Println("Installing crontab", filename)

	contents := "*/10 * * * * root " + selfPath + " sync_groups"

	check(ioutil.WriteFile(filename, []byte(contents), 0644))
}
<commit_msg>Rename cmd_name to cmdName<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"os/user"
	"path"
	"strings"

	"github.com/davidrjonas/ssh-iam-bridge/string_array"
)

func check(err error) {
	if err == nil {
		return
	}

	if exerr, ok := err.(*exec.ExitError); ok {
		os.Stderr.Write(exerr.Stderr)
	}

	panic(err)
}

func backupFile(filename string) {
	fmt.Println("Backing up", filename, "to", filename+".orig")
	err := exec.Command("cp", "-f", filename, filename+".orig").Run()
	check(err)
}

func install(selfPath, username string) {
	cmdName := installAuthorizedKeysCommandScript(selfPath)
	installUser(username)
	installToSshd(cmdName, username)
	installToPam(selfPath)
	installToCron(selfPath)
}

// ssh is picky about AuthorizedKeysCommand, see man sshd_config
func installAuthorizedKeysCommandScript(selfPath string) string {
	cmdName := "/usr/sbin/ssh-iam-bridge-public-keys"
	fmt.Println("Writing AuthorizedKeysCommand script", cmdName)

	script := fmt.Sprintf("#!/bin/sh\nexec %s authorized_keys \"$@\"\n", selfPath)

	check(ioutil.WriteFile(cmdName, []byte(script), 0755))

	return cmdName
}

func installUser(username string) {
	_, err := user.Lookup(username)

	if err == nil {
		// User already exists
		return
	}

	if _, ok := err.(user.UnknownUserError); !ok {
		panic(err)
	}

	fmt.Println("Creating SSH authorized keys lookup user", username)

	args := []string{
		"--system",
		"--shell", "/usr/sbin/nologin",
		"--comment", "SSH authorized keys lookup",
		username,
	}

	_, err = exec.Command("useradd", args...).Output()
	check(err)
}

func installToSshd(cmd, username string) {

	filename := "/etc/ssh/sshd_config"

	// TODO: Ensure 'PasswordAuthentication no' and 'UsePAM yes'

	lines_to_add := []string{
		"AuthorizedKeysCommand " + cmd + "\n",
		"AuthorizedKeysCommandUser " + username + "\n",
		"ChallengeResponseAuthentication yes\n",
		"AuthenticationMethods publickey keyboard-interactive:pam,publickey\n",
	}

	lines := string_array.ReadFile(filename)

	if string_array.ContainsAll(lines, lines_to_add) {
		return
	}

	fmt.Println("Updating", filename)

	// Comment out specific lines
	lines_to_comment := []string{
		"AuthorizedKeysCommand ",
		"AuthorizedKeysCommandUser ",
		"ChallengeResponseAuthentication ",
		"AuthenticationMethods ",
	}

	for idx, line := range lines {
		for _, check := range lines_to_comment {
			if strings.HasPrefix(line, check) {
				lines[idx] = "# " + line
			}
		}
	}

	backupFile(filename)

	check(string_array.WriteFile(filename, lines, lines_to_add))

	err := exec.Command("sshd", "-t").Run()

	if err != nil {
		if exerr, ok := err.(*exec.ExitError); ok {
			os.Stderr.Write(exerr.Stderr)
			os.Exit(1)
		}

		panic(err)
	}
}

func installToPam(selfPath string) {

	filename := "/etc/pam.d/sshd"
	fmt.Println("Updating", filename)

	pam_exec := "auth requisite pam_exec.so stdout quiet " + selfPath + " pam_create_user\n"

	lines := string_array.ReadFile(filename)

	for _, line := range lines {
		if line == pam_exec {
			return
		}
	}

	backupFile(filename)
	check(string_array.WriteFile(filename, []string{"# Next line added by " + selfPath + "\n", pam_exec}, lines))
}

func installToCron(selfPath string) {

	filename := "/etc/cron.d/" + path.Base(selfPath)

	fmt.Println("Installing crontab", filename)

	contents := "*/10 * * * * root " + selfPath + " sync_groups"

	check(ioutil.WriteFile(filename, []byte(contents), 0644))
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package x11

import (
	"bufio"
	"io"
	"os"
)

// Reads the DISPLAY environment variable, and returns the "12" in ":12.0".
func getDisplay() string {
	d := os.Getenv("DISPLAY")
	if len(d) < 1 || d[0] != ':' {
		return ""
	}
	i := 1
	for ; i < len(d); i++ {
		if d[i] < '0' || d[i] > '9' {
			break
		}
	}
	return d[1:i]
}

// Reads a big-endian uint16 from r, using b as a scratch buffer.
func readU16BE(r io.Reader, b []byte) (uint16, os.Error) {
	_, err := io.ReadFull(r, b[0:2])
	if err != nil {
		return 0, err
	}
	return uint16(b[0])<<8 + uint16(b[1]), nil
}

// Reads a length-prefixed string from r, using b as a scratch buffer.
func readStr(r io.Reader, b []byte) (s string, err os.Error) {
	n, err := readU16BE(r, b)
	if err != nil {
		return
	}
	if int(n) > len(b) {
		return s, os.NewError("Xauthority entry too long for buffer")
	}
	_, err = io.ReadFull(r, b[0:n])
	if err != nil {
		return
	}
	return string(b[0:n]), nil
}

// Reads the ~/.Xauthority file and returns the name/data pair for the DISPLAY.
// b is a scratch buffer to use, and should be at least 256 bytes long (i.e. it should be able to hold a hostname).
func readAuth(b []byte) (name, data string, err os.Error) {
	// As per /usr/include/X11/Xauth.h.
	const familyLocal = 256

	home := os.Getenv("HOME")
	if len(home) == 0 {
		err = os.NewError("unknown HOME")
		return
	}
	r, err := os.Open(home+"/.Xauthority", os.O_RDONLY, 0444)
	if err != nil {
		return
	}
	defer r.Close()
	br := bufio.NewReader(r)

	hostname, err := os.Hostname()
	if err != nil {
		return
	}
	display := getDisplay()
	for {
		family, err := readU16BE(br, b[0:2])
		if err != nil {
			return
		}
		addr, err := readStr(br, b[0:])
		if err != nil {
			return
		}
		disp, err := readStr(br, b[0:])
		if err != nil {
			return
		}
		name0, err := readStr(br, b[0:])
		if err != nil {
			return
		}
		data0, err := readStr(br, b[0:])
		if err != nil {
			return
		}
		if family == familyLocal && addr == hostname && disp == display {
			return name0, data0, nil
		}
	}
	panic("unreachable")
}
<commit_msg>Make exp/draw/x11 respect $XAUTHORITY.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package x11

import (
	"bufio"
	"io"
	"os"
)

// getDisplay reads the DISPLAY environment variable, and returns the "12" in ":12.0".
func getDisplay() string {
	d := os.Getenv("DISPLAY")
	if len(d) < 1 || d[0] != ':' {
		return ""
	}
	i := 1
	for ; i < len(d); i++ {
		if d[i] < '0' || d[i] > '9' {
			break
		}
	}
	return d[1:i]
}

// readU16BE reads a big-endian uint16 from r, using b as a scratch buffer.
func readU16BE(r io.Reader, b []byte) (uint16, os.Error) {
	_, err := io.ReadFull(r, b[0:2])
	if err != nil {
		return 0, err
	}
	return uint16(b[0])<<8 + uint16(b[1]), nil
}

// readStr reads a length-prefixed string from r, using b as a scratch buffer.
func readStr(r io.Reader, b []byte) (string, os.Error) {
	n, err := readU16BE(r, b)
	if err != nil {
		return "", err
	}
	if int(n) > len(b) {
		return "", os.NewError("Xauthority entry too long for buffer")
	}
	_, err = io.ReadFull(r, b[0:n])
	if err != nil {
		return "", err
	}
	return string(b[0:n]), nil
}

// readAuth reads the X authority file and returns the name/data pair for the DISPLAY.
// b is a scratch buffer to use, and should be at least 256 bytes long (i.e. it should be able to hold a hostname).
func readAuth(b []byte) (name, data string, err os.Error) {
	// As per /usr/include/X11/Xauth.h.
	const familyLocal = 256

	fn := os.Getenv("XAUTHORITY")
	if fn == "" {
		home := os.Getenv("HOME")
		if home == "" {
			err = os.NewError("Xauthority not found: $XAUTHORITY, $HOME not set")
			return
		}
		fn = home + "/.Xauthority"
	}
	r, err := os.Open(fn, os.O_RDONLY, 0444)
	if err != nil {
		return
	}
	defer r.Close()
	br := bufio.NewReader(r)

	hostname, err := os.Hostname()
	if err != nil {
		return
	}
	display := getDisplay()
	for {
		family, err := readU16BE(br, b[0:2])
		if err != nil {
			return
		}
		addr, err := readStr(br, b[0:])
		if err != nil {
			return
		}
		disp, err := readStr(br, b[0:])
		if err != nil {
			return
		}
		name0, err := readStr(br, b[0:])
		if err != nil {
			return
		}
		data0, err := readStr(br, b[0:])
		if err != nil {
			return
		}
		if family == familyLocal && addr == hostname && disp == display {
			return name0, data0, nil
		}
	}
	panic("unreachable")
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package scanner

import (
	"fmt"
	"go/token"
	"io"
	"sort"
)

// In an ErrorList, an error is represented by an *Error.
// The position Pos, if valid, points to the beginning of
// the offending token, and the error condition is described
// by Msg.
//
type Error struct {
	Pos token.Position
	Msg string
}

// Error implements the error interface.
func (e Error) Error() string {
	if e.Pos.Filename != "" || e.Pos.IsValid() {
		// don't print "<unknown position>"
		// TODO(gri) reconsider the semantics of Position.IsValid
		return e.Pos.String() + ": " + e.Msg
	}
	return e.Msg
}

// ErrorList is a list of *Errors.
// The zero value for an ErrorList is an empty ErrorList ready to use.
//
type ErrorList []*Error

// Add adds an Error with given position and error message to an ErrorList.
func (p *ErrorList) Add(pos token.Position, msg string) {
	*p = append(*p, &Error{pos, msg})
}

// Reset resets an ErrorList to no errors.
func (p *ErrorList) Reset() { *p = (*p)[0:0] }

// ErrorList implements the sort Interface.
func (p ErrorList) Len() int      { return len(p) }
func (p ErrorList) Swap(i, j int) { p[i], p[j] = p[j], p[i] }

func (p ErrorList) Less(i, j int) bool {
	e := &p[i].Pos
	f := &p[j].Pos
	// Note that it is not sufficient to simply compare file offsets because
	// the offsets do not reflect modified line information (through //line
	// comments).
	if e.Filename < f.Filename {
		return true
	}
	if e.Filename == f.Filename {
		if e.Line < f.Line {
			return true
		}
		if e.Line == f.Line {
			return e.Column < f.Column
		}
	}
	return false
}

// Sort sorts an ErrorList. *Error entries are sorted by position,
// other errors are sorted by error message, and before any *Error
// entry.
//
func (p ErrorList) Sort() {
	sort.Sort(p)
}

// RemoveMultiples sorts an ErrorList and removes all but the first error per line.
func (p *ErrorList) RemoveMultiples() {
	sort.Sort(p)
	var last token.Position // initial last.Line is != any legal error line
	i := 0
	for _, e := range *p {
		if e.Pos.Filename != last.Filename || e.Pos.Line != last.Line {
			last = e.Pos
			(*p)[i] = e
			i++
		}
	}
	(*p) = (*p)[0:i]
}

// An ErrorList implements the error interface.
func (p ErrorList) Error() string {
	switch len(p) {
	case 0:
		return "no errors"
	case 1:
		return p[0].Error()
	}
	return fmt.Sprintf("%s (and %d more errors)", p[0], len(p)-1)
}

// Err returns an error equivalent to this error list.
// If the list is empty, Err returns nil.
func (p ErrorList) Err() error {
	if len(p) == 0 {
		return nil
	}
	return p
}

// PrintError is a utility function that prints a list of errors to w,
// one error per line, if the err parameter is an ErrorList. Otherwise
// it prints the err string.
//
func PrintError(w io.Writer, err error) {
	if list, ok := err.(ErrorList); ok {
		for _, e := range list {
			fmt.Fprintf(w, "%s\n", e)
		}
	} else {
		fmt.Fprintf(w, "%s\n", err)
	}
}
<commit_msg>go/scanner: don't print garbage if there's no error<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package scanner

import (
	"fmt"
	"go/token"
	"io"
	"sort"
)

// In an ErrorList, an error is represented by an *Error.
// The position Pos, if valid, points to the beginning of
// the offending token, and the error condition is described
// by Msg.
//
type Error struct {
	Pos token.Position
	Msg string
}

// Error implements the error interface.
func (e Error) Error() string {
	if e.Pos.Filename != "" || e.Pos.IsValid() {
		// don't print "<unknown position>"
		// TODO(gri) reconsider the semantics of Position.IsValid
		return e.Pos.String() + ": " + e.Msg
	}
	return e.Msg
}

// ErrorList is a list of *Errors.
// The zero value for an ErrorList is an empty ErrorList ready to use.
//
type ErrorList []*Error

// Add adds an Error with given position and error message to an ErrorList.
func (p *ErrorList) Add(pos token.Position, msg string) {
	*p = append(*p, &Error{pos, msg})
}

// Reset resets an ErrorList to no errors.
func (p *ErrorList) Reset() { *p = (*p)[0:0] }

// ErrorList implements the sort Interface.
func (p ErrorList) Len() int      { return len(p) }
func (p ErrorList) Swap(i, j int) { p[i], p[j] = p[j], p[i] }

func (p ErrorList) Less(i, j int) bool {
	e := &p[i].Pos
	f := &p[j].Pos
	// Note that it is not sufficient to simply compare file offsets because
	// the offsets do not reflect modified line information (through //line
	// comments).
	if e.Filename < f.Filename {
		return true
	}
	if e.Filename == f.Filename {
		if e.Line < f.Line {
			return true
		}
		if e.Line == f.Line {
			return e.Column < f.Column
		}
	}
	return false
}

// Sort sorts an ErrorList. *Error entries are sorted by position,
// other errors are sorted by error message, and before any *Error
// entry.
//
func (p ErrorList) Sort() {
	sort.Sort(p)
}

// RemoveMultiples sorts an ErrorList and removes all but the first error per line.
func (p *ErrorList) RemoveMultiples() {
	sort.Sort(p)
	var last token.Position // initial last.Line is != any legal error line
	i := 0
	for _, e := range *p {
		if e.Pos.Filename != last.Filename || e.Pos.Line != last.Line {
			last = e.Pos
			(*p)[i] = e
			i++
		}
	}
	(*p) = (*p)[0:i]
}

// An ErrorList implements the error interface.
func (p ErrorList) Error() string {
	switch len(p) {
	case 0:
		return "no errors"
	case 1:
		return p[0].Error()
	}
	return fmt.Sprintf("%s (and %d more errors)", p[0], len(p)-1)
}

// Err returns an error equivalent to this error list.
// If the list is empty, Err returns nil.
func (p ErrorList) Err() error {
	if len(p) == 0 {
		return nil
	}
	return p
}

// PrintError is a utility function that prints a list of errors to w,
// one error per line, if the err parameter is an ErrorList. Otherwise
// it prints the err string.
//
func PrintError(w io.Writer, err error) {
	if list, ok := err.(ErrorList); ok {
		for _, e := range list {
			fmt.Fprintf(w, "%s\n", e)
		}
	} else if err != nil {
		fmt.Fprintf(w, "%s\n", err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Network interface identification for BSD variants

package net

import (
	"os"
	"syscall"
)

// IsUp returns true if ifi is up.
func (ifi *Interface) IsUp() bool {
	if ifi == nil {
		return false
	}
	return ifi.rawFlags&syscall.IFF_UP != 0
}

// IsLoopback returns true if ifi is a loopback interface.
func (ifi *Interface) IsLoopback() bool {
	if ifi == nil {
		return false
	}
	return ifi.rawFlags&syscall.IFF_LOOPBACK != 0
}

// CanBroadcast returns true if ifi supports a broadcast access
// capability.
func (ifi *Interface) CanBroadcast() bool {
	if ifi == nil {
		return false
	}
	return ifi.rawFlags&syscall.IFF_BROADCAST != 0
}

// IsPointToPoint returns true if ifi belongs to a point-to-point
// link.
func (ifi *Interface) IsPointToPoint() bool {
	if ifi == nil {
		return false
	}
	return ifi.rawFlags&syscall.IFF_POINTOPOINT != 0
}

// CanMulticast returns true if ifi supports a multicast access
// capability.
func (ifi *Interface) CanMulticast() bool {
	if ifi == nil {
		return false
	}
	return ifi.rawFlags&syscall.IFF_MULTICAST != 0
}

// If the ifindex is zero, interfaceTable returns mappings of all
// network interfaces.  Otheriwse it returns a mapping of a specific
// interface.
func interfaceTable(ifindex int) ([]Interface, os.Error) {
	var (
		tab  []byte
		e    int
		msgs []syscall.RoutingMessage
		ift  []Interface
	)

	tab, e = syscall.RouteRIB(syscall.NET_RT_IFLIST, ifindex)
	if e != 0 {
		return nil, os.NewSyscallError("route rib", e)
	}

	msgs, e = syscall.ParseRoutingMessage(tab)
	if e != 0 {
		return nil, os.NewSyscallError("route message", e)
	}

	for _, m := range msgs {
		switch v := m.(type) {
		case *syscall.InterfaceMessage:
			if ifindex == 0 || ifindex == int(v.Header.Index) {
				ifi, err := newLink(v)
				if err != nil {
					return nil, err
				}
				ift = append(ift, ifi...)
			}
		}
	}

	return ift, nil
}

func newLink(m *syscall.InterfaceMessage) ([]Interface, os.Error) {
	var ift []Interface

	sas, e := syscall.ParseRoutingSockaddr(m)
	if e != 0 {
		return nil, os.NewSyscallError("route sockaddr", e)
	}

	for _, s := range sas {
		switch v := s.(type) {
		case *syscall.SockaddrDatalink:
			ifi := Interface{Index: int(m.Header.Index), rawFlags: int(m.Header.Flags)}
			var name [syscall.IFNAMSIZ]byte
			for i := 0; i < int(v.Nlen); i++ {
				name[i] = byte(v.Data[i])
			}
			ifi.Name = string(name[:v.Nlen])
			ifi.MTU = int(m.Header.Data.Mtu)
			addr := make([]byte, v.Alen)
			for i := 0; i < int(v.Alen); i++ {
				addr[i] = byte(v.Data[int(v.Nlen)+i])
			}
			ifi.HardwareAddr = addr[:v.Alen]
			ift = append(ift, ifi)
		}
	}

	return ift, nil
}

// If the ifindex is zero, interfaceAddrTable returns addresses
// for all network interfaces.  Otherwise it returns addresses
// for a specific interface.
func interfaceAddrTable(ifindex int) ([]Addr, os.Error) {
	var (
		tab  []byte
		e    int
		msgs []syscall.RoutingMessage
		ifat []Addr
	)

	tab, e = syscall.RouteRIB(syscall.NET_RT_IFLIST, ifindex)
	if e != 0 {
		return nil, os.NewSyscallError("route rib", e)
	}

	msgs, e = syscall.ParseRoutingMessage(tab)
	if e != 0 {
		return nil, os.NewSyscallError("route message", e)
	}

	for _, m := range msgs {
		switch v := m.(type) {
		case *syscall.InterfaceAddrMessage:
			if ifindex == 0 || ifindex == int(v.Header.Index) {
				ifa, err := newAddr(v)
				if err != nil {
					return nil, err
				}
				ifat = append(ifat, ifa...)
			}
		}
	}

	return ifat, nil
}

func newAddr(m *syscall.InterfaceAddrMessage) ([]Addr, os.Error) {
	var ifat []Addr

	sas, e := syscall.ParseRoutingSockaddr(m)
	if e != 0 {
		return nil, os.NewSyscallError("route sockaddr", e)
	}

	for _, s := range sas {
		var ifa IPAddr
		switch v := s.(type) {
		case *syscall.SockaddrInet4:
			ifa.IP = IPv4(v.Addr[0], v.Addr[1], v.Addr[2], v.Addr[3])
		case *syscall.SockaddrInet6:
			ifa.IP = make(IP, IPv6len)
			copy(ifa.IP, v.Addr[:])
			// NOTE: KAME based IPv6 protcol stack usually embeds
			// the interface index in the interface-local or link-
			// local address as the kernel-internal form.
			if ifa.IP.IsLinkLocalUnicast() ||
				ifa.IP.IsInterfaceLocalMulticast() ||
				ifa.IP.IsLinkLocalMulticast() {
				// remove embedded scope zone ID
				ifa.IP[2], ifa.IP[3] = 0, 0
			}
		}
		ifat = append(ifat, ifa.toAddr())
	}

	return ifat, nil
}
<commit_msg>net: fix bug in net.Interfaces: handle elastic sdl_data size correctly<commit_after>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Network interface identification for BSD variants

package net

import (
	"os"
	"syscall"
	"unsafe"
)

// IsUp returns true if ifi is up.
func (ifi *Interface) IsUp() bool {
	if ifi == nil {
		return false
	}
	return ifi.rawFlags&syscall.IFF_UP != 0
}

// IsLoopback returns true if ifi is a loopback interface.
func (ifi *Interface) IsLoopback() bool {
	if ifi == nil {
		return false
	}
	return ifi.rawFlags&syscall.IFF_LOOPBACK != 0
}

// CanBroadcast returns true if ifi supports a broadcast access
// capability.
func (ifi *Interface) CanBroadcast() bool {
	if ifi == nil {
		return false
	}
	return ifi.rawFlags&syscall.IFF_BROADCAST != 0
}

// IsPointToPoint returns true if ifi belongs to a point-to-point
// link.
func (ifi *Interface) IsPointToPoint() bool {
	if ifi == nil {
		return false
	}
	return ifi.rawFlags&syscall.IFF_POINTOPOINT != 0
}

// CanMulticast returns true if ifi supports a multicast access
// capability.
func (ifi *Interface) CanMulticast() bool {
	if ifi == nil {
		return false
	}
	return ifi.rawFlags&syscall.IFF_MULTICAST != 0
}

// If the ifindex is zero, interfaceTable returns mappings of all
// network interfaces.  Otheriwse it returns a mapping of a specific
// interface.
func interfaceTable(ifindex int) ([]Interface, os.Error) {
	var (
		tab  []byte
		e    int
		msgs []syscall.RoutingMessage
		ift  []Interface
	)

	tab, e = syscall.RouteRIB(syscall.NET_RT_IFLIST, ifindex)
	if e != 0 {
		return nil, os.NewSyscallError("route rib", e)
	}

	msgs, e = syscall.ParseRoutingMessage(tab)
	if e != 0 {
		return nil, os.NewSyscallError("route message", e)
	}

	for _, m := range msgs {
		switch v := m.(type) {
		case *syscall.InterfaceMessage:
			if ifindex == 0 || ifindex == int(v.Header.Index) {
				ifi, err := newLink(v)
				if err != nil {
					return nil, err
				}
				ift = append(ift, ifi...)
			}
		}
	}

	return ift, nil
}

func newLink(m *syscall.InterfaceMessage) ([]Interface, os.Error) {
	var ift []Interface

	sas, e := syscall.ParseRoutingSockaddr(m)
	if e != 0 {
		return nil, os.NewSyscallError("route sockaddr", e)
	}

	for _, s := range sas {
		switch v := s.(type) {
		case *syscall.SockaddrDatalink:
			// NOTE: SockaddrDatalink.Data is minimum work area,
			// can be larger.
			m.Data = m.Data[unsafe.Offsetof(v.Data):]
			ifi := Interface{Index: int(m.Header.Index), rawFlags: int(m.Header.Flags)}
			var name [syscall.IFNAMSIZ]byte
			for i := 0; i < int(v.Nlen); i++ {
				name[i] = byte(m.Data[i])
			}
			ifi.Name = string(name[:v.Nlen])
			ifi.MTU = int(m.Header.Data.Mtu)
			addr := make([]byte, v.Alen)
			for i := 0; i < int(v.Alen); i++ {
				addr[i] = byte(m.Data[int(v.Nlen)+i])
			}
			ifi.HardwareAddr = addr[:v.Alen]
			ift = append(ift, ifi)
		}
	}

	return ift, nil
}

// If the ifindex is zero, interfaceAddrTable returns addresses
// for all network interfaces.  Otherwise it returns addresses
// for a specific interface.
func interfaceAddrTable(ifindex int) ([]Addr, os.Error) {
	var (
		tab  []byte
		e    int
		msgs []syscall.RoutingMessage
		ifat []Addr
	)

	tab, e = syscall.RouteRIB(syscall.NET_RT_IFLIST, ifindex)
	if e != 0 {
		return nil, os.NewSyscallError("route rib", e)
	}

	msgs, e = syscall.ParseRoutingMessage(tab)
	if e != 0 {
		return nil, os.NewSyscallError("route message", e)
	}

	for _, m := range msgs {
		switch v := m.(type) {
		case *syscall.InterfaceAddrMessage:
			if ifindex == 0 || ifindex == int(v.Header.Index) {
				ifa, err := newAddr(v)
				if err != nil {
					return nil, err
				}
				ifat = append(ifat, ifa...)
			}
		}
	}

	return ifat, nil
}

func newAddr(m *syscall.InterfaceAddrMessage) ([]Addr, os.Error) {
	var ifat []Addr

	sas, e := syscall.ParseRoutingSockaddr(m)
	if e != 0 {
		return nil, os.NewSyscallError("route sockaddr", e)
	}

	for _, s := range sas {
		var ifa IPAddr
		switch v := s.(type) {
		case *syscall.SockaddrInet4:
			ifa.IP = IPv4(v.Addr[0], v.Addr[1], v.Addr[2], v.Addr[3])
		case *syscall.SockaddrInet6:
			ifa.IP = make(IP, IPv6len)
			copy(ifa.IP, v.Addr[:])
			// NOTE: KAME based IPv6 protcol stack usually embeds
			// the interface index in the interface-local or link-
			// local address as the kernel-internal form.
			if ifa.IP.IsLinkLocalUnicast() ||
				ifa.IP.IsInterfaceLocalMulticast() ||
				ifa.IP.IsLinkLocalMulticast() {
				// remove embedded scope zone ID
				ifa.IP[2], ifa.IP[3] = 0, 0
			}
		}
		ifat = append(ifat, ifa.toAddr())
	}

	return ifat, nil
}
<|endoftext|>