<commit_before>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Download remote packages.

package main

import (
	"http"
	"os"
	"path"
	"regexp"
	"strings"
)

const dashboardURL = "http://godashboard.appspot.com/package"

// maybeReportToDashboard reports path to dashboard unless
// -dashboard=false is on command line.  It ignores errors.
func maybeReportToDashboard(path string) {
	// if -dashboard=false was on command line, do nothing
	if !*reportToDashboard {
		return
	}

	// otherwise lob url to dashboard
	r, _ := http.Post(dashboardURL, "application/x-www-form-urlencoded", strings.NewReader("path="+path))
	if r != nil && r.Body != nil {
		r.Body.Close()
	}
}

var googlecode = regexp.MustCompile(`^([a-z0-9\-]+\.googlecode\.com/(svn|hg))(/[a-z0-9A-Z_.\-/]*)?$`)
var github = regexp.MustCompile(`^(github\.com/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]*)?$`)
var bitbucket = regexp.MustCompile(`^(bitbucket\.org/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]*)?$`)
var launchpad = regexp.MustCompile(`^(launchpad\.net/([a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+))(/[a-z0-9A-Z_.\-/]+)?$`)

// download checks out or updates pkg from the remote server.
func download(pkg string) (string, os.Error) {
	if strings.Contains(pkg, "..") {
		return "", os.ErrorString("invalid path (contains ..)")
	}
	if m := bitbucket.FindStringSubmatch(pkg); m != nil {
		if err := vcsCheckout(&hg, root+m[1], "http://"+m[1], m[1]); err != nil {
			return "", err
		}
		return root + pkg, nil
	}
	if m := googlecode.FindStringSubmatch(pkg); m != nil {
		var v *vcs
		switch m[2] {
		case "hg":
			v = &hg
		case "svn":
			v = &svn
		default:
			// regexp only allows hg, svn to get through
			panic("missing case in download: " + pkg)
		}
		if err := vcsCheckout(v, root+m[1], "https://"+m[1], m[1]); err != nil {
			return "", err
		}
		return root + pkg, nil
	}
	if m := github.FindStringSubmatch(pkg); m != nil {
		if strings.HasSuffix(m[1], ".git") {
			return "", os.ErrorString("repository " + pkg + " should not have .git suffix")
		}
		if err := vcsCheckout(&git, root+m[1], "http://"+m[1]+".git", m[1]); err != nil {
			return "", err
		}
		return root + pkg, nil
	}
	if m := launchpad.FindStringSubmatch(pkg); m != nil {
		// Either lp.net/<project>[/<series>[/<path>]]
		//	 or lp.net/~<user or team>/<project>/<branch>[/<path>]
		if err := vcsCheckout(&bzr, root+m[1], "https://"+m[1], m[1]); err != nil {
			return "", err
		}
		return root + pkg, nil
	}
	return "", os.ErrorString("unknown repository: " + pkg)
}

// a vcs represents a version control system
// like Mercurial, Git, or Subversion.
type vcs struct {
	cmd               string
	metadir           string
	checkout          string
	clone             string
	update            string
	updateReleaseFlag string
	pull              string
	pullForceFlag     string
	log               string
	logLimitFlag      string
	logReleaseFlag    string
}

var hg = vcs{
	cmd:               "hg",
	metadir:           ".hg",
	checkout:          "checkout",
	clone:             "clone",
	update:            "update",
	updateReleaseFlag: "release",
	pull:              "pull",
	log:               "log",
	logLimitFlag:      "-l1",
	logReleaseFlag:    "-rrelease",
}

var git = vcs{
	cmd:               "git",
	metadir:           ".git",
	checkout:          "checkout",
	clone:             "clone",
	update:            "pull",
	updateReleaseFlag: "release",
	pull:              "fetch",
	log:               "show-ref",
	logLimitFlag:      "",
	logReleaseFlag:    "release",
}

var svn = vcs{
	cmd:               "svn",
	metadir:           ".svn",
	checkout:          "checkout",
	clone:             "checkout",
	update:            "update",
	updateReleaseFlag: "release",
	log:               "log",
	logLimitFlag:      "-l1",
	logReleaseFlag:    "release",
}

var bzr = vcs{
	cmd:               "bzr",
	metadir:           ".bzr",
	checkout:          "checkout",
	clone:             "branch",
	update:            "update",
	updateReleaseFlag: "-rrelease",
	pull:              "pull",
	pullForceFlag:     "--overwrite",
	log:               "log",
	logLimitFlag:      "-l1",
	logReleaseFlag:    "-rrelease",
}

// Try to detect if a "release" tag exists.  If it does, update
// to the tagged version, otherwise just update the current branch.
// NOTE(_nil): svn will always fail because it is trying to get
// the revision history of a file named "release" instead of
// looking for a commit with a release tag
func (v *vcs) updateRepo(dst string) os.Error {
	if err := quietRun(dst, nil, v.cmd, v.log, v.logLimitFlag, v.logReleaseFlag); err == nil {
		if err := run(dst, nil, v.cmd, v.checkout, v.updateReleaseFlag); err != nil {
			return err
		}
	} else if err := run(dst, nil, v.cmd, v.update); err != nil {
		return err
	}
	return nil
}

// vcsCheckout checks out repo into dst using vcs.
// It tries to check out (or update, if the dst already
// exists and -u was specified on the command line)
// the repository at tag/branch "release".  If there is no
// such tag or branch, it falls back to the repository tip.
func vcsCheckout(vcs *vcs, dst, repo, dashpath string) os.Error {
	dir, err := os.Stat(dst + "/" + vcs.metadir)
	if err == nil && !dir.IsDirectory() {
		return os.ErrorString("not a directory: " + dst)
	}
	if err != nil {
		parent, _ := path.Split(dst)
		if err := os.MkdirAll(parent, 0777); err != nil {
			return err
		}
		if err := run("/", nil, vcs.cmd, vcs.clone, repo, dst); err != nil {
			return err
		}
		if err := vcs.updateRepo(dst); err != nil {
			return err
		}
		// success on first installation - report
		maybeReportToDashboard(dashpath)
	} else if *update {
		// Retrieve new revisions from the remote branch, if the VCS
		// supports this operation independently (e.g. svn doesn't)
		if vcs.pull != "" {
			if vcs.pullForceFlag != "" {
				if err := run(dst, nil, vcs.cmd, vcs.pull, vcs.pullForceFlag); err != nil {
					return err
				}
			} else if err := run(dst, nil, vcs.cmd, vcs.pull); err != nil {
				return err
			}
		}

		// Update to release or latest revision
		if err := vcs.updateRepo(dst); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>goinstall: fix -u for bzr<commit_after>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Download remote packages.

package main

import (
	"http"
	"os"
	"path"
	"regexp"
	"strings"
)

const dashboardURL = "http://godashboard.appspot.com/package"

// maybeReportToDashboard reports path to dashboard unless
// -dashboard=false is on command line.  It ignores errors.
func maybeReportToDashboard(path string) {
	// if -dashboard=false was on command line, do nothing
	if !*reportToDashboard {
		return
	}

	// otherwise lob url to dashboard
	r, _ := http.Post(dashboardURL, "application/x-www-form-urlencoded", strings.NewReader("path="+path))
	if r != nil && r.Body != nil {
		r.Body.Close()
	}
}

var googlecode = regexp.MustCompile(`^([a-z0-9\-]+\.googlecode\.com/(svn|hg))(/[a-z0-9A-Z_.\-/]*)?$`)
var github = regexp.MustCompile(`^(github\.com/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]*)?$`)
var bitbucket = regexp.MustCompile(`^(bitbucket\.org/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]*)?$`)
var launchpad = regexp.MustCompile(`^(launchpad\.net/([a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+))(/[a-z0-9A-Z_.\-/]+)?$`)

// download checks out or updates pkg from the remote server.
func download(pkg string) (string, os.Error) {
	if strings.Contains(pkg, "..") {
		return "", os.ErrorString("invalid path (contains ..)")
	}
	if m := bitbucket.FindStringSubmatch(pkg); m != nil {
		if err := vcsCheckout(&hg, root+m[1], "http://"+m[1], m[1]); err != nil {
			return "", err
		}
		return root + pkg, nil
	}
	if m := googlecode.FindStringSubmatch(pkg); m != nil {
		var v *vcs
		switch m[2] {
		case "hg":
			v = &hg
		case "svn":
			v = &svn
		default:
			// regexp only allows hg, svn to get through
			panic("missing case in download: " + pkg)
		}
		if err := vcsCheckout(v, root+m[1], "https://"+m[1], m[1]); err != nil {
			return "", err
		}
		return root + pkg, nil
	}
	if m := github.FindStringSubmatch(pkg); m != nil {
		if strings.HasSuffix(m[1], ".git") {
			return "", os.ErrorString("repository " + pkg + " should not have .git suffix")
		}
		if err := vcsCheckout(&git, root+m[1], "http://"+m[1]+".git", m[1]); err != nil {
			return "", err
		}
		return root + pkg, nil
	}
	if m := launchpad.FindStringSubmatch(pkg); m != nil {
		// Either lp.net/<project>[/<series>[/<path>]]
		//	 or lp.net/~<user or team>/<project>/<branch>[/<path>]
		if err := vcsCheckout(&bzr, root+m[1], "https://"+m[1], m[1]); err != nil {
			return "", err
		}
		return root + pkg, nil
	}
	return "", os.ErrorString("unknown repository: " + pkg)
}

// a vcs represents a version control system
// like Mercurial, Git, or Subversion.
type vcs struct {
	cmd               string
	metadir           string
	checkout          string
	clone             string
	update            string
	updateReleaseFlag string
	pull              string
	pullForceFlag     string
	log               string
	logLimitFlag      string
	logReleaseFlag    string
}

var hg = vcs{
	cmd:               "hg",
	metadir:           ".hg",
	checkout:          "checkout",
	clone:             "clone",
	update:            "update",
	updateReleaseFlag: "release",
	pull:              "pull",
	log:               "log",
	logLimitFlag:      "-l1",
	logReleaseFlag:    "-rrelease",
}

var git = vcs{
	cmd:               "git",
	metadir:           ".git",
	checkout:          "checkout",
	clone:             "clone",
	update:            "pull",
	updateReleaseFlag: "release",
	pull:              "fetch",
	log:               "show-ref",
	logLimitFlag:      "",
	logReleaseFlag:    "release",
}

var svn = vcs{
	cmd:               "svn",
	metadir:           ".svn",
	checkout:          "checkout",
	clone:             "checkout",
	update:            "update",
	updateReleaseFlag: "release",
	log:               "log",
	logLimitFlag:      "-l1",
	logReleaseFlag:    "release",
}

var bzr = vcs{
	cmd:               "bzr",
	metadir:           ".bzr",
	checkout:          "update",
	clone:             "branch",
	update:            "update",
	updateReleaseFlag: "-rrelease",
	pull:              "pull",
	pullForceFlag:     "--overwrite",
	log:               "log",
	logLimitFlag:      "-l1",
	logReleaseFlag:    "-rrelease",
}

// Try to detect if a "release" tag exists.  If it does, update
// to the tagged version, otherwise just update the current branch.
// NOTE(_nil): svn will always fail because it is trying to get
// the revision history of a file named "release" instead of
// looking for a commit with a release tag
func (v *vcs) updateRepo(dst string) os.Error {
	if err := quietRun(dst, nil, v.cmd, v.log, v.logLimitFlag, v.logReleaseFlag); err == nil {
		if err := run(dst, nil, v.cmd, v.checkout, v.updateReleaseFlag); err != nil {
			return err
		}
	} else if err := run(dst, nil, v.cmd, v.update); err != nil {
		return err
	}
	return nil
}

// vcsCheckout checks out repo into dst using vcs.
// It tries to check out (or update, if the dst already
// exists and -u was specified on the command line)
// the repository at tag/branch "release".  If there is no
// such tag or branch, it falls back to the repository tip.
func vcsCheckout(vcs *vcs, dst, repo, dashpath string) os.Error {
	dir, err := os.Stat(dst + "/" + vcs.metadir)
	if err == nil && !dir.IsDirectory() {
		return os.ErrorString("not a directory: " + dst)
	}
	if err != nil {
		parent, _ := path.Split(dst)
		if err := os.MkdirAll(parent, 0777); err != nil {
			return err
		}
		if err := run("/", nil, vcs.cmd, vcs.clone, repo, dst); err != nil {
			return err
		}
		if err := vcs.updateRepo(dst); err != nil {
			return err
		}
		// success on first installation - report
		maybeReportToDashboard(dashpath)
	} else if *update {
		// Retrieve new revisions from the remote branch, if the VCS
		// supports this operation independently (e.g. svn doesn't)
		if vcs.pull != "" {
			if vcs.pullForceFlag != "" {
				if err := run(dst, nil, vcs.cmd, vcs.pull, vcs.pullForceFlag); err != nil {
					return err
				}
			} else if err := run(dst, nil, vcs.cmd, vcs.pull); err != nil {
				return err
			}
		}

		// Update to release or latest revision
		if err := vcs.updateRepo(dst); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2016-2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"net"
	"strconv"
	"sync"
	"time"

	"github.com/cilium/cilium/common"
	"github.com/cilium/cilium/common/addressing"
	"github.com/cilium/cilium/common/types"
	"github.com/cilium/cilium/pkg/endpoint"
	"github.com/cilium/cilium/pkg/labels"
	"github.com/cilium/cilium/pkg/policy"

	dTypes "github.com/docker/engine-api/types"
	dTypesEvents "github.com/docker/engine-api/types/events"
	ctx "golang.org/x/net/context"
	k8sDockerLbls "k8s.io/client-go/1.5/pkg/kubelet/types"
)

const (
	syncRateDocker = time.Duration(30 * time.Second)

	maxRetries = 3
)

// EnableDockerEventListener watches for docker events. Performs the plumbing for the
// containers started or dead.
func (d *Daemon) EnableDockerEventListener(since time.Time) error {
	eo := dTypes.EventsOptions{Since: strconv.FormatInt(since.Unix(), 10)}
	r, err := d.dockerClient.Events(ctx.Background(), eo)
	if err != nil {
		return err
	}
	log.Debugf("Listening for docker events")
	go d.listenForDockerEvents(r)
	return nil
}

func (d *Daemon) SyncDocker(wg *sync.WaitGroup) {
	cList, err := d.dockerClient.ContainerList(ctx.Background(), dTypes.ContainerListOptions{All: false})
	if err != nil {
		log.Errorf("Failed to retrieve the container list %s", err)
	}
	for _, cont := range cList {
		if d.IgnoredContainer(cont.ID) {
			continue
		}

		wg.Add(1)
		go func(wg *sync.WaitGroup, id string) {
			d.handleCreateContainer(id)
			wg.Done()
		}(wg, cont.ID)
	}
}

func (d *Daemon) EnableDockerSync() {
	var wg sync.WaitGroup
	for {
		d.SyncDocker(&wg)
		wg.Wait()
		time.Sleep(syncRateDocker)
	}
}

func (d *Daemon) listenForDockerEvents(reader io.ReadCloser) {
	scanner := bufio.NewScanner(reader)
	for scanner.Scan() {
		var e dTypesEvents.Message
		if err := json.Unmarshal(scanner.Bytes(), &e); err != nil {
			log.Errorf("Error while unmarshalling event: %+v", e)
		}
		log.Debugf("Processing an event %+v", e)
		go d.processEvent(e)
	}
	if err := scanner.Err(); err != nil {
		log.Errorf("Error while reading events: %+v", err)
	}
}

func (d *Daemon) processEvent(m dTypesEvents.Message) {
	if m.Type == "container" {
		switch m.Status {
		case "start":
			// A real event overwrites any memory of ignored containers
			d.StopIgnoringContainer(m.ID)
			d.handleCreateContainer(m.ID)
		case "die":
			d.deleteContainer(m.ID)
		}
	}
}

func getCiliumEndpointID(cont *dTypes.ContainerJSON, gwIP *addressing.NodeAddress) uint16 {
	for _, contNetwork := range cont.NetworkSettings.Networks {
		ipv6gw := net.ParseIP(contNetwork.IPv6Gateway)
		if ipv6gw.Equal(gwIP.IPv6Address.IP()) {
			ip, err := addressing.NewCiliumIPv6(contNetwork.GlobalIPv6Address)
			if err == nil {
				return ip.EndpointID()
			}
		}
	}

	return 0
}

func (d *Daemon) fetchK8sLabels(dockerLbls map[string]string) (map[string]string, error) {
	if !d.conf.IsK8sEnabled() {
		return nil, nil
	}
	ns := k8sDockerLbls.GetPodNamespace(dockerLbls)
	if ns == "" {
		ns = "default"
	}
	podName := k8sDockerLbls.GetPodName(dockerLbls)
	if podName == "" {
		return nil, nil
	}
	log.Debugf("Connecting to kubernetes to retrieve labels for pod %s ns %s", podName, ns)

	result, err := d.k8sClient.Pods(ns).Get(podName)
	if err != nil {
		return nil, err
	}
	k8sLabels := result.GetLabels()
	k8sLabels[common.K8sPodNamespaceLabel] = ns
	return k8sLabels, nil
}

func (d *Daemon) getFilteredLabels(allLabels map[string]string) labels.Labels {
	var ciliumLabels, k8sLabels, k8sSpecialLabels labels.Labels
	if podName := k8sDockerLbls.GetPodName(allLabels); podName != "" {
		k8sNormalLabels, err := d.fetchK8sLabels(allLabels)
		if err != nil {
			log.Warningf("Error while getting kubernetes labels: %s", err)
		} else if k8sNormalLabels != nil {
			k8sLabels = labels.Map2Labels(k8sNormalLabels, common.K8sLabelSource)

			// Transform all labels "k8s-app" and "version" to "io.cilium.k8s.k8s-app"
			// and  "io.cilium.k8s.version"
			d.conf.ValidLabelPrefixesMU.RLock()
			k8sSpecialLabels = d.conf.ValidK8sLabelPrefixes.FilterLabels(k8sLabels)
			d.conf.ValidLabelPrefixesMU.RUnlock()
			k8sSpecialLabels = k8sSpecialLabels.AppendPrefixInKey(common.K8sLabelPrefix)
			log.Debug("Special labels %s", k8sSpecialLabels)
		}
	}

	ciliumLabels = labels.Map2Labels(allLabels, common.CiliumLabelSource)

	ciliumLabels.MergeLabels(k8sLabels)

	d.conf.ValidLabelPrefixesMU.RLock()
	normalLabels := d.conf.ValidLabelPrefixes.FilterLabels(ciliumLabels)
	d.conf.ValidLabelPrefixesMU.RUnlock()

	normalLabels.MergeLabels(k8sSpecialLabels)
	return normalLabels
}

func (d *Daemon) retrieveWorkingContainerCopy(id string) (types.Container, bool) {
	d.containersMU.RLock()
	defer d.containersMU.RUnlock()

	if c, ok := d.containers[id]; ok {
		return *c, true
	} else {
		return types.Container{}, false
	}
}

func createContainer(dc *dTypes.ContainerJSON, l labels.Labels) types.Container {
	return types.Container{
		ContainerJSON: *dc,
		OpLabels: labels.OpLabels{
			Custom:        labels.Labels{},
			Disabled:      labels.Labels{},
			Orchestration: l.DeepCopy(),
		},
	}
}

func (d *Daemon) handleCreateContainer(id string) {
	log.Debugf("Processing create event for docker container %s", id)

	maxTries := 5

	for try := 1; try <= maxTries; try++ {
		if try > 1 {
			log.Debugf("Waiting for container %s to appear as endpoint [%d/%d]",
				id, try, maxTries)
			time.Sleep(time.Duration(try) * time.Second)
		}

		dockerContainer, lbls, err := d.retrieveDockerLabels(id)
		if err != nil {
			log.Warningf("unable to inspect container %d, retrying later (%s)", id, err)
			continue
		}

		dockerEpID := ""
		if dockerContainer.NetworkSettings != nil {
			dockerEpID = dockerContainer.NetworkSettings.EndpointID
		}

		d.endpointsMU.Lock()
		ep := d.lookupDockerID(id)
		if ep == nil {
			// container id is yet unknown, try and find endpoint via
			// the IP address assigned.
			cid := getCiliumEndpointID(dockerContainer, d.conf.NodeAddress)
			if cid != 0 {
				ep = d.lookupCiliumEndpoint(cid)
				if ep != nil {
					// Associate container id with endpoint
					ep.DockerID = id
					d.linkContainerID(ep)
				}
			}
		}

		d.endpointsMU.Unlock()
		if ep == nil {
			// Endpoint does not exist yet. This indicates that the
			// orchestration system has not requested us to handle
			// networking for this container yet (or never will). We
			// will retry a couple of times to wait for this to
			// happen.
			continue
		}

		containerCopy, ok := d.retrieveWorkingContainerCopy(id)
		if ok {
			if !updateOrchLabels(&containerCopy, lbls) {
				log.Debugf("No changes to orch labels, ignoring")
				return
			}
		} else {
			containerCopy = createContainer(dockerContainer, lbls)
		}

		identity, err := d.updateContainerIdentity(&containerCopy)
		if err != nil {
			log.Warningf("unable to update identity of container %s: %s", id, err)
			return
		}

		// FIXME:

		d.endpointsMU.Lock()
		ep = d.lookupDockerID(id)
		if ep == nil {
			d.endpointsMU.Unlock()
			log.Warningf("endpoint disappeared while processing event for %s, ignoring", id)
			return
		}

		d.containersMU.Lock()

		// If the container ID was known and found before, check if it still
		// exists, it may have disappared while we gave up the containers
		// lock to create/udpate the identity.
		if ok && d.containers[ep.DockerID] == nil {
			// endpoint is around but container id was removed, likely
			// a bug.
			//
			// FIXME: Disconnect endpoint?
			d.endpointsMU.Unlock()
			d.containersMU.Unlock()
			log.Errorf("BUG: unrefered container %s with endpoint %d present",
				id, ep.ID)
			return
		}

		// Commit label changes to container
		d.containers[ep.DockerID] = &containerCopy

		d.setEndpointIdentity(ep, containerCopy.ID, dockerEpID, identity)
		if err := ep.Regenerate(d); err != nil {
			// FIXME: Disconnect endpoint?
		}

		d.endpointsMU.Unlock()
		d.containersMU.Unlock()

		// FIXME: Does this rebuild epID twice?
		d.triggerPolicyUpdates([]policy.NumericIdentity{identity.ID})
		return
	}

	d.StartIgnoringContainer(id)
	log.Infof("Container %s did not appear as endpoint. Likely managed by other plugin", id)
}

func (d *Daemon) retrieveDockerLabels(dockerID string) (*dTypes.ContainerJSON, labels.Labels, error) {
	dockerCont, err := d.dockerClient.ContainerInspect(ctx.Background(), dockerID)
	if err != nil {
		return nil, nil, fmt.Errorf("unable to inspect container '%s': %s", dockerID, err)
	}

	newLabels := labels.Labels{}
	if dockerCont.Config != nil {
		newLabels = d.getFilteredLabels(dockerCont.Config.Labels)
	}

	return &dockerCont, newLabels, nil
}

func updateOrchLabels(c *types.Container, l labels.Labels) bool {
	changed := false

	c.OpLabels.Orchestration.MarkAllForDeletion()
	c.OpLabels.Disabled.MarkAllForDeletion()

	for k, v := range l {
		if c.OpLabels.Disabled[k] != nil {
			c.OpLabels.Disabled[k].DeletionMark = false
		} else {
			if c.OpLabels.Orchestration[k] != nil {
				c.OpLabels.Orchestration[k].DeletionMark = false
			} else {
				tmp := *v
				log.Debugf("Assigning orchestration label %+v", tmp)
				c.OpLabels.Orchestration[k] = &tmp
				changed = true
			}
		}
	}

	if c.OpLabels.Orchestration.DeleteMarked() || c.OpLabels.Disabled.DeleteMarked() {
		changed = true
	}

	return changed
}

func (d *Daemon) deleteContainer(dockerID string) {
	log.Debugf("Processing deletion event for docker container %s", dockerID)

	d.containersMU.Lock()
	if container, ok := d.containers[dockerID]; ok {
		sha256sum := container.OpLabels.Enabled().SHA256Sum()
		if err := d.DeleteIdentityBySHA256(sha256sum, dockerID); err != nil {
			log.Errorf("Error while deleting labels (SHA256SUM:%s) %+v: %s",
				sha256sum, container.OpLabels.Enabled(), err)
		}

		delete(d.containers, dockerID)
	}
	d.containersMU.Unlock()

	d.DeleteEndpoint(endpoint.NewID(endpoint.ContainerIdPrefix, dockerID))
}
<commit_msg>daemon: typo fix from %d to %s<commit_after>// Copyright 2016-2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"net"
	"strconv"
	"sync"
	"time"

	"github.com/cilium/cilium/common"
	"github.com/cilium/cilium/common/addressing"
	"github.com/cilium/cilium/common/types"
	"github.com/cilium/cilium/pkg/endpoint"
	"github.com/cilium/cilium/pkg/labels"
	"github.com/cilium/cilium/pkg/policy"

	dTypes "github.com/docker/engine-api/types"
	dTypesEvents "github.com/docker/engine-api/types/events"
	ctx "golang.org/x/net/context"
	k8sDockerLbls "k8s.io/client-go/1.5/pkg/kubelet/types"
)

const (
	syncRateDocker = time.Duration(30 * time.Second)

	maxRetries = 3
)

// EnableDockerEventListener watches for docker events. Performs the plumbing for the
// containers started or dead.
func (d *Daemon) EnableDockerEventListener(since time.Time) error {
	eo := dTypes.EventsOptions{Since: strconv.FormatInt(since.Unix(), 10)}
	r, err := d.dockerClient.Events(ctx.Background(), eo)
	if err != nil {
		return err
	}
	log.Debugf("Listening for docker events")
	go d.listenForDockerEvents(r)
	return nil
}

func (d *Daemon) SyncDocker(wg *sync.WaitGroup) {
	cList, err := d.dockerClient.ContainerList(ctx.Background(), dTypes.ContainerListOptions{All: false})
	if err != nil {
		log.Errorf("Failed to retrieve the container list %s", err)
	}
	for _, cont := range cList {
		if d.IgnoredContainer(cont.ID) {
			continue
		}

		wg.Add(1)
		go func(wg *sync.WaitGroup, id string) {
			d.handleCreateContainer(id)
			wg.Done()
		}(wg, cont.ID)
	}
}

func (d *Daemon) EnableDockerSync() {
	var wg sync.WaitGroup
	for {
		d.SyncDocker(&wg)
		wg.Wait()
		time.Sleep(syncRateDocker)
	}
}

func (d *Daemon) listenForDockerEvents(reader io.ReadCloser) {
	scanner := bufio.NewScanner(reader)
	for scanner.Scan() {
		var e dTypesEvents.Message
		if err := json.Unmarshal(scanner.Bytes(), &e); err != nil {
			log.Errorf("Error while unmarshalling event: %+v", e)
		}
		log.Debugf("Processing an event %+v", e)
		go d.processEvent(e)
	}
	if err := scanner.Err(); err != nil {
		log.Errorf("Error while reading events: %+v", err)
	}
}

func (d *Daemon) processEvent(m dTypesEvents.Message) {
	if m.Type == "container" {
		switch m.Status {
		case "start":
			// A real event overwrites any memory of ignored containers
			d.StopIgnoringContainer(m.ID)
			d.handleCreateContainer(m.ID)
		case "die":
			d.deleteContainer(m.ID)
		}
	}
}

func getCiliumEndpointID(cont *dTypes.ContainerJSON, gwIP *addressing.NodeAddress) uint16 {
	for _, contNetwork := range cont.NetworkSettings.Networks {
		ipv6gw := net.ParseIP(contNetwork.IPv6Gateway)
		if ipv6gw.Equal(gwIP.IPv6Address.IP()) {
			ip, err := addressing.NewCiliumIPv6(contNetwork.GlobalIPv6Address)
			if err == nil {
				return ip.EndpointID()
			}
		}
	}

	return 0
}

func (d *Daemon) fetchK8sLabels(dockerLbls map[string]string) (map[string]string, error) {
	if !d.conf.IsK8sEnabled() {
		return nil, nil
	}
	ns := k8sDockerLbls.GetPodNamespace(dockerLbls)
	if ns == "" {
		ns = "default"
	}
	podName := k8sDockerLbls.GetPodName(dockerLbls)
	if podName == "" {
		return nil, nil
	}
	log.Debugf("Connecting to kubernetes to retrieve labels for pod %s ns %s", podName, ns)

	result, err := d.k8sClient.Pods(ns).Get(podName)
	if err != nil {
		return nil, err
	}
	k8sLabels := result.GetLabels()
	k8sLabels[common.K8sPodNamespaceLabel] = ns
	return k8sLabels, nil
}

func (d *Daemon) getFilteredLabels(allLabels map[string]string) labels.Labels {
	var ciliumLabels, k8sLabels, k8sSpecialLabels labels.Labels
	if podName := k8sDockerLbls.GetPodName(allLabels); podName != "" {
		k8sNormalLabels, err := d.fetchK8sLabels(allLabels)
		if err != nil {
			log.Warningf("Error while getting kubernetes labels: %s", err)
		} else if k8sNormalLabels != nil {
			k8sLabels = labels.Map2Labels(k8sNormalLabels, common.K8sLabelSource)

			// Transform all labels "k8s-app" and "version" to "io.cilium.k8s.k8s-app"
			// and  "io.cilium.k8s.version"
			d.conf.ValidLabelPrefixesMU.RLock()
			k8sSpecialLabels = d.conf.ValidK8sLabelPrefixes.FilterLabels(k8sLabels)
			d.conf.ValidLabelPrefixesMU.RUnlock()
			k8sSpecialLabels = k8sSpecialLabels.AppendPrefixInKey(common.K8sLabelPrefix)
			log.Debug("Special labels %s", k8sSpecialLabels)
		}
	}

	ciliumLabels = labels.Map2Labels(allLabels, common.CiliumLabelSource)

	ciliumLabels.MergeLabels(k8sLabels)

	d.conf.ValidLabelPrefixesMU.RLock()
	normalLabels := d.conf.ValidLabelPrefixes.FilterLabels(ciliumLabels)
	d.conf.ValidLabelPrefixesMU.RUnlock()

	normalLabels.MergeLabels(k8sSpecialLabels)
	return normalLabels
}

func (d *Daemon) retrieveWorkingContainerCopy(id string) (types.Container, bool) {
	d.containersMU.RLock()
	defer d.containersMU.RUnlock()

	if c, ok := d.containers[id]; ok {
		return *c, true
	} else {
		return types.Container{}, false
	}
}

func createContainer(dc *dTypes.ContainerJSON, l labels.Labels) types.Container {
	return types.Container{
		ContainerJSON: *dc,
		OpLabels: labels.OpLabels{
			Custom:        labels.Labels{},
			Disabled:      labels.Labels{},
			Orchestration: l.DeepCopy(),
		},
	}
}

func (d *Daemon) handleCreateContainer(id string) {
	log.Debugf("Processing create event for docker container %s", id)

	maxTries := 5

	for try := 1; try <= maxTries; try++ {
		if try > 1 {
			log.Debugf("Waiting for container %s to appear as endpoint [%d/%d]",
				id, try, maxTries)
			time.Sleep(time.Duration(try) * time.Second)
		}

		dockerContainer, lbls, err := d.retrieveDockerLabels(id)
		if err != nil {
			log.Warningf("unable to inspect container %s, retrying later (%s)", id, err)
			continue
		}

		dockerEpID := ""
		if dockerContainer.NetworkSettings != nil {
			dockerEpID = dockerContainer.NetworkSettings.EndpointID
		}

		d.endpointsMU.Lock()
		ep := d.lookupDockerID(id)
		if ep == nil {
			// container id is yet unknown, try and find endpoint via
			// the IP address assigned.
			cid := getCiliumEndpointID(dockerContainer, d.conf.NodeAddress)
			if cid != 0 {
				ep = d.lookupCiliumEndpoint(cid)
				if ep != nil {
					// Associate container id with endpoint
					ep.DockerID = id
					d.linkContainerID(ep)
				}
			}
		}

		d.endpointsMU.Unlock()
		if ep == nil {
			// Endpoint does not exist yet. This indicates that the
			// orchestration system has not requested us to handle
			// networking for this container yet (or never will). We
			// will retry a couple of times to wait for this to
			// happen.
			continue
		}

		containerCopy, ok := d.retrieveWorkingContainerCopy(id)
		if ok {
			if !updateOrchLabels(&containerCopy, lbls) {
				log.Debugf("No changes to orch labels, ignoring")
				return
			}
		} else {
			containerCopy = createContainer(dockerContainer, lbls)
		}

		identity, err := d.updateContainerIdentity(&containerCopy)
		if err != nil {
			log.Warningf("unable to update identity of container %s: %s", id, err)
			return
		}

		// FIXME:

		d.endpointsMU.Lock()
		ep = d.lookupDockerID(id)
		if ep == nil {
			d.endpointsMU.Unlock()
			log.Warningf("endpoint disappeared while processing event for %s, ignoring", id)
			return
		}

		d.containersMU.Lock()

		// If the container ID was known and found before, check if it still
		// exists, it may have disappared while we gave up the containers
		// lock to create/udpate the identity.
		if ok && d.containers[ep.DockerID] == nil {
			// endpoint is around but container id was removed, likely
			// a bug.
			//
			// FIXME: Disconnect endpoint?
			d.endpointsMU.Unlock()
			d.containersMU.Unlock()
			log.Errorf("BUG: unrefered container %s with endpoint %d present",
				id, ep.ID)
			return
		}

		// Commit label changes to container
		d.containers[ep.DockerID] = &containerCopy

		d.setEndpointIdentity(ep, containerCopy.ID, dockerEpID, identity)
		if err := ep.Regenerate(d); err != nil {
			// FIXME: Disconnect endpoint?
		}

		d.endpointsMU.Unlock()
		d.containersMU.Unlock()

		// FIXME: Does this rebuild epID twice?
		d.triggerPolicyUpdates([]policy.NumericIdentity{identity.ID})
		return
	}

	d.StartIgnoringContainer(id)
	log.Infof("Container %s did not appear as endpoint. Likely managed by other plugin", id)
}

func (d *Daemon) retrieveDockerLabels(dockerID string) (*dTypes.ContainerJSON, labels.Labels, error) {
	dockerCont, err := d.dockerClient.ContainerInspect(ctx.Background(), dockerID)
	if err != nil {
		return nil, nil, fmt.Errorf("unable to inspect container '%s': %s", dockerID, err)
	}

	newLabels := labels.Labels{}
	if dockerCont.Config != nil {
		newLabels = d.getFilteredLabels(dockerCont.Config.Labels)
	}

	return &dockerCont, newLabels, nil
}

func updateOrchLabels(c *types.Container, l labels.Labels) bool {
	changed := false

	c.OpLabels.Orchestration.MarkAllForDeletion()
	c.OpLabels.Disabled.MarkAllForDeletion()

	for k, v := range l {
		if c.OpLabels.Disabled[k] != nil {
			c.OpLabels.Disabled[k].DeletionMark = false
		} else {
			if c.OpLabels.Orchestration[k] != nil {
				c.OpLabels.Orchestration[k].DeletionMark = false
			} else {
				tmp := *v
				log.Debugf("Assigning orchestration label %+v", tmp)
				c.OpLabels.Orchestration[k] = &tmp
				changed = true
			}
		}
	}

	if c.OpLabels.Orchestration.DeleteMarked() || c.OpLabels.Disabled.DeleteMarked() {
		changed = true
	}

	return changed
}

func (d *Daemon) deleteContainer(dockerID string) {
	log.Debugf("Processing deletion event for docker container %s", dockerID)

	d.containersMU.Lock()
	if container, ok := d.containers[dockerID]; ok {
		sha256sum := container.OpLabels.Enabled().SHA256Sum()
		if err := d.DeleteIdentityBySHA256(sha256sum, dockerID); err != nil {
			log.Errorf("Error while deleting labels (SHA256SUM:%s) %+v: %s",
				sha256sum, container.OpLabels.Enabled(), err)
		}

		delete(d.containers, dockerID)
	}
	d.containersMU.Unlock()

	d.DeleteEndpoint(endpoint.NewID(endpoint.ContainerIdPrefix, dockerID))
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"os/exec"
	"strconv"
	"strings"
)

var videoSuffixes = []string{".mp4", ".avi", ".mkv", ".flv", ".wmv", ".mov", ".mpg"}

type VideoInfo struct {
	Duration float64 `json:"duration"`
}

func FilenameLooksLikeVideo(name string) bool {
	for _, suffix := range videoSuffixes {
		if strings.HasSuffix(name, suffix) {
			return true
		}
	}
	return false
}

func GetRawFFMPEGInfo(path string) ([]byte, error) {
	debug.Printf("Executing ffprobe for %v", path)
	cmd := exec.Command("./tools/ffprobe", "-v", "quiet", "-print_format", "json", "-show_format", ""+path+"")
	data, err := cmd.Output()
	if err != nil {
		return nil, fmt.Errorf("Error executing ffprobe for file '%v':", path, err)
	}
	return data, nil
}

func GetFFMPEGJson(path string) (map[string]interface{}, error) {
	data, cmderr := GetRawFFMPEGInfo(path)
	if cmderr != nil {
		return nil, cmderr
	}
	var info map[string]interface{}
	err := json.Unmarshal(data, &info)
	if err != nil {
		return nil, fmt.Errorf("Error unmarshalling JSON from ffprobe output for file '%v':", path, err)
	}
	return info, nil
}

func GetVideoInformation(path string) (*VideoInfo, error) {
	info, jsonerr := GetFFMPEGJson(path)
	if jsonerr != nil {
		return nil, jsonerr
	}
	debug.Printf("ffprobe for %v returned", path, info)
	if _, ok := info["format"]; !ok {
		return nil, fmt.Errorf("ffprobe data for '%v' does not contain format info", path)
	}
	format := info["format"].(map[string]interface{})
	if _, ok := format["duration"]; !ok {
		return nil, fmt.Errorf("ffprobe format data for '%v' does not contain duration", path)
	}
	duration, perr := strconv.ParseFloat(format["duration"].(string), 64)
	if perr != nil {
		return nil, fmt.Errorf("Could not parse duration (%v) of '%v': ", format["duration"].(string), path, perr)
	}
	return &VideoInfo{duration}, nil
}
<commit_msg>Added caching for video infos<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"os/exec"
	"strconv"
	"strings"
)

var videoSuffixes = []string{".mp4", ".avi", ".mkv", ".flv", ".wmv", ".mov", ".mpg"}

var videoInfos = make(map[string]*VideoInfo)

type VideoInfo struct {
	Duration float64 `json:"duration"`
}

func FilenameLooksLikeVideo(name string) bool {
	for _, suffix := range videoSuffixes {
		if strings.HasSuffix(name, suffix) {
			return true
		}
	}
	return false
}

func GetRawFFMPEGInfo(path string) ([]byte, error) {
	debug.Printf("Executing ffprobe for %v", path)
	cmd := exec.Command("./tools/ffprobe", "-v", "quiet", "-print_format", "json", "-show_format", ""+path+"")
	data, err := cmd.Output()
	if err != nil {
		return nil, fmt.Errorf("Error executing ffprobe for file '%v':", path, err)
	}
	return data, nil
}

func GetFFMPEGJson(path string) (map[string]interface{}, error) {
	data, cmderr := GetRawFFMPEGInfo(path)
	if cmderr != nil {
		return nil, cmderr
	}
	var info map[string]interface{}
	err := json.Unmarshal(data, &info)
	if err != nil {
		return nil, fmt.Errorf("Error unmarshalling JSON from ffprobe output for file '%v':", path, err)
	}
	return info, nil
}

func GetVideoInformation(path string) (*VideoInfo, error) {
	if data, ok := videoInfos[path]; ok {
		return data, nil
	}
	info, jsonerr := GetFFMPEGJson(path)
	if jsonerr != nil {
		return nil, jsonerr
	}
	debug.Printf("ffprobe for %v returned", path, info)
	if _, ok := info["format"]; !ok {
		return nil, fmt.Errorf("ffprobe data for '%v' does not contain format info", path)
	}
	format := info["format"].(map[string]interface{})
	if _, ok := format["duration"]; !ok {
		return nil, fmt.Errorf("ffprobe format data for '%v' does not contain duration", path)
	}
	duration, perr := strconv.ParseFloat(format["duration"].(string), 64)
	if perr != nil {
		return nil, fmt.Errorf("Could not parse duration (%v) of '%v': ", format["duration"].(string), path, perr)
	}
	var vi = &VideoInfo{duration}
	videoInfos[path] = vi
	return vi, nil
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// createmock is used to generate source code for mock versions of interfaces
// from installed packages.
package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"path"
	"text/template"

	// Ensure that the generate package, which is used by the generated code, is
	// installed by goinstall.
	_ "github.com/jacobsa/oglemock/generate"
)

// A template for generated code that is used to print the result.
const tmplStr =
`
package main

import (
	"{{.InputPkg}}"
	"github.com/jacobsa/oglemock/generate"
	"os"
	"reflect"
)

func getTypeForPtr(ptr interface{}) reflect.Type {
	return reflect.TypeOf(ptr).Elem()
}

func main() {
	// Reduce noise in logging output.
	log.SetFlags(0)

	interfaces := []reflect.Type{
		{{range $typeName := .TypeNames}}
			getTypeForPtr((*{{$typeName}})(nil)),
		{{end}}
	}

	err := generate.GenerateMockSource(os.Stdout, "{{.OutputPkg}}", interfaces)
	if err != nil {
		log.Fatalf("Error generating mock source: %v", err)
	}
}
`

type tmplArg struct {
	InputPkg string
	OutputPkg string

	// Types to be mocked, relative to their package's name.
	TypeNames []string
}

func main() {
	// Reduce noise in logging output.
	log.SetFlags(0)

	// Check the command-line arguments.
	flag.Parse()

	cmdLineArgs := flag.Args()
	if len(cmdLineArgs) < 2 {
		fmt.Println("Usage: createmock [package] [interface ...]")
		os.Exit(1)
	}

	// Create an appropriate template argument.
	var arg tmplArg
	arg.InputPkg = cmdLineArgs[0]
	arg.OutputPkg = "mock_" + path.Base(arg.InputPkg)
	arg.TypeNames = cmdLineArgs[1:]

	// Execute the template to generate code that will itself generate the mock
	// code.
	tmpl := template.Must(template.New("code").Parse(tmplStr))
	if err := tmpl.Execute(os.Stdout, arg); err != nil {
		log.Fatalf("Error executing template: %v", err)
	}
}
<commit_msg>Fixed a bug, for #2.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// createmock is used to generate source code for mock versions of interfaces
// from installed packages.
package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"path"
	"text/template"

	// Ensure that the generate package, which is used by the generated code, is
	// installed by goinstall.
	_ "github.com/jacobsa/oglemock/generate"
)

// A template for generated code that is used to print the result.
const tmplStr =
`
{{$inputPkg := .InputPkg}}
{{$outputPkg := .OutputPkg}}

package main

import (
	"{{$inputPkg}}"
	"github.com/jacobsa/oglemock/generate"
	"os"
	"reflect"
)

func getTypeForPtr(ptr interface{}) reflect.Type {
	return reflect.TypeOf(ptr).Elem()
}

func main() {
	// Reduce noise in logging output.
	log.SetFlags(0)

	interfaces := []reflect.Type{
		{{range $typeName := .TypeNames}}
			getTypeForPtr((*{{$inputPkg}}.{{$typeName}})(nil)),
		{{end}}
	}

	err := generate.GenerateMockSource(os.Stdout, "{{$outputPkg}}", interfaces)
	if err != nil {
		log.Fatalf("Error generating mock source: %v", err)
	}
}
`

type tmplArg struct {
	InputPkg string
	OutputPkg string

	// Types to be mocked, relative to their package's name.
	TypeNames []string
}

func main() {
	// Reduce noise in logging output.
	log.SetFlags(0)

	// Check the command-line arguments.
	flag.Parse()

	cmdLineArgs := flag.Args()
	if len(cmdLineArgs) < 2 {
		fmt.Println("Usage: createmock [package] [interface ...]")
		os.Exit(1)
	}

	// Create an appropriate template argument.
	var arg tmplArg
	arg.InputPkg = cmdLineArgs[0]
	arg.OutputPkg = "mock_" + path.Base(arg.InputPkg)
	arg.TypeNames = cmdLineArgs[1:]

	// Execute the template to generate code that will itself generate the mock
	// code.
	tmpl := template.Must(template.New("code").Parse(tmplStr))
	if err := tmpl.Execute(os.Stdout, arg); err != nil {
		log.Fatalf("Error executing template: %v", err)
	}
}
<|endoftext|>


<commit_before>package core

import (
	"strconv"
	"sync"

	"github.com/af83/edwig/audit"
	"github.com/af83/edwig/model"
	"github.com/af83/edwig/siri"
)

type StopMonitoringSubscriptionBroadcaster interface {
	model.Stopable
	model.Startable

	HandleStopMonitoringBroadcastEvent(*model.StopMonitoringBroadcastEvent)
	HandleSubscriptionRequest([]*siri.XMLStopMonitoringSubscriptionRequestEntry)
}

type SIRIStopMonitoringSubscriptionBroadcaster struct {
	model.ClockConsumer
	model.UUIDConsumer

	siriConnector

	stopMonitoringBroadcaster SIRIStopMonitoringBroadcaster
	toBroadcast               map[SubscriptionId][]model.StopVisitId
	mutex                     *sync.Mutex //protect the map
}

type SIRIStopMonitoringSubscriptionBroadcasterFactory struct{}

func (factory *SIRIStopMonitoringSubscriptionBroadcasterFactory) CreateConnector(partner *Partner) Connector {
	if _, ok := partner.Connector(SIRI_SUBSCRIPTION_REQUEST_DISPATCHER); !ok {
		partner.CreateSubscriptionRequestDispatcher()
	}
	return newSIRIStopMonitoringSubscriptionBroadcaster(partner)
}

func (factory *SIRIStopMonitoringSubscriptionBroadcasterFactory) Validate(apiPartner *APIPartner) bool {
	ok := apiPartner.ValidatePresenceOfSetting("remote_objectid_kind")
	ok = ok && apiPartner.ValidatePresenceOfSetting("remote_url")
	ok = ok && apiPartner.ValidatePresenceOfSetting("remote_credential")
	return ok
}

func newSIRIStopMonitoringSubscriptionBroadcaster(partner *Partner) *SIRIStopMonitoringSubscriptionBroadcaster {
	siriStopMonitoringSubscriptionBroadcaster := &SIRIStopMonitoringSubscriptionBroadcaster{}
	siriStopMonitoringSubscriptionBroadcaster.partner = partner
	siriStopMonitoringSubscriptionBroadcaster.mutex = &sync.Mutex{}
	siriStopMonitoringSubscriptionBroadcaster.toBroadcast = make(map[SubscriptionId][]model.StopVisitId)

	siriStopMonitoringSubscriptionBroadcaster.stopMonitoringBroadcaster = NewSIRIStopMonitoringBroadcaster(siriStopMonitoringSubscriptionBroadcaster)

	return siriStopMonitoringSubscriptionBroadcaster
}

func (connector *SIRIStopMonitoringSubscriptionBroadcaster) Stop() {
	connector.stopMonitoringBroadcaster.Stop()
}

func (connector *SIRIStopMonitoringSubscriptionBroadcaster) Start() {
	connector.stopMonitoringBroadcaster.Start()
}

func (connector *SIRIStopMonitoringSubscriptionBroadcaster) HandleStopMonitoringBroadcastEvent(event *model.StopMonitoringBroadcastEvent) {
	tx := connector.Partner().Referential().NewTransaction()
	defer tx.Close()

	switch event.ModelType {
	case "StopVisit":
		sv, ok := tx.Model().StopVisits().Find(model.StopVisitId(event.ModelId))
		if ok {
			subsIds := connector.checkEvent(sv, tx)
			connector.addStopVisit(subsIds, sv.Id())
		}
	case "VehicleJourney":
		for _, sv := range tx.Model().StopVisits().FindFollowingByVehicleJourneyId(model.VehicleJourneyId(event.ModelId)) {
			subsIds := connector.checkEvent(sv, tx)
			connector.addStopVisit(subsIds, sv.Id())
		}
	default:
		return
	}
}

func (connector *SIRIStopMonitoringSubscriptionBroadcaster) addStopVisit(subsIds []SubscriptionId, svId model.StopVisitId) {
	connector.mutex.Lock()
	for _, subId := range subsIds {
		connector.toBroadcast[SubscriptionId(subId)] = append(connector.toBroadcast[SubscriptionId(subId)], svId)
	}
	connector.mutex.Unlock()
}

func (connector *SIRIStopMonitoringSubscriptionBroadcaster) checkEvent(sv model.StopVisit, tx *model.Transaction) []SubscriptionId {
	subscriptionIds := []SubscriptionId{}

	stopArea, ok := tx.Model().StopAreas().Find(sv.StopAreaId)
	if !ok {
		return subscriptionIds
	}

	obj, ok := stopArea.ObjectID(connector.partner.RemoteObjectIDKind(SIRI_STOP_MONITORING_SUBSCRIPTION_BROADCASTER))
	if !ok {
		return subscriptionIds
	}

	subs := connector.partner.Subscriptions().FindByRessourceId(obj.String(), "StopMonitoringBroadcast")

	for _, sub := range subs {
		resource, ok := sub.ResourcesByObjectID()[obj.String()]
		if !ok || resource.SubscribedUntil.Before(connector.Clock().Now()) {
			continue
		}

		lastState, ok := resource.LastStates[string(sv.Id())]

		if ok && !lastState.(*stopMonitoringLastChange).Haschanged(sv) {
			continue
		}

		if !ok {
			smlc := &stopMonitoringLastChange{}
			smlc.InitState(&sv, sub)
			resource.LastStates[string(sv.Id())] = smlc
		}
		subscriptionIds = append(subscriptionIds, sub.Id())
	}

	return subscriptionIds
}

func (connector *SIRIStopMonitoringSubscriptionBroadcaster) HandleSubscriptionRequest(request *siri.XMLSubscriptionRequest) []siri.SIRIResponseStatus {
	tx := connector.Partner().Referential().NewTransaction()
	defer tx.Close()

	resps := []siri.SIRIResponseStatus{}

	for _, sm := range request.XMLSubscriptionSMEntries() {
		logStashEvent := connector.newLogStashEvent()
		logXMLStopMonitoringSubscriptionEntry(logStashEvent, sm)

		rs := siri.SIRIResponseStatus{
			RequestMessageRef: sm.MessageIdentifier(),
			SubscriberRef:     sm.SubscriberRef(),
			SubscriptionRef:   sm.SubscriptionIdentifier(),
			ResponseTimestamp: connector.Clock().Now(),
		}

		objectid := model.NewObjectID(connector.partner.RemoteObjectIDKind(SIRI_STOP_MONITORING_SUBSCRIPTION_BROADCASTER), sm.MonitoringRef())
		sa, ok := tx.Model().StopAreas().FindByObjectId(objectid)
		if !ok {
			rs.ErrorType = "InvalidDataReferencesError"
			rs.ErrorText = "Stop Area not found"
			resps = append(resps, rs)

			logSIRIStopMonitoringSubscriptionResponseEntry(logStashEvent, &rs)
			audit.CurrentLogStash().WriteEvent(logStashEvent)

			continue
		}

		sub, ok := connector.Partner().Subscriptions().FindByExternalId(sm.SubscriptionIdentifier())
		if !ok {
			sub = connector.Partner().Subscriptions().New("StopMonitoringBroadcast")
			sub.SetExternalId(sm.SubscriptionIdentifier())
		}

		ref := model.Reference{
			ObjectId: &objectid,
			Type:     "StopArea",
		}

		r := sub.CreateAddNewResource(ref)
		r.SubscribedAt = connector.Clock().Now()
		r.SubscribedUntil = sm.InitialTerminationTime()

		connector.fillOptions(sub, r, request, sm)

		rs.Status = true
		rs.ValidUntil = sm.InitialTerminationTime()
		resps = append(resps, rs)

		logSIRIStopMonitoringSubscriptionResponseEntry(logStashEvent, &rs)
		audit.CurrentLogStash().WriteEvent(logStashEvent)

		connector.addStopAreaStopVisits(sa, sub, r)
	}

	return resps
}

func (connector *SIRIStopMonitoringSubscriptionBroadcaster) addStopAreaStopVisits(sa model.StopArea, sub *Subscription, res *SubscribedResource) {
	tx := connector.Partner().Referential().NewTransaction()
	defer tx.Close()

	for _, sv := range tx.Model().StopVisits().FindFollowingByStopAreaId(sa.Id()) {
		smlc := &stopMonitoringLastChange{}
		smlc.InitState(&sv, sub)
		res.LastStates[string(sv.Id())] = smlc
		connector.addStopVisit([]SubscriptionId{sub.Id()}, sv.Id())
	}
}

func (smsb *SIRIStopMonitoringSubscriptionBroadcaster) fillOptions(s *Subscription, r *SubscribedResource, request *siri.XMLSubscriptionRequest, sm *siri.XMLStopMonitoringSubscriptionRequestEntry) {
	ro := r.ResourcesOptions()
	ro["StopVisitTypes"] = sm.StopVisitTypes()
	changeBeforeUpdates := request.ChangeBeforeUpdates()
	if changeBeforeUpdates == "" {
		changeBeforeUpdates = "PT1M"
	}

	so := s.SubscriptionOptions()

	so["IncrementalUpdates"] = request.IncrementalUpdates()
	so["MaximumStopVisits"] = request.MaximumStopVisits()
	so["ChangeBeforeUpdates"] = changeBeforeUpdates
	so["MessageIdentifier"] = request.MessageIdentifier()
}

func (connector *SIRIStopMonitoringSubscriptionBroadcaster) newLogStashEvent() audit.LogStashEvent {
	event := connector.partner.NewLogStashEvent()
	event["connector"] = "StopMonitoringSubscriptionBroadcaster"
	return event
}

func logXMLStopMonitoringSubscriptionEntry(logStashEvent audit.LogStashEvent, request *siri.XMLStopMonitoringSubscriptionRequestEntry) {
	logStashEvent["type"] = "StopMonitoringSubscriptionEntry"
	logStashEvent["messageIdentifier"] = request.MessageIdentifier()
	logStashEvent["monitoringRef"] = request.MonitoringRef()
	logStashEvent["stopVisitTypes"] = request.StopVisitTypes()
	logStashEvent["subscriberRef"] = request.SubscriberRef()
	logStashEvent["subscriptionIdentifier"] = request.SubscriptionIdentifier()
	logStashEvent["initialTerminationTime"] = request.InitialTerminationTime().String()
	logStashEvent["requestTimestamp"] = request.RequestTimestamp().String()
	logStashEvent["requestXML"] = request.RawXML()
}

func logSIRIStopMonitoringSubscriptionResponseEntry(logStashEvent audit.LogStashEvent, smEntry *siri.SIRIResponseStatus) {
	logStashEvent["requestMessageRef"] = smEntry.RequestMessageRef
	logStashEvent["subscriptionRef"] = smEntry.SubscriptionRef
	logStashEvent["responseTimestamp"] = smEntry.ResponseTimestamp.String()
	logStashEvent["validUntil"] = smEntry.ValidUntil.String()
	logStashEvent["status"] = strconv.FormatBool(smEntry.Status)
	if !smEntry.Status {
		logStashEvent["errorType"] = smEntry.ErrorType
		if smEntry.ErrorType == "OtherError" {
			logStashEvent["errorNumber"] = strconv.Itoa(smEntry.ErrorNumber)
		}
		logStashEvent["errorText"] = smEntry.ErrorText
	}
}

// START TEST

type TestSIRIStopMonitoringSubscriptionBroadcasterFactory struct{}

type TestStopMonitoringSubscriptionBroadcaster struct {
	model.UUIDConsumer

	events                    []*model.StopMonitoringBroadcastEvent
	stopMonitoringBroadcaster SIRIStopMonitoringBroadcaster
}

func NewTestStopMonitoringSubscriptionBroadcaster() *TestStopMonitoringSubscriptionBroadcaster {
	connector := &TestStopMonitoringSubscriptionBroadcaster{}
	return connector
}

func (connector *TestStopMonitoringSubscriptionBroadcaster) HandleStopMonitoringBroadcastEvent(event *model.StopMonitoringBroadcastEvent) {
	connector.events = append(connector.events, event)
}

func (factory *TestSIRIStopMonitoringSubscriptionBroadcasterFactory) Validate(apiPartner *APIPartner) bool {
	return true
}

func (factory *TestSIRIStopMonitoringSubscriptionBroadcasterFactory) CreateConnector(partner *Partner) Connector {
	return NewTestStopMonitoringSubscriptionBroadcaster()
}

// END TEST
<commit_msg>Now not sending previously sended notification during subscription edit<commit_after>package core

import (
	"strconv"
	"sync"

	"github.com/af83/edwig/audit"
	"github.com/af83/edwig/model"
	"github.com/af83/edwig/siri"
)

type StopMonitoringSubscriptionBroadcaster interface {
	model.Stopable
	model.Startable

	HandleStopMonitoringBroadcastEvent(*model.StopMonitoringBroadcastEvent)
	HandleSubscriptionRequest([]*siri.XMLStopMonitoringSubscriptionRequestEntry)
}

type SIRIStopMonitoringSubscriptionBroadcaster struct {
	model.ClockConsumer
	model.UUIDConsumer

	siriConnector

	stopMonitoringBroadcaster SIRIStopMonitoringBroadcaster
	toBroadcast               map[SubscriptionId][]model.StopVisitId
	mutex                     *sync.Mutex //protect the map
}

type SIRIStopMonitoringSubscriptionBroadcasterFactory struct{}

func (factory *SIRIStopMonitoringSubscriptionBroadcasterFactory) CreateConnector(partner *Partner) Connector {
	if _, ok := partner.Connector(SIRI_SUBSCRIPTION_REQUEST_DISPATCHER); !ok {
		partner.CreateSubscriptionRequestDispatcher()
	}
	return newSIRIStopMonitoringSubscriptionBroadcaster(partner)
}

func (factory *SIRIStopMonitoringSubscriptionBroadcasterFactory) Validate(apiPartner *APIPartner) bool {
	ok := apiPartner.ValidatePresenceOfSetting("remote_objectid_kind")
	ok = ok && apiPartner.ValidatePresenceOfSetting("remote_url")
	ok = ok && apiPartner.ValidatePresenceOfSetting("remote_credential")
	return ok
}

func newSIRIStopMonitoringSubscriptionBroadcaster(partner *Partner) *SIRIStopMonitoringSubscriptionBroadcaster {
	siriStopMonitoringSubscriptionBroadcaster := &SIRIStopMonitoringSubscriptionBroadcaster{}
	siriStopMonitoringSubscriptionBroadcaster.partner = partner
	siriStopMonitoringSubscriptionBroadcaster.mutex = &sync.Mutex{}
	siriStopMonitoringSubscriptionBroadcaster.toBroadcast = make(map[SubscriptionId][]model.StopVisitId)

	siriStopMonitoringSubscriptionBroadcaster.stopMonitoringBroadcaster = NewSIRIStopMonitoringBroadcaster(siriStopMonitoringSubscriptionBroadcaster)

	return siriStopMonitoringSubscriptionBroadcaster
}

func (connector *SIRIStopMonitoringSubscriptionBroadcaster) Stop() {
	connector.stopMonitoringBroadcaster.Stop()
}

func (connector *SIRIStopMonitoringSubscriptionBroadcaster) Start() {
	connector.stopMonitoringBroadcaster.Start()
}

func (connector *SIRIStopMonitoringSubscriptionBroadcaster) HandleStopMonitoringBroadcastEvent(event *model.StopMonitoringBroadcastEvent) {
	tx := connector.Partner().Referential().NewTransaction()
	defer tx.Close()

	switch event.ModelType {
	case "StopVisit":
		sv, ok := tx.Model().StopVisits().Find(model.StopVisitId(event.ModelId))
		if ok {
			subsIds := connector.checkEvent(sv, tx)
			connector.addStopVisit(subsIds, sv.Id())
		}
	case "VehicleJourney":
		for _, sv := range tx.Model().StopVisits().FindFollowingByVehicleJourneyId(model.VehicleJourneyId(event.ModelId)) {
			subsIds := connector.checkEvent(sv, tx)
			connector.addStopVisit(subsIds, sv.Id())
		}
	default:
		return
	}
}

func (connector *SIRIStopMonitoringSubscriptionBroadcaster) addStopVisit(subsIds []SubscriptionId, svId model.StopVisitId) {
	connector.mutex.Lock()
	for _, subId := range subsIds {
		connector.toBroadcast[SubscriptionId(subId)] = append(connector.toBroadcast[SubscriptionId(subId)], svId)
	}
	connector.mutex.Unlock()
}

func (connector *SIRIStopMonitoringSubscriptionBroadcaster) checkEvent(sv model.StopVisit, tx *model.Transaction) []SubscriptionId {
	subscriptionIds := []SubscriptionId{}

	stopArea, ok := tx.Model().StopAreas().Find(sv.StopAreaId)
	if !ok {
		return subscriptionIds
	}

	obj, ok := stopArea.ObjectID(connector.partner.RemoteObjectIDKind(SIRI_STOP_MONITORING_SUBSCRIPTION_BROADCASTER))
	if !ok {
		return subscriptionIds
	}

	subs := connector.partner.Subscriptions().FindByRessourceId(obj.String(), "StopMonitoringBroadcast")

	for _, sub := range subs {
		resource, ok := sub.ResourcesByObjectID()[obj.String()]
		if !ok || resource.SubscribedUntil.Before(connector.Clock().Now()) {
			continue
		}

		lastState, ok := resource.LastStates[string(sv.Id())]

		if ok && !lastState.(*stopMonitoringLastChange).Haschanged(sv) {
			continue
		}

		if !ok {
			smlc := &stopMonitoringLastChange{}
			smlc.InitState(&sv, sub)
			resource.LastStates[string(sv.Id())] = smlc
		}
		subscriptionIds = append(subscriptionIds, sub.Id())
	}

	return subscriptionIds
}

func (connector *SIRIStopMonitoringSubscriptionBroadcaster) HandleSubscriptionRequest(request *siri.XMLSubscriptionRequest) []siri.SIRIResponseStatus {
	tx := connector.Partner().Referential().NewTransaction()
	defer tx.Close()

	resps := []siri.SIRIResponseStatus{}

	for _, sm := range request.XMLSubscriptionSMEntries() {
		logStashEvent := connector.newLogStashEvent()
		logXMLStopMonitoringSubscriptionEntry(logStashEvent, sm)

		rs := siri.SIRIResponseStatus{
			RequestMessageRef: sm.MessageIdentifier(),
			SubscriberRef:     sm.SubscriberRef(),
			SubscriptionRef:   sm.SubscriptionIdentifier(),
			ResponseTimestamp: connector.Clock().Now(),
		}

		objectid := model.NewObjectID(connector.partner.RemoteObjectIDKind(SIRI_STOP_MONITORING_SUBSCRIPTION_BROADCASTER), sm.MonitoringRef())
		sa, ok := tx.Model().StopAreas().FindByObjectId(objectid)
		if !ok {
			rs.ErrorType = "InvalidDataReferencesError"
			rs.ErrorText = "Stop Area not found"
			resps = append(resps, rs)

			logSIRIStopMonitoringSubscriptionResponseEntry(logStashEvent, &rs)
			audit.CurrentLogStash().WriteEvent(logStashEvent)

			continue
		}

		sub, ok := connector.Partner().Subscriptions().FindByExternalId(sm.SubscriptionIdentifier())
		if !ok {
			sub = connector.Partner().Subscriptions().New("StopMonitoringBroadcast")
			sub.SetExternalId(sm.SubscriptionIdentifier())
		}

		ref := model.Reference{
			ObjectId: &objectid,
			Type:     "StopArea",
		}

		r := sub.CreateAddNewResource(ref)
		r.SubscribedAt = connector.Clock().Now()
		r.SubscribedUntil = sm.InitialTerminationTime()

		connector.fillOptions(sub, r, request, sm)

		rs.Status = true
		rs.ValidUntil = sm.InitialTerminationTime()
		resps = append(resps, rs)

		logSIRIStopMonitoringSubscriptionResponseEntry(logStashEvent, &rs)
		audit.CurrentLogStash().WriteEvent(logStashEvent)

		connector.addStopAreaStopVisits(sa, sub, r)
	}

	return resps
}

func (connector *SIRIStopMonitoringSubscriptionBroadcaster) addStopAreaStopVisits(sa model.StopArea, sub *Subscription, res *SubscribedResource) {
	tx := connector.Partner().Referential().NewTransaction()
	defer tx.Close()

	for _, sv := range tx.Model().StopVisits().FindFollowingByStopAreaId(sa.Id()) {
		if _, ok := res.LastStates[string(sv.Id())]; ok {
			continue
		}
		smlc := &stopMonitoringLastChange{}
		smlc.InitState(&sv, sub)
		res.LastStates[string(sv.Id())] = smlc
		connector.addStopVisit([]SubscriptionId{sub.Id()}, sv.Id())
	}
}

func (smsb *SIRIStopMonitoringSubscriptionBroadcaster) fillOptions(s *Subscription, r *SubscribedResource, request *siri.XMLSubscriptionRequest, sm *siri.XMLStopMonitoringSubscriptionRequestEntry) {
	ro := r.ResourcesOptions()
	ro["StopVisitTypes"] = sm.StopVisitTypes()
	changeBeforeUpdates := request.ChangeBeforeUpdates()
	if changeBeforeUpdates == "" {
		changeBeforeUpdates = "PT1M"
	}

	so := s.SubscriptionOptions()

	so["IncrementalUpdates"] = request.IncrementalUpdates()
	so["MaximumStopVisits"] = request.MaximumStopVisits()
	so["ChangeBeforeUpdates"] = changeBeforeUpdates
	so["MessageIdentifier"] = request.MessageIdentifier()
}

func (connector *SIRIStopMonitoringSubscriptionBroadcaster) newLogStashEvent() audit.LogStashEvent {
	event := connector.partner.NewLogStashEvent()
	event["connector"] = "StopMonitoringSubscriptionBroadcaster"
	return event
}

func logXMLStopMonitoringSubscriptionEntry(logStashEvent audit.LogStashEvent, request *siri.XMLStopMonitoringSubscriptionRequestEntry) {
	logStashEvent["type"] = "StopMonitoringSubscriptionEntry"
	logStashEvent["messageIdentifier"] = request.MessageIdentifier()
	logStashEvent["monitoringRef"] = request.MonitoringRef()
	logStashEvent["stopVisitTypes"] = request.StopVisitTypes()
	logStashEvent["subscriberRef"] = request.SubscriberRef()
	logStashEvent["subscriptionIdentifier"] = request.SubscriptionIdentifier()
	logStashEvent["initialTerminationTime"] = request.InitialTerminationTime().String()
	logStashEvent["requestTimestamp"] = request.RequestTimestamp().String()
	logStashEvent["requestXML"] = request.RawXML()
}

func logSIRIStopMonitoringSubscriptionResponseEntry(logStashEvent audit.LogStashEvent, smEntry *siri.SIRIResponseStatus) {
	logStashEvent["requestMessageRef"] = smEntry.RequestMessageRef
	logStashEvent["subscriptionRef"] = smEntry.SubscriptionRef
	logStashEvent["responseTimestamp"] = smEntry.ResponseTimestamp.String()
	logStashEvent["validUntil"] = smEntry.ValidUntil.String()
	logStashEvent["status"] = strconv.FormatBool(smEntry.Status)
	if !smEntry.Status {
		logStashEvent["errorType"] = smEntry.ErrorType
		if smEntry.ErrorType == "OtherError" {
			logStashEvent["errorNumber"] = strconv.Itoa(smEntry.ErrorNumber)
		}
		logStashEvent["errorText"] = smEntry.ErrorText
	}
}

// START TEST

type TestSIRIStopMonitoringSubscriptionBroadcasterFactory struct{}

type TestStopMonitoringSubscriptionBroadcaster struct {
	model.UUIDConsumer

	events                    []*model.StopMonitoringBroadcastEvent
	stopMonitoringBroadcaster SIRIStopMonitoringBroadcaster
}

func NewTestStopMonitoringSubscriptionBroadcaster() *TestStopMonitoringSubscriptionBroadcaster {
	connector := &TestStopMonitoringSubscriptionBroadcaster{}
	return connector
}

func (connector *TestStopMonitoringSubscriptionBroadcaster) HandleStopMonitoringBroadcastEvent(event *model.StopMonitoringBroadcastEvent) {
	connector.events = append(connector.events, event)
}

func (factory *TestSIRIStopMonitoringSubscriptionBroadcasterFactory) Validate(apiPartner *APIPartner) bool {
	return true
}

func (factory *TestSIRIStopMonitoringSubscriptionBroadcasterFactory) CreateConnector(partner *Partner) Connector {
	return NewTestStopMonitoringSubscriptionBroadcaster()
}

// END TEST
<|endoftext|>


<commit_before>package main

// This file is used for fooling around with the syntax

import (
  //"net/http"
  "fmt"
  "time"
  "math/rand"
  "math/cmplx"
)

/**
 * Two assignment operators
 *    x := foo()    and    x = foo()
 *
 * Where := defines a new variable x and assigns the return value of foo()
 * to it. The operator = assigns the return value of foo() to the existing
 * variable x.
 */

const (
  my_number = 1234
  )

var (
  j string = "initialized"
  woo complex128 = cmplx.Sqrt(-5 + 12i)
)

func foo(x string) (ret string) {
  //ret = x + "Fish" + string(time.Now()) + "!"
  defer fmt.Println("two")
  fmt.Println("one")
  ret = x + " Fish "
  j = "foo ran"
  return
}

func main() {
  var x, y, z = "Foo", false, -10
  var jjj int
  var tmp string
  //tmp string
  //tmp := "Rumble"
  fmt.Println(j)
  for jjj = 0; jjj < 2; jjj++ {
    tmp = foo("Rumble")
  }
  fmt.Println(j)
  fmt.Println(tmp)
  rand.Seed(time.Now().Unix())
  fmt.Println("Number", rand.Intn(100))
  fmt.Println(x, y, z)
  //resp, err := http.Get("http://www.spatineo.com/")
  fmt.Println(woo)
  for jjj = 0; jjj < 10; jjj++ {
    fmt.Println(jjj)
  }
  switch {
    case jjj < 5:
      fmt.Println("Never??")
    case jjj < 15:
      fmt.Println("Yeah!")
    case jjj < 25:
      fmt.Println("Interesting!")
  }
}
<commit_msg>Testing classes<commit_after>package main

// This file is used for fooling around with the syntax

import (
  //"net/http"
  "fmt"
  "time"
  "math/rand"
  "math/cmplx"
)

/**
 * Two assignment operators
 *    x := foo()    and    x = foo()
 *
 * Where := defines a new variable x and assigns the return value of foo()
 * to it. The operator = assigns the return value of foo() to the existing
 * variable x.
 */

const (
  my_number = 1234
  )

var (
  j string = "initialized"
  woo complex128 = cmplx.Sqrt(-5 + 12i)
)

// Fields
type MyClass struct {
  i int
}

// Method
func (h MyClass) GetI() int {
  return h.i
}

// Constructor
func NewMyClass() *MyClass {
  ret := new(MyClass)
  ret.i = 10
  return ret
}

func foo(x string) (ret string) {
  //ret = x + "Fish" + string(time.Now()) + "!"
  defer fmt.Println("two")
  fmt.Println("one")
  ret = x + " Fish "
  j = "foo ran"
  return
}

func main() {
  var d MyClass
  fmt.Println(d.GetI())
  var p = NewMyClass();
  fmt.Println(p.GetI())

  var x, y, z = "Foo", false, -10
  var jjj int
  var tmp string
  //tmp string
  //tmp := "Rumble"
  fmt.Println(j)
  for jjj = 0; jjj < 2; jjj++ {
    tmp = foo("Rumble")
  }
  fmt.Println(j)
  fmt.Println(tmp)
  rand.Seed(time.Now().Unix())
  fmt.Println("Number", rand.Intn(100))
  fmt.Println(x, y, z)
  //resp, err := http.Get("http://www.spatineo.com/")
  fmt.Println(woo)
  for jjj = 0; jjj < 10; jjj++ {
    fmt.Println(jjj)
  }
  switch {
    case jjj < 5:
      fmt.Println("Never??")
    case jjj < 15:
      fmt.Println("Yeah!")
    case jjj < 25:
      fmt.Println("Interesting!")
  }
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"strconv"
	"strings"

	"github.com/phil-mansfield/gotetra/render/halo"
	"github.com/phil-mansfield/gotetra/render/io"
)

var valMap = map[string]halo.Val {
	"scale":halo.Scale,
	"id":halo.ID,
    "descscale":halo.DescScale,
    "descid":halo.DescID,
    "numprog":halo.NumProg,
    "pid":halo.PID,
    "upid":halo.UPID,
    "descpid":halo.DescPID,
    "phantom":halo.Phantom,
    "sammvir":halo.SAMMVir,
    "mvir":halo.MVir,
    "rvir":halo.RVir,
    "rs":halo.Rs,
    "vrms":halo.Vrms,
    "mmp":halo.MMP,
    "scaleoflastmmp":halo.ScaleOfLastMMP,
    "vmax":halo.VMax,
    "x":halo.X,
    "y":halo.Y,
    "z":halo.Z,
    "vx":halo.Vx,
    "vy":halo.Vy,
    "vz":halo.Vz,
    "jx":halo.Jx,
    "jy":halo.Jy,
    "jz":halo.Jz,
    "spin":halo.Spin,
    "breadthfirstid":halo.BreadthFirstID,
    "depthfirstid":halo.DepthFirstID,
    "treerootid":halo.TreeRootID,
    "orighaloid":halo.OrigHaloID,
    "snapnum":halo.SnapNum,
    "nextcoprogenitordepthfirstid":halo.NextCoprogenitorDepthFirstID,
    "lastprogenitordepthfirstid":halo.LastProgenitorDepthFirstID,
    "rsklypin":halo.RsKylpin,
    "mvirall":halo.MVirAll,
    "m200b":halo.M200b,
    "m200c":halo.M200c,
    "m500c":halo.M500c,
    "m2500c":halo.M2500c,
    "xoff":halo.XOff,
    "voff":halo.Voff,
    "spinbullock":halo.SpinBullock,
    "btoa":halo.BToA,
    "ctoa":halo.CToA,
    "ax":halo.Ax,
    "ay":halo.Ay,
    "az":halo.Az,
    "btoa500c":halo.BToA500c,
    "ctoa500c":halo.CToA500c,
    "ax500c":halo.Ax500c,
    "ay500c":halo.Ay500c,
    "az500c":halo.Az500c,
    "tu":halo.TU,
    "macc":halo.MAcc,
    "mpeak":halo.MPeak,
    "vacc":halo.VAcc,
    "vpeak":halo.VPeak,
    "halfmassscale":halo.HalfmassScale,
    "accrateinst":halo.AccRateInst,
    "accrate100myr":halo.AccRate100Myr,
    "accratetdyn":halo.AccRateTdyn,
}

func main() {
	valFlags, err := parseCmd()
	if err != nil { log.Fatal(err.Error()) }
	if len(valFlags) == 0 {
		err = printStdin()
		if err != nil { log.Fatal(err.Error()) }
	}

	ids, snaps, inVals, err := parseStdin()
	if err != nil { log.Fatal(err.Error()) }
	vals, err := readVals(ids, snaps, valFlags)
	if err != nil { log.Fatal(err.Error()) }

	printVals(ids, snaps, inVals, vals)
}

func parseCmd() ([]halo.Val, error) {
	flag.Parse()
	args := flag.Args()
	vals := make([]halo.Val, len(args))
	var ok bool
	for i, arg := range args {
		vals[i], ok = valMap[strings.ToLower(arg)]
		if !ok {
			return nil, fmt.Errorf("Flag %d, %s, not recognized.", i+1, arg)
		}
	}
	return vals, nil
}

func printStdin() error {
	lines, err := stdinLines()
	if err != nil { return err }
	fmt.Println(strings.Join(lines, "\n"))
	return nil
}

func stdinLines() ([]string, error) {
	bs, err := ioutil.ReadAll(os.Stdin)
	if err != nil {
		return nil, fmt.Errorf(
			"Error reading stdin: %s.", err.Error(),
		)
	}

	text := string(bs)
	return strings.Split(text, "\n"), nil
}

func parseStdin() (ids, snaps []int, inVals [][]float64, err error) {
	ids, snaps, inVals = []int{}, []int{}, [][]float64{}
	lines, err := stdinLines()
	if err != nil { return nil, nil, nil, err }
	for i, line := range lines {
		rawTokens := strings.Split(line, " ")
		tokens := make([]string, 0, len(rawTokens))
		for _, tok := range rawTokens {
			if len(tok) != 0 { tokens = append(tokens, tok) }
		}

		var (
			id, snap int
			vals []float64
			err error
		)
		switch {
		case len(tokens) == 0:
			continue
		case len(tokens) == 1:
			if tokens[0] == "" { continue }
			return nil, nil, nil, fmt.Errorf(
				"Line %d of stdin has 1 token, but 2 are required.", i + 1,
			)
		case len(tokens) >= 2:
			id, err = strconv.Atoi(tokens[0])
			if err != nil {
				return nil, nil, nil, fmt.Errorf(
					"One line %d of stdin, %s does not parse as an int.",
					i + 1, tokens[0],
				)
			}
			snap, err = strconv.Atoi(tokens[1])
			if err != nil {
				return nil, nil, nil, fmt.Errorf(
					"One line %d of stdin, %s does not parse as an int.",
					i + 1, tokens[1],
				)
			}

			vals = make([]float64, len(tokens) - 2)
			for i := range vals {
				vals[i], err = strconv.ParseFloat(tokens[i + 2], 64)
			}
		}

		ids = append(ids, id)
		snaps = append(snaps, snap)
		inVals = append(inVals, vals)
	}

	return ids, snaps, inVals, nil
}

func readVals(ids, snaps []int, valFlags []halo.Val) ([][]float64, error) {
	snapBins, idxBins := binBySnap(snaps, ids)
	vals := make([][]float64, len(ids))
	for snap, idSet := range snapBins {
		idxSet := idxBins[snap]
		var (
			snapVals [][]float64
			err error
		)
		if snap == -1 { // Handle blank halos.
			snapVals = make([][]float64, len(idSet))
			for i := range snapVals {
				snapVals[i] = make([]float64, len(valFlags))
			}
		} else {
			snapVals, err = readSnapVals(idSet, snap, valFlags)
			if err != nil { return nil, err }
		}

		for i := range idSet {
			vals[idxSet[i]] = snapVals[i]
		}
	}
	return vals, nil
}

func binBySnap(snaps, ids []int) (snapBins, idxBins map[int][]int) {
	snapBins = make(map[int][]int)
	idxBins = make(map[int][]int)
	for i, snap := range snaps {
		id := ids[i]
		snapBins[snap] = append(snapBins[snap], id)
		idxBins[snap] = append(idxBins[snap], i)
	}
	return snapBins, idxBins
}

func readSnapVals(
	ids []int, snap int, valFlags []halo.Val,
) ([][]float64, error) {
	// Read in Rockstar catalog.
	rockstarDir := os.Getenv("GTET_ROCKSTAR_DIR")
	if rockstarDir == "" {
		return nil, fmt.Errorf("$GTET_ROCKSTAR_DIR not set.")
	}

	files, err := dirContents(rockstarDir)
	if err != nil { return nil, err }
	// Can't have a halo at the 0th snapshot.
	hlist := files[snap - 1]

	// Read in SheetHeader
	gtetFmt := os.Getenv("GTET_FMT")
	if gtetFmt == "" {
		return nil, fmt.Errorf("$GTET_FMT not set.")
	}
	files, err = dirContents(fmt.Sprintf(gtetFmt, snap))
	hd := &io.SheetHeader{}
	err = io.ReadSheetHeaderAt(files[0], hd)
	if err != nil { return nil, err }

	// Read in values and search for them.
	rids, vals, err := halo.ReadRockstarVals(hlist, &hd.Cosmo, valFlags...)
	if err != nil { return nil, err }

	outVals := make([][]float64, len(ids))
	for i := range outVals {
		outVals[i] = make([]float64, len(valFlags))
	}

	for i, id := range ids {
		// TODO: benchmark at what point sorting the list is important.
		idx := -1
		for j, rid := range rids {
			if rid == id {
				idx = j
				break
			}
		}
		if idx == -1 {
			return nil, fmt.Errorf("ID %d, %d, not found in hlist.", i+1, id)
		}
		for j := range outVals[i] {
			outVals[i][j] = vals[j][idx]
		}
	}

	return outVals, nil
}

func dirContents(dir string) ([]string, error) {
	infos, err := ioutil.ReadDir(dir)
	if err != nil { return nil, err }

	files := make([]string, len(infos))
	for i := range infos {
		files[i] = path.Join(dir, infos[i].Name())
	}

	return files, nil
}

func printVals(ids, snaps []int, inVals, vals [][]float64) {
	for i := range inVals {
		inVals[i] = append(inVals[i], vals[i]...)
	}
	vals = inVals

	idWidth, snapWidth := 0, 0
	valWidths := make([]int, len(vals[0]))
	for i := range ids {
		iWidth := len(fmt.Sprintf("%d", ids[i]))
		sWidth := len(fmt.Sprintf("%d", snaps[i]))
		if iWidth > idWidth { idWidth = iWidth }
		if sWidth > snapWidth { snapWidth = sWidth }
		for j := range vals[i] {
			width := len(fmt.Sprintf("%.10g", vals[i][j]))
			if width > valWidths[j] { valWidths[j] = width }
		}
	}

	rowFmt := fmt.Sprintf("%%%dd %%%dd", idWidth, snapWidth)
	valFmts := make([]string, len(vals[0]))
	for i := range valFmts {
		rowFmt += fmt.Sprintf(" %%%d.10g", valWidths[i])
	}
	rowFmt += "\n"
	for i := range ids {
		args := append([]interface{}{ids[i], snaps[i]}, intr(vals[i])...)
		fmt.Printf(rowFmt, args...)
	}
}

func intr(xs []float64) []interface{} {
	is := make([]interface{}, len(xs))
	for i, x := range xs { is[i] = x }
	return is
}

<commit_msg>Added radial flags to gtet_append<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"strconv"
	"strings"

	"github.com/phil-mansfield/gotetra/render/halo"
	"github.com/phil-mansfield/gotetra/render/io"
)

var valMap = map[string]halo.Val {
	"scale":halo.Scale,
	"id":halo.ID,
    "descscale":halo.DescScale,
    "descid":halo.DescID,
    "numprog":halo.NumProg,
    "pid":halo.PID,
    "upid":halo.UPID,
    "descpid":halo.DescPID,
    "phantom":halo.Phantom,
    "sammvir":halo.SAMMVir,
    "mvir":halo.MVir,
    "rvir":halo.RVir,
    "rs":halo.Rs,
    "vrms":halo.Vrms,
    "mmp":halo.MMP,
    "scaleoflastmmp":halo.ScaleOfLastMMP,
    "vmax":halo.VMax,
    "x":halo.X,
    "y":halo.Y,
    "z":halo.Z,
    "vx":halo.Vx,
    "vy":halo.Vy,
    "vz":halo.Vz,
    "jx":halo.Jx,
    "jy":halo.Jy,
    "jz":halo.Jz,
    "spin":halo.Spin,
    "breadthfirstid":halo.BreadthFirstID,
    "depthfirstid":halo.DepthFirstID,
    "treerootid":halo.TreeRootID,
    "orighaloid":halo.OrigHaloID,
    "snapnum":halo.SnapNum,
    "nextcoprogenitordepthfirstid":halo.NextCoprogenitorDepthFirstID,
    "lastprogenitordepthfirstid":halo.LastProgenitorDepthFirstID,
    "rsklypin":halo.RsKylpin,
    "mvirall":halo.MVirAll,
    "m200b":halo.M200b,
    "m200c":halo.M200c,
    "m500c":halo.M500c,
    "m2500c":halo.M2500c,
    "xoff":halo.XOff,
    "voff":halo.Voff,
    "spinbullock":halo.SpinBullock,
    "btoa":halo.BToA,
    "ctoa":halo.CToA,
    "ax":halo.Ax,
    "ay":halo.Ay,
    "az":halo.Az,
    "btoa500c":halo.BToA500c,
    "ctoa500c":halo.CToA500c,
    "ax500c":halo.Ax500c,
    "ay500c":halo.Ay500c,
    "az500c":halo.Az500c,
    "tu":halo.TU,
    "macc":halo.MAcc,
    "mpeak":halo.MPeak,
    "vacc":halo.VAcc,
    "vpeak":halo.VPeak,
    "halfmassscale":halo.HalfmassScale,
    "accrateinst":halo.AccRateInst,
    "accrate100myr":halo.AccRate100Myr,
    "accratetdyn":halo.AccRateTdyn,
    "rad200b":halo.Rad200b,
    "rad200c":halo.Rad200c,
    "rad500c":halo.Rad500c,
    "rad2500c":halo.Rad2500c,
}

func main() {
	valFlags, err := parseCmd()
	if err != nil { log.Fatal(err.Error()) }
	if len(valFlags) == 0 {
		err = printStdin()
		if err != nil { log.Fatal(err.Error()) }
	}

	ids, snaps, inVals, err := parseStdin()
	if err != nil { log.Fatal(err.Error()) }
	vals, err := readVals(ids, snaps, valFlags)
	if err != nil { log.Fatal(err.Error()) }

	printVals(ids, snaps, inVals, vals)
}

func parseCmd() ([]halo.Val, error) {
	flag.Parse()
	args := flag.Args()
	vals := make([]halo.Val, len(args))
	var ok bool
	for i, arg := range args {
		vals[i], ok = valMap[strings.ToLower(arg)]
		if !ok {
			return nil, fmt.Errorf("Flag %d, %s, not recognized.", i+1, arg)
		}
	}
	return vals, nil
}

func printStdin() error {
	lines, err := stdinLines()
	if err != nil { return err }
	fmt.Println(strings.Join(lines, "\n"))
	return nil
}

func stdinLines() ([]string, error) {
	bs, err := ioutil.ReadAll(os.Stdin)
	if err != nil {
		return nil, fmt.Errorf(
			"Error reading stdin: %s.", err.Error(),
		)
	}

	text := string(bs)
	return strings.Split(text, "\n"), nil
}

func parseStdin() (ids, snaps []int, inVals [][]float64, err error) {
	ids, snaps, inVals = []int{}, []int{}, [][]float64{}
	lines, err := stdinLines()
	if err != nil { return nil, nil, nil, err }
	for i, line := range lines {
		rawTokens := strings.Split(line, " ")
		tokens := make([]string, 0, len(rawTokens))
		for _, tok := range rawTokens {
			if len(tok) != 0 { tokens = append(tokens, tok) }
		}

		var (
			id, snap int
			vals []float64
			err error
		)
		switch {
		case len(tokens) == 0:
			continue
		case len(tokens) == 1:
			if tokens[0] == "" { continue }
			return nil, nil, nil, fmt.Errorf(
				"Line %d of stdin has 1 token, but 2 are required.", i + 1,
			)
		case len(tokens) >= 2:
			id, err = strconv.Atoi(tokens[0])
			if err != nil {
				return nil, nil, nil, fmt.Errorf(
					"One line %d of stdin, %s does not parse as an int.",
					i + 1, tokens[0],
				)
			}
			snap, err = strconv.Atoi(tokens[1])
			if err != nil {
				return nil, nil, nil, fmt.Errorf(
					"One line %d of stdin, %s does not parse as an int.",
					i + 1, tokens[1],
				)
			}

			vals = make([]float64, len(tokens) - 2)
			for i := range vals {
				vals[i], err = strconv.ParseFloat(tokens[i + 2], 64)
			}
		}

		ids = append(ids, id)
		snaps = append(snaps, snap)
		inVals = append(inVals, vals)
	}

	return ids, snaps, inVals, nil
}

func readVals(ids, snaps []int, valFlags []halo.Val) ([][]float64, error) {
	snapBins, idxBins := binBySnap(snaps, ids)
	vals := make([][]float64, len(ids))
	for snap, idSet := range snapBins {
		idxSet := idxBins[snap]
		var (
			snapVals [][]float64
			err error
		)
		if snap == -1 { // Handle blank halos.
			snapVals = make([][]float64, len(idSet))
			for i := range snapVals {
				snapVals[i] = make([]float64, len(valFlags))
			}
		} else {
			snapVals, err = readSnapVals(idSet, snap, valFlags)
			if err != nil { return nil, err }
		}

		for i := range idSet {
			vals[idxSet[i]] = snapVals[i]
		}
	}
	return vals, nil
}

func binBySnap(snaps, ids []int) (snapBins, idxBins map[int][]int) {
	snapBins = make(map[int][]int)
	idxBins = make(map[int][]int)
	for i, snap := range snaps {
		id := ids[i]
		snapBins[snap] = append(snapBins[snap], id)
		idxBins[snap] = append(idxBins[snap], i)
	}
	return snapBins, idxBins
}

func readSnapVals(
	ids []int, snap int, valFlags []halo.Val,
) ([][]float64, error) {
	// Read in Rockstar catalog.
	rockstarDir := os.Getenv("GTET_ROCKSTAR_DIR")
	if rockstarDir == "" {
		return nil, fmt.Errorf("$GTET_ROCKSTAR_DIR not set.")
	}

	files, err := dirContents(rockstarDir)
	if err != nil { return nil, err }
	// Can't have a halo at the 0th snapshot.
	hlist := files[snap - 1]

	// Read in SheetHeader
	gtetFmt := os.Getenv("GTET_FMT")
	if gtetFmt == "" {
		return nil, fmt.Errorf("$GTET_FMT not set.")
	}
	files, err = dirContents(fmt.Sprintf(gtetFmt, snap))
	hd := &io.SheetHeader{}
	err = io.ReadSheetHeaderAt(files[0], hd)
	if err != nil { return nil, err }

	// Read in values and search for them.
	rids, vals, err := halo.ReadRockstarVals(hlist, &hd.Cosmo, valFlags...)
	if err != nil { return nil, err }

	outVals := make([][]float64, len(ids))
	for i := range outVals {
		outVals[i] = make([]float64, len(valFlags))
	}

	for i, id := range ids {
		// TODO: benchmark at what point sorting the list is important.
		idx := -1
		for j, rid := range rids {
			if rid == id {
				idx = j
				break
			}
		}
		if idx == -1 {
			return nil, fmt.Errorf("ID %d, %d, not found in hlist.", i+1, id)
		}
		for j := range outVals[i] {
			outVals[i][j] = vals[j][idx]
		}
	}

	return outVals, nil
}

func dirContents(dir string) ([]string, error) {
	infos, err := ioutil.ReadDir(dir)
	if err != nil { return nil, err }

	files := make([]string, len(infos))
	for i := range infos {
		files[i] = path.Join(dir, infos[i].Name())
	}

	return files, nil
}

func printVals(ids, snaps []int, inVals, vals [][]float64) {
	for i := range inVals {
		inVals[i] = append(inVals[i], vals[i]...)
	}
	vals = inVals

	idWidth, snapWidth := 0, 0
	valWidths := make([]int, len(vals[0]))
	for i := range ids {
		iWidth := len(fmt.Sprintf("%d", ids[i]))
		sWidth := len(fmt.Sprintf("%d", snaps[i]))
		if iWidth > idWidth { idWidth = iWidth }
		if sWidth > snapWidth { snapWidth = sWidth }
		for j := range vals[i] {
			width := len(fmt.Sprintf("%.10g", vals[i][j]))
			if width > valWidths[j] { valWidths[j] = width }
		}
	}

	rowFmt := fmt.Sprintf("%%%dd %%%dd", idWidth, snapWidth)
	valFmts := make([]string, len(vals[0]))
	for i := range valFmts {
		rowFmt += fmt.Sprintf(" %%%d.10g", valWidths[i])
	}
	rowFmt += "\n"
	for i := range ids {
		args := append([]interface{}{ids[i], snaps[i]}, intr(vals[i])...)
		fmt.Printf(rowFmt, args...)
	}
}

func intr(xs []float64) []interface{} {
	is := make([]interface{}, len(xs))
	for i, x := range xs { is[i] = x }
	return is
}

<|endoftext|>