<commit_before>package container // import "github.com/docker/docker/integration/container"

import (
	"bytes"
	"context"
	"fmt"
	"path/filepath"
	"testing"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/mount"
	"github.com/docker/docker/api/types/network"
	"github.com/docker/docker/client"
	"github.com/docker/docker/integration-cli/daemon"
	"github.com/docker/docker/integration/util/request"
	"github.com/docker/docker/pkg/stdcopy"
	"github.com/docker/docker/pkg/system"
	"github.com/gotestyourself/gotestyourself/fs"
	"github.com/gotestyourself/gotestyourself/skip"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestContainerShmNoLeak(t *testing.T) {
	t.Parallel()
	d := daemon.New(t, "docker", "dockerd", daemon.Config{})
	client, err := d.NewClient()
	if err != nil {
		t.Fatal(err)
	}
	d.StartWithBusybox(t)
	defer d.Stop(t)

	ctx := context.Background()
	cfg := container.Config{
		Image: "busybox",
		Cmd:   []string{"top"},
	}

	ctr, err := client.ContainerCreate(ctx, &cfg, nil, nil, "")
	if err != nil {
		t.Fatal(err)
	}
	defer client.ContainerRemove(ctx, ctr.ID, types.ContainerRemoveOptions{Force: true})

	if err := client.ContainerStart(ctx, ctr.ID, types.ContainerStartOptions{}); err != nil {
		t.Fatal(err)
	}

	// this should recursively bind mount everything in the test daemons root
	// except of course we are hoping that the previous containers /dev/shm mount did not leak into this new container
	hc := container.HostConfig{
		Mounts: []mount.Mount{
			{
				Type:   mount.TypeBind,
				Source: d.Root,
				Target: "/testdaemonroot",
			},
		},
	}
	cfg.Cmd = []string{"/bin/sh", "-c", fmt.Sprintf("mount | grep testdaemonroot | grep containers | grep %s", ctr.ID)}
	cfg.AttachStdout = true
	cfg.AttachStderr = true
	ctrLeak, err := client.ContainerCreate(ctx, &cfg, &hc, nil, "")
	if err != nil {
		t.Fatal(err)
	}

	attach, err := client.ContainerAttach(ctx, ctrLeak.ID, types.ContainerAttachOptions{
		Stream: true,
		Stdout: true,
		Stderr: true,
	})
	if err != nil {
		t.Fatal(err)
	}

	if err := client.ContainerStart(ctx, ctrLeak.ID, types.ContainerStartOptions{}); err != nil {
		t.Fatal(err)
	}

	buf := bytes.NewBuffer(nil)

	if _, err := stdcopy.StdCopy(buf, buf, attach.Reader); err != nil {
		t.Fatal(err)
	}

	out := bytes.TrimSpace(buf.Bytes())
	if !bytes.Equal(out, []byte{}) {
		t.Fatalf("mount leaked: %s", string(out))
	}
}

func TestContainerNetworkMountsNoChown(t *testing.T) {
	// chown only applies to Linux bind mounted volumes; must be same host to verify
	skip.If(t, testEnv.DaemonInfo.OSType != "linux" || !testEnv.IsLocalDaemon())

	defer setupTest(t)()

	ctx := context.Background()

	tmpDir := fs.NewDir(t, "network-file-mounts", fs.WithMode(0755), fs.WithFile("nwfile", "network file bind mount", fs.WithMode(0644)))
	defer tmpDir.Remove()

	tmpNWFileMount := tmpDir.Join("nwfile")

	config := container.Config{
		Image: "busybox",
	}
	hostConfig := container.HostConfig{
		Mounts: []mount.Mount{
			{
				Type:   "bind",
				Source: tmpNWFileMount,
				Target: "/etc/resolv.conf",
			},
			{
				Type:   "bind",
				Source: tmpNWFileMount,
				Target: "/etc/hostname",
			},
			{
				Type:   "bind",
				Source: tmpNWFileMount,
				Target: "/etc/hosts",
			},
		},
	}

	cli, err := client.NewEnvClient()
	require.NoError(t, err)
	defer cli.Close()

	ctrCreate, err := cli.ContainerCreate(ctx, &config, &hostConfig, &network.NetworkingConfig{}, "")
	require.NoError(t, err)
	// container will exit immediately because of no tty, but we only need the start sequence to test the condition
	err = cli.ContainerStart(ctx, ctrCreate.ID, types.ContainerStartOptions{})
	require.NoError(t, err)

	// Check that host-located bind mount network file did not change ownership when the container was started
	// Note: If the user specifies a mountpath from the host, we should not be
	// attempting to chown files outside the daemon's metadata directory
	// (represented by `daemon.repository` at init time).
	// This forces users who want to use user namespaces to handle the
	// ownership needs of any external files mounted as network files
	// (/etc/resolv.conf, /etc/hosts, /etc/hostname) separately from the
	// daemon. In all other volume/bind mount situations we have taken this
	// same line--we don't chown host file content.
	// See GitHub PR 34224 for details.
	statT, err := system.Stat(tmpNWFileMount)
	require.NoError(t, err)
	assert.Equal(t, uint32(0), statT.UID(), "bind mounted network file should not change ownership from root")
}

func TestMountDaemonRoot(t *testing.T) {
	t.Parallel()

	client := request.NewAPIClient(t)
	ctx := context.Background()
	info, err := client.Info(ctx)
	if err != nil {
		t.Fatal(err)
	}

	for _, test := range []struct {
		desc        string
		propagation mount.Propagation
		expected    mount.Propagation
	}{
		{
			desc:        "default",
			propagation: "",
			expected:    mount.PropagationRSlave,
		},
		{
			desc:        "private",
			propagation: mount.PropagationPrivate,
		},
		{
			desc:        "rprivate",
			propagation: mount.PropagationRPrivate,
		},
		{
			desc:        "slave",
			propagation: mount.PropagationSlave,
		},
		{
			desc:        "rslave",
			propagation: mount.PropagationRSlave,
			expected:    mount.PropagationRSlave,
		},
		{
			desc:        "shared",
			propagation: mount.PropagationShared,
		},
		{
			desc:        "rshared",
			propagation: mount.PropagationRShared,
			expected:    mount.PropagationRShared,
		},
	} {
		t.Run(test.desc, func(t *testing.T) {
			test := test
			t.Parallel()

			propagationSpec := fmt.Sprintf(":%s", test.propagation)
			if test.propagation == "" {
				propagationSpec = ""
			}
			bindSpecRoot := info.DockerRootDir + ":" + "/foo" + propagationSpec
			bindSpecSub := filepath.Join(info.DockerRootDir, "containers") + ":/foo" + propagationSpec

			for name, hc := range map[string]*container.HostConfig{
				"bind root":    {Binds: []string{bindSpecRoot}},
				"bind subpath": {Binds: []string{bindSpecSub}},
				"mount root": {
					Mounts: []mount.Mount{
						{
							Type:        mount.TypeBind,
							Source:      info.DockerRootDir,
							Target:      "/foo",
							BindOptions: &mount.BindOptions{Propagation: test.propagation},
						},
					},
				},
				"mount subpath": {
					Mounts: []mount.Mount{
						{
							Type:        mount.TypeBind,
							Source:      filepath.Join(info.DockerRootDir, "containers"),
							Target:      "/foo",
							BindOptions: &mount.BindOptions{Propagation: test.propagation},
						},
					},
				},
			} {
				t.Run(name, func(t *testing.T) {
					hc := hc
					t.Parallel()

					c, err := client.ContainerCreate(ctx, &container.Config{
						Image: "busybox",
						Cmd:   []string{"true"},
					}, hc, nil, "")

					if err != nil {
						if test.expected != "" {
							t.Fatal(err)
						}
						// expected an error, so this is ok and should not continue
						return
					}
					if test.expected == "" {
						t.Fatal("expected create to fail")
					}

					defer func() {
						if err := client.ContainerRemove(ctx, c.ID, types.ContainerRemoveOptions{Force: true}); err != nil {
							panic(err)
						}
					}()

					inspect, err := client.ContainerInspect(ctx, c.ID)
					if err != nil {
						t.Fatal(err)
					}
					if len(inspect.Mounts) != 1 {
						t.Fatalf("unexpected number of mounts: %+v", inspect.Mounts)
					}

					m := inspect.Mounts[0]
					if m.Propagation != test.expected {
						t.Fatalf("got unexpected propagation mode, expected %q, got: %v", test.expected, m.Propagation)
					}
				})
			}
		})
	}
}
<commit_msg>Fix import path<commit_after>package container // import "github.com/docker/docker/integration/container"

import (
	"bytes"
	"context"
	"fmt"
	"path/filepath"
	"testing"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/mount"
	"github.com/docker/docker/api/types/network"
	"github.com/docker/docker/client"
	"github.com/docker/docker/integration-cli/daemon"
	"github.com/docker/docker/integration/internal/request"
	"github.com/docker/docker/pkg/stdcopy"
	"github.com/docker/docker/pkg/system"
	"github.com/gotestyourself/gotestyourself/fs"
	"github.com/gotestyourself/gotestyourself/skip"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestContainerShmNoLeak(t *testing.T) {
	t.Parallel()
	d := daemon.New(t, "docker", "dockerd", daemon.Config{})
	client, err := d.NewClient()
	if err != nil {
		t.Fatal(err)
	}
	d.StartWithBusybox(t)
	defer d.Stop(t)

	ctx := context.Background()
	cfg := container.Config{
		Image: "busybox",
		Cmd:   []string{"top"},
	}

	ctr, err := client.ContainerCreate(ctx, &cfg, nil, nil, "")
	if err != nil {
		t.Fatal(err)
	}
	defer client.ContainerRemove(ctx, ctr.ID, types.ContainerRemoveOptions{Force: true})

	if err := client.ContainerStart(ctx, ctr.ID, types.ContainerStartOptions{}); err != nil {
		t.Fatal(err)
	}

	// this should recursively bind mount everything in the test daemons root
	// except of course we are hoping that the previous containers /dev/shm mount did not leak into this new container
	hc := container.HostConfig{
		Mounts: []mount.Mount{
			{
				Type:   mount.TypeBind,
				Source: d.Root,
				Target: "/testdaemonroot",
			},
		},
	}
	cfg.Cmd = []string{"/bin/sh", "-c", fmt.Sprintf("mount | grep testdaemonroot | grep containers | grep %s", ctr.ID)}
	cfg.AttachStdout = true
	cfg.AttachStderr = true
	ctrLeak, err := client.ContainerCreate(ctx, &cfg, &hc, nil, "")
	if err != nil {
		t.Fatal(err)
	}

	attach, err := client.ContainerAttach(ctx, ctrLeak.ID, types.ContainerAttachOptions{
		Stream: true,
		Stdout: true,
		Stderr: true,
	})
	if err != nil {
		t.Fatal(err)
	}

	if err := client.ContainerStart(ctx, ctrLeak.ID, types.ContainerStartOptions{}); err != nil {
		t.Fatal(err)
	}

	buf := bytes.NewBuffer(nil)

	if _, err := stdcopy.StdCopy(buf, buf, attach.Reader); err != nil {
		t.Fatal(err)
	}

	out := bytes.TrimSpace(buf.Bytes())
	if !bytes.Equal(out, []byte{}) {
		t.Fatalf("mount leaked: %s", string(out))
	}
}

func TestContainerNetworkMountsNoChown(t *testing.T) {
	// chown only applies to Linux bind mounted volumes; must be same host to verify
	skip.If(t, testEnv.DaemonInfo.OSType != "linux" || !testEnv.IsLocalDaemon())

	defer setupTest(t)()

	ctx := context.Background()

	tmpDir := fs.NewDir(t, "network-file-mounts", fs.WithMode(0755), fs.WithFile("nwfile", "network file bind mount", fs.WithMode(0644)))
	defer tmpDir.Remove()

	tmpNWFileMount := tmpDir.Join("nwfile")

	config := container.Config{
		Image: "busybox",
	}
	hostConfig := container.HostConfig{
		Mounts: []mount.Mount{
			{
				Type:   "bind",
				Source: tmpNWFileMount,
				Target: "/etc/resolv.conf",
			},
			{
				Type:   "bind",
				Source: tmpNWFileMount,
				Target: "/etc/hostname",
			},
			{
				Type:   "bind",
				Source: tmpNWFileMount,
				Target: "/etc/hosts",
			},
		},
	}

	cli, err := client.NewEnvClient()
	require.NoError(t, err)
	defer cli.Close()

	ctrCreate, err := cli.ContainerCreate(ctx, &config, &hostConfig, &network.NetworkingConfig{}, "")
	require.NoError(t, err)
	// container will exit immediately because of no tty, but we only need the start sequence to test the condition
	err = cli.ContainerStart(ctx, ctrCreate.ID, types.ContainerStartOptions{})
	require.NoError(t, err)

	// Check that host-located bind mount network file did not change ownership when the container was started
	// Note: If the user specifies a mountpath from the host, we should not be
	// attempting to chown files outside the daemon's metadata directory
	// (represented by `daemon.repository` at init time).
	// This forces users who want to use user namespaces to handle the
	// ownership needs of any external files mounted as network files
	// (/etc/resolv.conf, /etc/hosts, /etc/hostname) separately from the
	// daemon. In all other volume/bind mount situations we have taken this
	// same line--we don't chown host file content.
	// See GitHub PR 34224 for details.
	statT, err := system.Stat(tmpNWFileMount)
	require.NoError(t, err)
	assert.Equal(t, uint32(0), statT.UID(), "bind mounted network file should not change ownership from root")
}

func TestMountDaemonRoot(t *testing.T) {
	t.Parallel()

	client := request.NewAPIClient(t)
	ctx := context.Background()
	info, err := client.Info(ctx)
	if err != nil {
		t.Fatal(err)
	}

	for _, test := range []struct {
		desc        string
		propagation mount.Propagation
		expected    mount.Propagation
	}{
		{
			desc:        "default",
			propagation: "",
			expected:    mount.PropagationRSlave,
		},
		{
			desc:        "private",
			propagation: mount.PropagationPrivate,
		},
		{
			desc:        "rprivate",
			propagation: mount.PropagationRPrivate,
		},
		{
			desc:        "slave",
			propagation: mount.PropagationSlave,
		},
		{
			desc:        "rslave",
			propagation: mount.PropagationRSlave,
			expected:    mount.PropagationRSlave,
		},
		{
			desc:        "shared",
			propagation: mount.PropagationShared,
		},
		{
			desc:        "rshared",
			propagation: mount.PropagationRShared,
			expected:    mount.PropagationRShared,
		},
	} {
		t.Run(test.desc, func(t *testing.T) {
			test := test
			t.Parallel()

			propagationSpec := fmt.Sprintf(":%s", test.propagation)
			if test.propagation == "" {
				propagationSpec = ""
			}
			bindSpecRoot := info.DockerRootDir + ":" + "/foo" + propagationSpec
			bindSpecSub := filepath.Join(info.DockerRootDir, "containers") + ":/foo" + propagationSpec

			for name, hc := range map[string]*container.HostConfig{
				"bind root":    {Binds: []string{bindSpecRoot}},
				"bind subpath": {Binds: []string{bindSpecSub}},
				"mount root": {
					Mounts: []mount.Mount{
						{
							Type:        mount.TypeBind,
							Source:      info.DockerRootDir,
							Target:      "/foo",
							BindOptions: &mount.BindOptions{Propagation: test.propagation},
						},
					},
				},
				"mount subpath": {
					Mounts: []mount.Mount{
						{
							Type:        mount.TypeBind,
							Source:      filepath.Join(info.DockerRootDir, "containers"),
							Target:      "/foo",
							BindOptions: &mount.BindOptions{Propagation: test.propagation},
						},
					},
				},
			} {
				t.Run(name, func(t *testing.T) {
					hc := hc
					t.Parallel()

					c, err := client.ContainerCreate(ctx, &container.Config{
						Image: "busybox",
						Cmd:   []string{"true"},
					}, hc, nil, "")

					if err != nil {
						if test.expected != "" {
							t.Fatal(err)
						}
						// expected an error, so this is ok and should not continue
						return
					}
					if test.expected == "" {
						t.Fatal("expected create to fail")
					}

					defer func() {
						if err := client.ContainerRemove(ctx, c.ID, types.ContainerRemoveOptions{Force: true}); err != nil {
							panic(err)
						}
					}()

					inspect, err := client.ContainerInspect(ctx, c.ID)
					if err != nil {
						t.Fatal(err)
					}
					if len(inspect.Mounts) != 1 {
						t.Fatalf("unexpected number of mounts: %+v", inspect.Mounts)
					}

					m := inspect.Mounts[0]
					if m.Propagation != test.expected {
						t.Fatalf("got unexpected propagation mode, expected %q, got: %v", test.expected, m.Propagation)
					}
				})
			}
		})
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Markus Lindenberg, Stig Bakken
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bufio"
	"flag"
	"fmt"
	"net/http"
	"os"
	"os/exec"
	"os/signal"
	"regexp"
	"syscall"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/log"
)

const (
	namespace = "varnish_request"
)

type path_mappings struct {
	Pattern     *regexp.Regexp
	Replacement string
}

func main() {
	var (
		listenAddress = flag.String("http.port", ":9151", "Host/port for HTTP server")
		metricsPath   = flag.String("http.metricsurl", "/metrics", "Prometheus metrics path")
		httpHost      = flag.String("varnish.host", "", "Virtual host to look for in Varnish logs (defaults to all hosts)")
		mappingsFile  = flag.String("varnish.path-mappings", "", "Name of file with path mappings")
		instance      = flag.String("varnish.instance", "", "Name of Varnish instance")
		befirstbyte   = flag.Bool("varnish.firstbyte", false, "Also export metrics for backend time to first byte")
		user_query    = flag.String("varnish.query", "", "VSL query override (defaults to one that is generated")
		sizes         = flag.Bool("varnish.sizes", false, "Also export metrics for response size")
	)
	flag.Parse()

	// Listen to signals
	sigchan := make(chan os.Signal, 1)
	signal.Notify(sigchan, syscall.SIGTERM, syscall.SIGINT)

	// Set up 'varnishncsa' pipe
	cmdName := "varnishncsa"
	vslQuery := buildVslQuery(*httpHost, *user_query)
	varnishFormat := buildVarnishncsaFormat(*befirstbyte, *sizes, *user_query)
	cmdArgs := buildVarnishncsaArgs(vslQuery, *instance, varnishFormat)
	log.Infof("Running command: %v %v\n", cmdName, cmdArgs)
	cmd := exec.Command(cmdName, cmdArgs...)
	cmdReader, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}
	scanner := bufio.NewScanner(cmdReader)

	path_mappings, err := parseMappings(*mappingsFile)
	if err != nil {
		log.Fatal(err)
	}

	// Setup metrics
	varnishMessages := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: namespace,
		Name:      "exporter_log_messages",
		Help:      "Current total log messages received.",
	})
	err = prometheus.Register(varnishMessages)
	if err != nil {
		log.Fatal(err)
	}
	varnishParseFailures := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: namespace,
		Name:      "exporter_log_parse_failure",
		Help:      "Number of errors while parsing log messages.",
	})
	err = prometheus.Register(varnishParseFailures)
	if err != nil {
		log.Fatal(err)
	}
	var msgs int64

	go func() {
		for scanner.Scan() {
			varnishMessages.Inc()
			content := scanner.Text()
			msgs++
			metrics, labels, err := parseMessage(content, path_mappings)
			if err != nil {
				log.Error(err)
				continue
			}
			for _, metric := range metrics {
				var collector prometheus.Collector
				collector, err = prometheus.RegisterOrGet(prometheus.NewHistogramVec(prometheus.HistogramOpts{
					Namespace: namespace,
					Name:      metric.Name,
					Help:      fmt.Sprintf("Varnish request log value for %s", metric.Name),
				}, labels.Names))
				if err != nil {
					log.Error(err)
					continue
				}
				collector.(*prometheus.HistogramVec).WithLabelValues(labels.Values...).Observe(metric.Value)
			}
		}
	}()

	// Setup HTTP server
	http.Handle(*metricsPath, prometheus.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`
             <title>Varnish Request Exporter</title>

Varnish Request Exporter

<a href='` + *metricsPath + `'>Metrics

             `))
	})

	go func() {
		log.Infof("Starting Server: %s", *listenAddress)
		log.Fatal(http.ListenAndServe(*listenAddress, nil))
	}()

	go func() {
		err = cmd.Start()
		if err != nil {
			log.Fatal(err)
		}
		err = cmd.Wait()
		if err != nil {
			log.Fatal(err)
		}
		log.Infof("varnishncsa command exited")
		log.Infof("Messages received: %d", msgs)
		os.Exit(0)
	}()

	s := <-sigchan
	log.Infof("Received %v, terminating", s)
	log.Infof("Messages received: %d", msgs)

	os.Exit(0)
}

func parseMappings(mappings_file string) (mappings []path_mappings, err error) {
	mappings = make([]path_mappings, 0)
	if mappings_file == "" {
		return
	}
	in_file, err := os.Open(mappings_file)
	if err != nil {
		log.Fatal(err)
	}
	scanner := bufio.NewScanner(in_file)
	scanner.Split(bufio.ScanLines)
	comment_re := regexp.MustCompile("(#.*|^\\s+|\\s+$)")
	split_re := regexp.MustCompile("\\s+")
	line_no := 0
	for scanner.Scan() {
		line_no++
		line := comment_re.ReplaceAllString(scanner.Text(), "")
		if line == "" {
			continue
		}
		parts := split_re.Split(line, 2)
		switch len(parts) {
		case 1:
			log.Debugf("mapping strip: %s", parts[0])
			mappings = append(mappings, path_mappings{regexp.MustCompile(parts[0]), ""})
		case 2:
			log.Debugf("mapping replace: %s => %s", parts[0], parts[1])
			mappings = append(mappings, path_mappings{regexp.MustCompile(parts[0]), parts[1]})
		}
	}
	in_file.Close()
	return
}

func buildVslQuery(httpHost string, user_query string) (query string) {
	query = user_query
	if httpHost != "" {
		if query != "" {
			query += " and "
		}
		query += "ReqHeader:host eq \"" + httpHost + "\""
	}
	return
}

func buildVarnishncsaFormat(befirstbyte bool, sizes bool, user_format string) (format string) {
	if user_format != "" {
		format = user_format + " "
	}
	format += "method=\"%m\" status=%s path=\"%U\" cache=\"%{Varnish:hitmiss}x\" host=\"%{host}i\" time:%D"
	if befirstbyte {
		format += " time_firstbyte:%{Varnish:time_firstbyte}x"
	}
	if sizes {
		format += " respsize:%b"
	}
	return
}

func buildVarnishncsaArgs(vsl_query string, instance string, format string) (args []string) {
	args = make([]string, 0, 6)
	args = append(args, "-F")
	args = append(args, format)
	if vsl_query != "" {
		args = append(args, "-q")
		args = append(args, vsl_query)
	}
	if instance != "" {
		args = append(args, "-n", instance)
	}
	return
}
<commit_msg>Fix bug with `--varnish.query`<commit_after>// Copyright 2016 Markus Lindenberg, Stig Bakken
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bufio"
	"flag"
	"fmt"
	"net/http"
	"os"
	"os/exec"
	"os/signal"
	"regexp"
	"syscall"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/log"
)

const (
	namespace = "varnish_request"
)

type path_mappings struct {
	Pattern     *regexp.Regexp
	Replacement string
}

func main() {
	var (
		listenAddress = flag.String("http.port", ":9151", "Host/port for HTTP server")
		metricsPath   = flag.String("http.metricsurl", "/metrics", "Prometheus metrics path")
		httpHost      = flag.String("varnish.host", "", "Virtual host to look for in Varnish logs (defaults to all hosts)")
		mappingsFile  = flag.String("varnish.path-mappings", "", "Name of file with path mappings")
		instance      = flag.String("varnish.instance", "", "Name of Varnish instance")
		befirstbyte   = flag.Bool("varnish.firstbyte", false, "Also export metrics for backend time to first byte")
		user_query    = flag.String("varnish.query", "", "VSL query override (defaults to one that is generated")
		sizes         = flag.Bool("varnish.sizes", false, "Also export metrics for response size")
	)
	flag.Parse()

	// Listen to signals
	sigchan := make(chan os.Signal, 1)
	signal.Notify(sigchan, syscall.SIGTERM, syscall.SIGINT)

	// Set up 'varnishncsa' pipe
	cmdName := "varnishncsa"
	vslQuery := buildVslQuery(*httpHost, *user_query)
	varnishFormat := buildVarnishncsaFormat(*befirstbyte, *sizes)
	cmdArgs := buildVarnishncsaArgs(vslQuery, *instance, varnishFormat)
	log.Infof("Running command: %v %v\n", cmdName, cmdArgs)
	cmd := exec.Command(cmdName, cmdArgs...)
	cmdReader, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}
	scanner := bufio.NewScanner(cmdReader)

	path_mappings, err := parseMappings(*mappingsFile)
	if err != nil {
		log.Fatal(err)
	}

	// Setup metrics
	varnishMessages := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: namespace,
		Name:      "exporter_log_messages",
		Help:      "Current total log messages received.",
	})
	err = prometheus.Register(varnishMessages)
	if err != nil {
		log.Fatal(err)
	}
	varnishParseFailures := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: namespace,
		Name:      "exporter_log_parse_failure",
		Help:      "Number of errors while parsing log messages.",
	})
	err = prometheus.Register(varnishParseFailures)
	if err != nil {
		log.Fatal(err)
	}
	var msgs int64

	go func() {
		for scanner.Scan() {
			varnishMessages.Inc()
			content := scanner.Text()
			msgs++
			metrics, labels, err := parseMessage(content, path_mappings)
			if err != nil {
				log.Error(err)
				continue
			}
			for _, metric := range metrics {
				var collector prometheus.Collector
				collector, err = prometheus.RegisterOrGet(prometheus.NewHistogramVec(prometheus.HistogramOpts{
					Namespace: namespace,
					Name:      metric.Name,
					Help:      fmt.Sprintf("Varnish request log value for %s", metric.Name),
				}, labels.Names))
				if err != nil {
					log.Error(err)
					continue
				}
				collector.(*prometheus.HistogramVec).WithLabelValues(labels.Values...).Observe(metric.Value)
			}
		}
	}()

	// Setup HTTP server
	http.Handle(*metricsPath, prometheus.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`
             <title>Varnish Request Exporter</title>

Varnish Request Exporter

<a href='` + *metricsPath + `'>Metrics

             `))
	})

	go func() {
		log.Infof("Starting Server: %s", *listenAddress)
		log.Fatal(http.ListenAndServe(*listenAddress, nil))
	}()

	go func() {
		err = cmd.Start()
		if err != nil {
			log.Fatal(err)
		}
		err = cmd.Wait()
		if err != nil {
			log.Fatal(err)
		}
		log.Infof("varnishncsa command exited")
		log.Infof("Messages received: %d", msgs)
		os.Exit(0)
	}()

	s := <-sigchan
	log.Infof("Received %v, terminating", s)
	log.Infof("Messages received: %d", msgs)

	os.Exit(0)
}

func parseMappings(mappings_file string) (mappings []path_mappings, err error) {
	mappings = make([]path_mappings, 0)
	if mappings_file == "" {
		return
	}
	in_file, err := os.Open(mappings_file)
	if err != nil {
		log.Fatal(err)
	}
	scanner := bufio.NewScanner(in_file)
	scanner.Split(bufio.ScanLines)
	comment_re := regexp.MustCompile("(#.*|^\\s+|\\s+$)")
	split_re := regexp.MustCompile("\\s+")
	line_no := 0
	for scanner.Scan() {
		line_no++
		line := comment_re.ReplaceAllString(scanner.Text(), "")
		if line == "" {
			continue
		}
		parts := split_re.Split(line, 2)
		switch len(parts) {
		case 1:
			log.Debugf("mapping strip: %s", parts[0])
			mappings = append(mappings, path_mappings{regexp.MustCompile(parts[0]), ""})
		case 2:
			log.Debugf("mapping replace: %s => %s", parts[0], parts[1])
			mappings = append(mappings, path_mappings{regexp.MustCompile(parts[0]), parts[1]})
		}
	}
	in_file.Close()
	return
}

func buildVslQuery(httpHost string, user_query string) (query string) {
	query = user_query
	if httpHost != "" {
		if query != "" {
			query += " and "
		}
		query += "ReqHeader:host eq \"" + httpHost + "\""
	}
	return
}

func buildVarnishncsaFormat(befirstbyte bool, sizes bool) (format string) {
	format += "method=\"%m\" status=%s path=\"%U\" cache=\"%{Varnish:hitmiss}x\" host=\"%{host}i\" time:%D"
	if befirstbyte {
		format += " time_firstbyte:%{Varnish:time_firstbyte}x"
	}
	if sizes {
		format += " respsize:%b"
	}
	return
}

func buildVarnishncsaArgs(vsl_query string, instance string, format string) (args []string) {
	args = make([]string, 0, 6)
	args = append(args, "-F")
	args = append(args, format)
	if vsl_query != "" {
		args = append(args, "-q")
		args = append(args, vsl_query)
	}
	if instance != "" {
		args = append(args, "-n", instance)
	}
	return
}
<|endoftext|>


<commit_before><commit_msg>Signed-off-by: Alborz A. Amir-Khalili <alborza@bisicl.ad.ece.ubc.ca><commit_after><|endoftext|>


<commit_before>// +build integration

/*
Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package containeranalysis

import (
	"fmt"
	"testing"
	"time"

	kritisv1beta1 "github.com/grafeas/kritis/pkg/kritis/apis/kritis/v1beta1"
	"github.com/grafeas/kritis/pkg/kritis/metadata"
	"github.com/grafeas/kritis/pkg/kritis/secrets"
	"github.com/grafeas/kritis/pkg/kritis/testutil"
	"github.com/grafeas/kritis/pkg/kritis/util"
	"google.golang.org/genproto/googleapis/devtools/containeranalysis/v1beta1/grafeas"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

var (
	IntTestNoteName = "test-aa-note"
	IntProject      = "kritis-int-test"
)

func GetAA() *kritisv1beta1.AttestationAuthority {
	aa := &kritisv1beta1.AttestationAuthority{
		Spec: kritisv1beta1.AttestationAuthoritySpec{
			NoteReference: fmt.Sprintf("projects/%s/notes/%s", IntProject, IntTestNoteName),
		},
		ObjectMeta: metav1.ObjectMeta{
			Name: IntTestNoteName,
		},
	}
	return aa
}

func TestGetVulnerabilities(t *testing.T) {
	client, err := New()
	if err != nil {
		t.Fatalf("Could not initialize the client %s", err)
	}
	vuln, err := client.Vulnerabilities("gcr.io/kritis-int-test/java-with-vulnz@sha256:358687cfd3ec8e1dfeb2bf51b5110e4e16f6df71f64fba01986f720b2fcba68a")
	if err != nil {
		t.Fatalf("Found err %s", err)
	}
	if vuln == nil {
		t.Fatalf("Expected some vulnerabilities. Nil found")
	}
}

func TestCreateAttestationNoteAndOccurrence(t *testing.T) {
	client, err := New()
	if err != nil {
		t.Fatalf("Could not initialize the client %s", err)
	}
	aa := GetAA()
	_, err = client.CreateAttestationNote(aa)
	if err != nil {
		t.Fatalf("Unexpected error while creating Note %v", err)
	}
	defer client.DeleteAttestationNote(aa)

	note, err := client.AttestationNote(aa)
	if err != nil {
		t.Fatalf("Unexpected no error while getting attestation note %v", err)
	}

	expectedNoteName := fmt.Sprintf("projects/%s/notes/%s", IntProject, IntTestNoteName)
	if note.Name != expectedNoteName {
		t.Fatalf("Expected %s.\n Got %s", expectedNoteName, note.Name)
	}

	actualHint := note.GetAttestationAuthority().Hint.GetHumanReadableName()
	if actualHint != IntTestNoteName {
		t.Fatalf("Expected %s.\n Got %s", expectedNoteName, actualHint)
	}

	// Test Create Attestation Occurence
	pub, priv := testutil.CreateKeyPair(t, "test")
	pgpKey, err := secrets.NewPgpKey(priv, "", pub)
	if err != nil {
		t.Fatalf("Unexpected error while creating PGP key %v", err)
	}
	secret := &secrets.PGPSigningSecret{
		PgpKey:     pgpKey,
		SecretName: "test",
	}

	proj, _, err := metadata.ParseNoteReference(aa.Spec.NoteReference)
	if err != nil {
		t.Fatalf("Failed to extract project ID %v", err)
	}
	occ, err := client.CreateAttestationOccurrence(note, testutil.IntTestImage, secret, proj)
	if err != nil {
		t.Fatalf("Unexpected error while creating Occurence %v", err)
	}
	expectedPgpKeyID := pgpKey.Fingerprint()
	if err != nil {
		t.Fatalf("Unexpected error while extracting PGP key id %v", err)
	}
	pgpKeyID := occ.GetAttestation().GetAttestation().GetPgpSignedAttestation().GetPgpKeyId()
	if pgpKeyID != expectedPgpKeyID {
		t.Errorf("Expected PGP key id: %q, got %q", expectedPgpKeyID, pgpKeyID)
	}
	defer client.DeleteOccurrence(occ.GetName())

	// Keep trying to list attestation occurrences until we time out.
	// Because the staleness bound is on the order of seconds, no need to try faster than once a second.
	timeout := time.After(20 * time.Second)
	tick := time.Tick(1 * time.Second)
	for {
		select {
		// Got a timeout! fail with a timeout error
		case <-timeout:
			t.Fatal("Should have created at least 1 occurrence")

			// Got a tick, we should check note occurrences
		case <-tick:
			if occurrences, err := client.Attestations(testutil.IntTestImage, aa); err != nil {
				t.Fatalf("Failed to retrieve attestations: %v", err)
			} else if len(occurrences) > 0 {
				// Successfully retrieved attestations, exit the loop and the test.
				return
			}
		}
	}
}

func TestGetMultiplePagesVulnerabilities(t *testing.T) {
	client, err := New()
	if err != nil {
		t.Fatalf("Could not initialize the client %s", err)
	}

	// Set PageSize to 300
	createListOccurrencesRequest = createListOccurrencesRequestTest

	vuln, err := client.Vulnerabilities("gcr.io/kritis-int-test/java-with-vulnz@sha256:358687cfd3ec8e1dfeb2bf51b5110e4e16f6df71f64fba01986f720b2fcba68a")
	if err != nil {
		t.Fatalf("Found err %s", err)
	}

	if len(vuln) <= 900 {
		t.Fatalf("Pagination error: expected at least 900 results on image 'gcr.io/kritis-int-test/java-with-vulnz'. Received %d.", len(vuln))
	}
}

func createListOccurrencesRequestTest(containerImage, kind string) *grafeas.ListOccurrencesRequest {
	return &grafeas.ListOccurrencesRequest{
		Filter:   fmt.Sprintf("resourceUrl=%q AND kind=%q", util.GetResourceURL(containerImage), kind),
		Parent:   fmt.Sprintf("projects/%s", getProjectFromContainerImage(containerImage)),
		PageSize: int32(100),
	}
}
<commit_msg>Correct pagesize comment<commit_after>// +build integration

/*
Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package containeranalysis

import (
	"fmt"
	"testing"
	"time"

	kritisv1beta1 "github.com/grafeas/kritis/pkg/kritis/apis/kritis/v1beta1"
	"github.com/grafeas/kritis/pkg/kritis/metadata"
	"github.com/grafeas/kritis/pkg/kritis/secrets"
	"github.com/grafeas/kritis/pkg/kritis/testutil"
	"github.com/grafeas/kritis/pkg/kritis/util"
	"google.golang.org/genproto/googleapis/devtools/containeranalysis/v1beta1/grafeas"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

var (
	IntTestNoteName = "test-aa-note"
	IntProject      = "kritis-int-test"
)

func GetAA() *kritisv1beta1.AttestationAuthority {
	aa := &kritisv1beta1.AttestationAuthority{
		Spec: kritisv1beta1.AttestationAuthoritySpec{
			NoteReference: fmt.Sprintf("projects/%s/notes/%s", IntProject, IntTestNoteName),
		},
		ObjectMeta: metav1.ObjectMeta{
			Name: IntTestNoteName,
		},
	}
	return aa
}

func TestGetVulnerabilities(t *testing.T) {
	client, err := New()
	if err != nil {
		t.Fatalf("Could not initialize the client %s", err)
	}
	vuln, err := client.Vulnerabilities("gcr.io/kritis-int-test/java-with-vulnz@sha256:358687cfd3ec8e1dfeb2bf51b5110e4e16f6df71f64fba01986f720b2fcba68a")
	if err != nil {
		t.Fatalf("Found err %s", err)
	}
	if vuln == nil {
		t.Fatalf("Expected some vulnerabilities. Nil found")
	}
}

func TestCreateAttestationNoteAndOccurrence(t *testing.T) {
	client, err := New()
	if err != nil {
		t.Fatalf("Could not initialize the client %s", err)
	}
	aa := GetAA()
	_, err = client.CreateAttestationNote(aa)
	if err != nil {
		t.Fatalf("Unexpected error while creating Note %v", err)
	}
	defer client.DeleteAttestationNote(aa)

	note, err := client.AttestationNote(aa)
	if err != nil {
		t.Fatalf("Unexpected no error while getting attestation note %v", err)
	}

	expectedNoteName := fmt.Sprintf("projects/%s/notes/%s", IntProject, IntTestNoteName)
	if note.Name != expectedNoteName {
		t.Fatalf("Expected %s.\n Got %s", expectedNoteName, note.Name)
	}

	actualHint := note.GetAttestationAuthority().Hint.GetHumanReadableName()
	if actualHint != IntTestNoteName {
		t.Fatalf("Expected %s.\n Got %s", expectedNoteName, actualHint)
	}

	// Test Create Attestation Occurence
	pub, priv := testutil.CreateKeyPair(t, "test")
	pgpKey, err := secrets.NewPgpKey(priv, "", pub)
	if err != nil {
		t.Fatalf("Unexpected error while creating PGP key %v", err)
	}
	secret := &secrets.PGPSigningSecret{
		PgpKey:     pgpKey,
		SecretName: "test",
	}

	proj, _, err := metadata.ParseNoteReference(aa.Spec.NoteReference)
	if err != nil {
		t.Fatalf("Failed to extract project ID %v", err)
	}
	occ, err := client.CreateAttestationOccurrence(note, testutil.IntTestImage, secret, proj)
	if err != nil {
		t.Fatalf("Unexpected error while creating Occurence %v", err)
	}
	expectedPgpKeyID := pgpKey.Fingerprint()
	if err != nil {
		t.Fatalf("Unexpected error while extracting PGP key id %v", err)
	}
	pgpKeyID := occ.GetAttestation().GetAttestation().GetPgpSignedAttestation().GetPgpKeyId()
	if pgpKeyID != expectedPgpKeyID {
		t.Errorf("Expected PGP key id: %q, got %q", expectedPgpKeyID, pgpKeyID)
	}
	defer client.DeleteOccurrence(occ.GetName())

	// Keep trying to list attestation occurrences until we time out.
	// Because the staleness bound is on the order of seconds, no need to try faster than once a second.
	timeout := time.After(20 * time.Second)
	tick := time.Tick(1 * time.Second)
	for {
		select {
		// Got a timeout! fail with a timeout error
		case <-timeout:
			t.Fatal("Should have created at least 1 occurrence")

			// Got a tick, we should check note occurrences
		case <-tick:
			if occurrences, err := client.Attestations(testutil.IntTestImage, aa); err != nil {
				t.Fatalf("Failed to retrieve attestations: %v", err)
			} else if len(occurrences) > 0 {
				// Successfully retrieved attestations, exit the loop and the test.
				return
			}
		}
	}
}

func TestGetMultiplePagesVulnerabilities(t *testing.T) {
	client, err := New()
	if err != nil {
		t.Fatalf("Could not initialize the client %s", err)
	}

	// Set PageSize to 100
	createListOccurrencesRequest = createListOccurrencesRequestTest

	vuln, err := client.Vulnerabilities("gcr.io/kritis-int-test/java-with-vulnz@sha256:358687cfd3ec8e1dfeb2bf51b5110e4e16f6df71f64fba01986f720b2fcba68a")
	if err != nil {
		t.Fatalf("Found err %s", err)
	}

	if len(vuln) <= 900 {
		t.Fatalf("Pagination error: expected at least 900 results on image 'gcr.io/kritis-int-test/java-with-vulnz'. Received %d.", len(vuln))
	}
}

func createListOccurrencesRequestTest(containerImage, kind string) *grafeas.ListOccurrencesRequest {
	return &grafeas.ListOccurrencesRequest{
		Filter:   fmt.Sprintf("resourceUrl=%q AND kind=%q", util.GetResourceURL(containerImage), kind),
		Parent:   fmt.Sprintf("projects/%s", getProjectFromContainerImage(containerImage)),
		PageSize: int32(100),
	}
}
<|endoftext|>


<commit_before>// Copyright 2018 Documize Inc. <legal@documize.com>. All rights reserved.
//
// This software (Documize Community Edition) is licensed under
// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
//
// You can operate outside the AGPL restrictions by purchasing
// Documize Enterprise Edition and obtaining a commercial license
// by contacting <sales@documize.com>.
//
// https://documize.com

package mysql

import (
	"database/sql"
	"fmt"
	"time"

	"github.com/documize/community/core/env"
	"github.com/documize/community/domain"
	"github.com/documize/community/domain/store/mysql"
	"github.com/documize/community/model/group"
	"github.com/pkg/errors"
)

// Scope provides data access to MySQL.
type Scope struct {
	Runtime *env.Runtime
}

// Add inserts new user group into store.
func (s Scope) Add(ctx domain.RequestContext, g group.Group) (err error) {
	g.Created = time.Now().UTC()
	g.Revised = time.Now().UTC()

	_, err = ctx.Transaction.Exec("INSERT INTO role (refid, orgid, role, purpose, created, revised) VALUES (?, ?, ?, ?, ?, ?)",
		g.RefID, g.OrgID, g.Name, g.Purpose, g.Created, g.Revised)

	if err != nil {
		err = errors.Wrap(err, "insert group")
	}

	return
}

// Get returns requested group.
func (s Scope) Get(ctx domain.RequestContext, refID string) (g group.Group, err error) {
	err = s.Runtime.Db.Get(&g,
		`SELECT id, refid, orgid, role as name, purpose, created, revised FROM role WHERE orgid=? AND refid=?`,
		ctx.OrgID, refID)

	if err != nil {
		err = errors.Wrap(err, "select group")
	}

	return
}

// GetAll returns all user groups for current orgID.
func (s Scope) GetAll(ctx domain.RequestContext) (groups []group.Group, err error) {
	err = s.Runtime.Db.Select(&groups,
		`SELECT a.id, a.refid, a.orgid, a.role as name, a.purpose, a.created, a.revised, COUNT(b.roleid) AS members
		FROM role a
		LEFT JOIN rolemember b ON a.refid=b.roleid
		WHERE a.orgid=?
		GROUP BY a.id, a.refid, a.orgid, a.role, a.purpose, a.created, a.revised
		ORDER BY a.role`,
		ctx.OrgID)

	if err == sql.ErrNoRows || len(groups) == 0 {
		err = nil
		groups = []group.Group{}
	}
	if err != nil {
		err = errors.Wrap(err, "select groups")
	}

	return
}

// Update group name and description.
func (s Scope) Update(ctx domain.RequestContext, g group.Group) (err error) {
	g.Revised = time.Now().UTC()

	_, err = ctx.Transaction.Exec("UPDATE role SET role=?, purpose=?, revised=? WHERE orgid=? AND refid=?",
		g.Name, g.Purpose, g.Revised, ctx.OrgID, g.RefID)

	if err != nil {
		err = errors.Wrap(err, "update group")
	}

	return
}

// Delete removes group from store.
func (s Scope) Delete(ctx domain.RequestContext, refID string) (rows int64, err error) {
	b := mysql.BaseQuery{}
	b.DeleteConstrained(ctx.Transaction, "role", ctx.OrgID, refID)
	return b.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM rolemember WHERE orgid=\"%s\" AND roleid=\"%s\"", ctx.OrgID, refID))
}

// GetGroupMembers returns all user associated with given group.
func (s Scope) GetGroupMembers(ctx domain.RequestContext, groupID string) (members []group.Member, err error) {
	err = s.Runtime.Db.Select(&members,
		`SELECT a.id, a.orgid, a.roleid, a.userid,
		IFNULL(b.firstname, '') as firstname, IFNULL(b.lastname, '') as lastname
		FROM rolemember a
		LEFT JOIN user b ON b.refid=a.userid
		WHERE a.orgid=? AND a.roleid=?
		ORDER BY b.firstname, b.lastname`,
		ctx.OrgID, groupID)

	if err == sql.ErrNoRows || len(members) == 0 {
		err = nil
		members = []group.Member{}
	}
	if err != nil {
		err = errors.Wrap(err, "select members")
	}

	return
}

// JoinGroup adds user to group.
func (s Scope) JoinGroup(ctx domain.RequestContext, groupID, userID string) (err error) {
	_, err = ctx.Transaction.Exec("INSERT INTO rolemember (orgid, roleid, userid) VALUES (?, ?, ?)", ctx.OrgID, groupID, userID)
	if err != nil {
		err = errors.Wrap(err, "insert group member")
	}

	return
}

// LeaveGroup removes user from group.
func (s Scope) LeaveGroup(ctx domain.RequestContext, groupID, userID string) (err error) {
	b := mysql.BaseQuery{}
	_, err = b.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM rolemember WHERE orgid=\"%s\" AND roleid=\"%s\" AND userid=\"%s\"", ctx.OrgID, groupID, userID))
	if err != nil {
		err = errors.Wrap(err, "clear group member")
	}

	return
}

// GetMembers returns members for every group.
// Useful when you need to bulk fetch membership records
// for subsequent processing.
func (s Scope) GetMembers(ctx domain.RequestContext) (r []group.Record, err error) {
	err = s.Runtime.Db.Select(&r,
		`SELECT a.id, a.orgid, a.roleid, a.userid, b.role as name, b.purpose
		FROM rolemember a, role b
		WHERE a.orgid=? AND a.roleid=b.refid
		ORDER BY a.userid`,
		ctx.OrgID)

	if err == sql.ErrNoRows || len(r) == 0 {
		err = nil
		r = []group.Record{}
	}
	if err != nil {
		err = errors.Wrap(err, "select group members")
	}

	return
}
<commit_msg>Fix silent error drop<commit_after>// Copyright 2018 Documize Inc. <legal@documize.com>. All rights reserved.
//
// This software (Documize Community Edition) is licensed under
// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
//
// You can operate outside the AGPL restrictions by purchasing
// Documize Enterprise Edition and obtaining a commercial license
// by contacting <sales@documize.com>.
//
// https://documize.com

package mysql

import (
	"database/sql"
	"fmt"
	"time"

	"github.com/documize/community/core/env"
	"github.com/documize/community/domain"
	"github.com/documize/community/domain/store/mysql"
	"github.com/documize/community/model/group"
	"github.com/pkg/errors"
)

// Scope provides data access to MySQL.
type Scope struct {
	Runtime *env.Runtime
}

// Add inserts new user group into store.
func (s Scope) Add(ctx domain.RequestContext, g group.Group) (err error) {
	g.Created = time.Now().UTC()
	g.Revised = time.Now().UTC()

	_, err = ctx.Transaction.Exec("INSERT INTO role (refid, orgid, role, purpose, created, revised) VALUES (?, ?, ?, ?, ?, ?)",
		g.RefID, g.OrgID, g.Name, g.Purpose, g.Created, g.Revised)

	if err != nil {
		err = errors.Wrap(err, "insert group")
	}

	return
}

// Get returns requested group.
func (s Scope) Get(ctx domain.RequestContext, refID string) (g group.Group, err error) {
	err = s.Runtime.Db.Get(&g,
		`SELECT id, refid, orgid, role as name, purpose, created, revised FROM role WHERE orgid=? AND refid=?`,
		ctx.OrgID, refID)

	if err != nil {
		err = errors.Wrap(err, "select group")
	}

	return
}

// GetAll returns all user groups for current orgID.
func (s Scope) GetAll(ctx domain.RequestContext) (groups []group.Group, err error) {
	groups = []group.Group{}

	err = s.Runtime.Db.Select(&groups,
		`SELECT a.id, a.refid, a.orgid, a.role as name, a.purpose, a.created, a.revised, COUNT(b.roleid) AS members
		FROM role a
		LEFT JOIN rolemember b ON a.refid=b.roleid
		WHERE a.orgid=?
		GROUP BY a.id, a.refid, a.orgid, a.role, a.purpose, a.created, a.revised
		ORDER BY a.role`,
		ctx.OrgID)

	if err == sql.ErrNoRows {
		err = nil
	}
	if err != nil {
		err = errors.Wrap(err, "select groups")
	}

	return
}

// Update group name and description.
func (s Scope) Update(ctx domain.RequestContext, g group.Group) (err error) {
	g.Revised = time.Now().UTC()

	_, err = ctx.Transaction.Exec("UPDATE role SET role=?, purpose=?, revised=? WHERE orgid=? AND refid=?",
		g.Name, g.Purpose, g.Revised, ctx.OrgID, g.RefID)

	if err != nil {
		err = errors.Wrap(err, "update group")
	}

	return
}

// Delete removes group from store.
func (s Scope) Delete(ctx domain.RequestContext, refID string) (rows int64, err error) {
	b := mysql.BaseQuery{}
	b.DeleteConstrained(ctx.Transaction, "role", ctx.OrgID, refID)
	return b.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM rolemember WHERE orgid=\"%s\" AND roleid=\"%s\"", ctx.OrgID, refID))
}

// GetGroupMembers returns all user associated with given group.
func (s Scope) GetGroupMembers(ctx domain.RequestContext, groupID string) (members []group.Member, err error) {
	members = []group.Member{}

	err = s.Runtime.Db.Select(&members,
		`SELECT a.id, a.orgid, a.roleid, a.userid,
		IFNULL(b.firstname, '') as firstname, IFNULL(b.lastname, '') as lastname
		FROM rolemember a
		LEFT JOIN user b ON b.refid=a.userid
		WHERE a.orgid=? AND a.roleid=?
		ORDER BY b.firstname, b.lastname`,
		ctx.OrgID, groupID)

	if err == sql.ErrNoRows {
		err = nil
	}
	if err != nil {
		err = errors.Wrap(err, "select members")
	}

	return
}

// JoinGroup adds user to group.
func (s Scope) JoinGroup(ctx domain.RequestContext, groupID, userID string) (err error) {
	_, err = ctx.Transaction.Exec("INSERT INTO rolemember (orgid, roleid, userid) VALUES (?, ?, ?)", ctx.OrgID, groupID, userID)
	if err != nil {
		err = errors.Wrap(err, "insert group member")
	}

	return
}

// LeaveGroup removes user from group.
func (s Scope) LeaveGroup(ctx domain.RequestContext, groupID, userID string) (err error) {
	b := mysql.BaseQuery{}
	_, err = b.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM rolemember WHERE orgid=\"%s\" AND roleid=\"%s\" AND userid=\"%s\"", ctx.OrgID, groupID, userID))
	if err != nil {
		err = errors.Wrap(err, "clear group member")
	}

	return
}

// GetMembers returns members for every group.
// Useful when you need to bulk fetch membership records
// for subsequent processing.
func (s Scope) GetMembers(ctx domain.RequestContext) (r []group.Record, err error) {
	r = []group.Record{}

	err = s.Runtime.Db.Select(&r,
		`SELECT a.id, a.orgid, a.roleid, a.userid, b.role as name, b.purpose
		FROM rolemember a, role b
		WHERE a.orgid=? AND a.roleid=b.refid
		ORDER BY a.userid`,
		ctx.OrgID)

	if err == sql.ErrNoRows {
		err = nil
	}
	if err != nil {
		err = errors.Wrap(err, "select group members")
	}

	return
}
<|endoftext|>


<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build java android

package syncbase

import (
	"os"
	"path/filepath"
	"unsafe"

	"v.io/v23"
	"v.io/x/ref/services/syncbase/server"

	jrpc "v.io/x/jni/impl/google/rpc"
	jutil "v.io/x/jni/util"
	jcontext "v.io/x/jni/v23/context"
	jaccess "v.io/x/jni/v23/security/access"
)

// #include "jni.h"
import "C"

var (
	permissionsSign   = jutil.ClassSign("io.v.v23.security.access.Permissions")
	contextSign       = jutil.ClassSign("io.v.v23.context.VContext")
	storageEngineSign = jutil.ClassSign("io.v.impl.google.services.syncbase.SyncbaseServer$StorageEngine")
	serverSign        = jutil.ClassSign("io.v.v23.rpc.Server")

	jVRuntimeImplClass jutil.Class
)

// Init initializes the JNI code with the given Java environment.  This method
// must be invoked before any other method in this package and must be called
// from the main Java thread (e.g., On_Load()).
func Init(env jutil.Env) error {
	var err error
	jVRuntimeImplClass, err = jutil.JFindClass(env, "io/v/impl/google/rt/VRuntimeImpl")
	if err != nil {
		return err
	}
	return nil
}

//export Java_io_v_impl_google_services_syncbase_SyncbaseServer_nativeWithNewServer
func Java_io_v_impl_google_services_syncbase_SyncbaseServer_nativeWithNewServer(jenv *C.JNIEnv, jSyncbaseServerClass C.jclass, jContext C.jobject, jSyncbaseServerParams C.jobject) C.jobject {
	env := jutil.Env(uintptr(unsafe.Pointer(jenv)))
	jCtx := jutil.Object(uintptr(unsafe.Pointer(jContext)))
	jParams := jutil.Object(uintptr(unsafe.Pointer(jSyncbaseServerParams)))

	// Read and translate all of the server params.
	jPerms, err := jutil.CallObjectMethod(env, jParams, "getPermissions", nil, permissionsSign)
	if err != nil {
		jutil.JThrowV(env, err)
		return nil
	}
	perms, err := jaccess.GoPermissions(env, jPerms)
	if err != nil {
		jutil.JThrowV(env, err)
		return nil
	}
	name, err := jutil.CallStringMethod(env, jParams, "getName", nil)
	if err != nil {
		jutil.JThrowV(env, err)
		return nil
	}
	rootDir, err := jutil.CallStringMethod(env, jParams, "getStorageRootDir", nil)
	if err != nil {
		jutil.JThrowV(env, err)
		return nil
	}
	if rootDir == "" {
		rootDir = filepath.Join(os.TempDir(), "syncbaseserver")
		if err := os.Mkdir(rootDir, 0755); err != nil && !os.IsExist(err) {
			jutil.JThrowV(env, err)
			return nil
		}
	}
	jEngine, err := jutil.CallObjectMethod(env, jParams, "getStorageEngine", nil, storageEngineSign)
	if err != nil {
		jutil.JThrowV(env, err)
		return nil
	}
	engine, err := GoStorageEngine(env, jEngine)
	if err != nil {
		jutil.JThrowV(env, err)
		return nil
	}
	ctx, err := jcontext.GoContext(env, jCtx)
	if err != nil {
		jutil.JThrowV(env, err)
		return nil
	}

	// Start the server.
	service, err := server.NewService(ctx, nil, server.ServiceOptions{
		Perms:   perms,
		RootDir: rootDir,
		Engine:  engine,
	})
	if err != nil {
		jutil.JThrowV(env, err)
		return nil
	}
	d := server.NewDispatcher(service)
	newCtx, s, err := v23.WithNewDispatchingServer(ctx, name, d)
	if err != nil {
		jutil.JThrowV(env, err)
		return nil
	}
	if err := service.AddNames(ctx, s); err != nil {
		jutil.JThrowV(env, err)
		return nil
	}
	jNewCtx, err := jcontext.JavaContext(env, newCtx)
	if err != nil {
		jutil.JThrowV(env, err)
		return nil
	}
	jServer, err := jrpc.JavaServer(env, s)
	if err != nil {
		jutil.JThrowV(env, err)
		return nil
	}
	// Attach a server to the new context.
	jServerAttCtx, err := jutil.CallStaticObjectMethod(env, jVRuntimeImplClass, "withServer", []jutil.Sign{contextSign, serverSign}, contextSign, jNewCtx, jServer)
	if err != nil {
		jutil.JThrowV(env, err)
		return nil
	}
	return C.jobject(unsafe.Pointer(jServerAttCtx))
}
<commit_msg>jni/impl/google/services/syncbase: remove 'call' arg from NewService<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build java android

package syncbase

import (
	"os"
	"path/filepath"
	"unsafe"

	"v.io/v23"
	"v.io/x/ref/services/syncbase/server"

	jrpc "v.io/x/jni/impl/google/rpc"
	jutil "v.io/x/jni/util"
	jcontext "v.io/x/jni/v23/context"
	jaccess "v.io/x/jni/v23/security/access"
)

// #include "jni.h"
import "C"

var (
	permissionsSign   = jutil.ClassSign("io.v.v23.security.access.Permissions")
	contextSign       = jutil.ClassSign("io.v.v23.context.VContext")
	storageEngineSign = jutil.ClassSign("io.v.impl.google.services.syncbase.SyncbaseServer$StorageEngine")
	serverSign        = jutil.ClassSign("io.v.v23.rpc.Server")

	jVRuntimeImplClass jutil.Class
)

// Init initializes the JNI code with the given Java environment.  This method
// must be invoked before any other method in this package and must be called
// from the main Java thread (e.g., On_Load()).
func Init(env jutil.Env) error {
	var err error
	jVRuntimeImplClass, err = jutil.JFindClass(env, "io/v/impl/google/rt/VRuntimeImpl")
	if err != nil {
		return err
	}
	return nil
}

//export Java_io_v_impl_google_services_syncbase_SyncbaseServer_nativeWithNewServer
func Java_io_v_impl_google_services_syncbase_SyncbaseServer_nativeWithNewServer(jenv *C.JNIEnv, jSyncbaseServerClass C.jclass, jContext C.jobject, jSyncbaseServerParams C.jobject) C.jobject {
	env := jutil.Env(uintptr(unsafe.Pointer(jenv)))
	jCtx := jutil.Object(uintptr(unsafe.Pointer(jContext)))
	jParams := jutil.Object(uintptr(unsafe.Pointer(jSyncbaseServerParams)))

	// Read and translate all of the server params.
	jPerms, err := jutil.CallObjectMethod(env, jParams, "getPermissions", nil, permissionsSign)
	if err != nil {
		jutil.JThrowV(env, err)
		return nil
	}
	perms, err := jaccess.GoPermissions(env, jPerms)
	if err != nil {
		jutil.JThrowV(env, err)
		return nil
	}
	name, err := jutil.CallStringMethod(env, jParams, "getName", nil)
	if err != nil {
		jutil.JThrowV(env, err)
		return nil
	}
	rootDir, err := jutil.CallStringMethod(env, jParams, "getStorageRootDir", nil)
	if err != nil {
		jutil.JThrowV(env, err)
		return nil
	}
	if rootDir == "" {
		rootDir = filepath.Join(os.TempDir(), "syncbaseserver")
		if err := os.Mkdir(rootDir, 0755); err != nil && !os.IsExist(err) {
			jutil.JThrowV(env, err)
			return nil
		}
	}
	jEngine, err := jutil.CallObjectMethod(env, jParams, "getStorageEngine", nil, storageEngineSign)
	if err != nil {
		jutil.JThrowV(env, err)
		return nil
	}
	engine, err := GoStorageEngine(env, jEngine)
	if err != nil {
		jutil.JThrowV(env, err)
		return nil
	}
	ctx, err := jcontext.GoContext(env, jCtx)
	if err != nil {
		jutil.JThrowV(env, err)
		return nil
	}

	// Start the server.
	service, err := server.NewService(ctx, server.ServiceOptions{
		Perms:   perms,
		RootDir: rootDir,
		Engine:  engine,
	})
	if err != nil {
		jutil.JThrowV(env, err)
		return nil
	}
	d := server.NewDispatcher(service)
	newCtx, s, err := v23.WithNewDispatchingServer(ctx, name, d)
	if err != nil {
		jutil.JThrowV(env, err)
		return nil
	}
	if err := service.AddNames(ctx, s); err != nil {
		jutil.JThrowV(env, err)
		return nil
	}
	jNewCtx, err := jcontext.JavaContext(env, newCtx)
	if err != nil {
		jutil.JThrowV(env, err)
		return nil
	}
	jServer, err := jrpc.JavaServer(env, s)
	if err != nil {
		jutil.JThrowV(env, err)
		return nil
	}
	// Attach a server to the new context.
	jServerAttCtx, err := jutil.CallStaticObjectMethod(env, jVRuntimeImplClass, "withServer", []jutil.Sign{contextSign, serverSign}, contextSign, jNewCtx, jServer)
	if err != nil {
		jutil.JThrowV(env, err)
		return nil
	}
	return C.jobject(unsafe.Pointer(jServerAttCtx))
}
<|endoftext|>


<commit_before>package arn

import (
	"fmt"

	"github.com/animenotifier/anilist"
)

// AniListAnimeFinder holds an internal map of ID to anime mappings
// and is therefore very efficient to use when trying to find
// anime by a given service and ID.
type AniListAnimeFinder struct {
	idToAnime    map[string]*Anime
	malIDToAnime map[string]*Anime
}

// NewAniListAnimeFinder creates a new finder for Anilist anime.
func NewAniListAnimeFinder() *AniListAnimeFinder {
	finder := &AniListAnimeFinder{
		idToAnime:    map[string]*Anime{},
		malIDToAnime: map[string]*Anime{},
	}

	for anime := range StreamAnime() {
		id := anime.GetMapping("anilist/anime")

		if id != "" {
			finder.idToAnime[id] = anime
		}

		malID := anime.GetMapping("myanimelist/anime")

		if malID != "" {
			finder.malIDToAnime[malID] = anime
		}
	}

	return finder
}

// GetAnime tries to find an AniList anime in our anime database.
func (finder *AniListAnimeFinder) GetAnime(id string, malID string) *Anime {
	animeByID, existsByID := finder.idToAnime[id]
	animeByMALID, existsByMALID := finder.malIDToAnime[malID]

	// If both MAL ID and AniList ID are matched, but the matched anime are different,
	// then we're trusting the MAL ID matching more and deleting the incorrect mapping.
	if existsByID && existsByMALID && animeByID.ID != animeByMALID.ID {
		animeByID.RemoveMapping("anilist/anime")
		animeByID.Save()

		delete(finder.idToAnime, id)

		fmt.Println("MAL / Anilist mismatch:")
		fmt.Println(animeByID.ID, animeByID)
		fmt.Println(animeByMALID.ID, animeByMALID)
	}

	// Add anilist mapping to the MAL mapped anime if it's missing
	if existsByMALID && animeByMALID.GetMapping("anilist/anime") != id {
		animeByMALID.SetMapping("anilist/anime", id)
		animeByMALID.Save()

		finder.idToAnime[id] = animeByMALID
	}

	if existsByID {
		return animeByID
	}

	if existsByMALID {
		return animeByMALID
	}

	return nil
}

// AniListAnimeListStatus returns the ARN version of the anime status.
func AniListAnimeListStatus(item *anilist.AnimeListItem) string {
	switch item.Status {
	case "CURRENT", "REPEATING":
		return AnimeListStatusWatching
	case "COMPLETED":
		return AnimeListStatusCompleted
	case "PLANNING":
		return AnimeListStatusPlanned
	case "PAUSED":
		return AnimeListStatusHold
	case "DROPPED":
		return AnimeListStatusDropped
	default:
		return AnimeListStatusPlanned
	}
}
<commit_msg>Improved anilist importer<commit_after>package arn

import (
	"fmt"

	"github.com/animenotifier/anilist"
)

// AniListAnimeFinder holds an internal map of ID to anime mappings
// and is therefore very efficient to use when trying to find
// anime by a given service and ID.
type AniListAnimeFinder struct {
	idToAnime    map[string]*Anime
	malIDToAnime map[string]*Anime
}

// NewAniListAnimeFinder creates a new finder for Anilist anime.
func NewAniListAnimeFinder() *AniListAnimeFinder {
	finder := &AniListAnimeFinder{
		idToAnime:    map[string]*Anime{},
		malIDToAnime: map[string]*Anime{},
	}

	for anime := range StreamAnime() {
		id := anime.GetMapping("anilist/anime")

		if id != "" {
			finder.idToAnime[id] = anime
		}

		malID := anime.GetMapping("myanimelist/anime")

		if malID != "" {
			finder.malIDToAnime[malID] = anime
		}
	}

	return finder
}

// GetAnime tries to find an AniList anime in our anime database.
func (finder *AniListAnimeFinder) GetAnime(id string, malID string) *Anime {
	animeByID, existsByID := finder.idToAnime[id]
	animeByMALID, existsByMALID := finder.malIDToAnime[malID]

	// Add anilist mapping to the MAL mapped anime if it's missing
	if existsByMALID && animeByMALID.GetMapping("anilist/anime") != id {
		animeByMALID.SetMapping("anilist/anime", id)
		animeByMALID.Save()

		finder.idToAnime[id] = animeByMALID
	}

	// If both MAL ID and AniList ID are matched, but the matched anime are different,
	// while the MAL IDs are different as well,
	// then we're trusting the MAL ID matching more and deleting the incorrect mapping.
	if existsByID && existsByMALID && animeByID.ID != animeByMALID.ID && animeByID.GetMapping("myanimelist/anime") != animeByMALID.GetMapping("myanimelist/anime") {
		animeByID.RemoveMapping("anilist/anime")
		animeByID.Save()

		delete(finder.idToAnime, id)

		fmt.Println("MAL / Anilist mismatch:")
		fmt.Println(animeByID.ID, animeByID)
		fmt.Println(animeByMALID.ID, animeByMALID)
	}

	if existsByID {
		return animeByID
	}

	if existsByMALID {
		return animeByMALID
	}

	return nil
}

// AniListAnimeListStatus returns the ARN version of the anime status.
func AniListAnimeListStatus(item *anilist.AnimeListItem) string {
	switch item.Status {
	case "CURRENT", "REPEATING":
		return AnimeListStatusWatching
	case "COMPLETED":
		return AnimeListStatusCompleted
	case "PLANNING":
		return AnimeListStatusPlanned
	case "PAUSED":
		return AnimeListStatusHold
	case "DROPPED":
		return AnimeListStatusDropped
	default:
		return AnimeListStatusPlanned
	}
}
<|endoftext|>


<commit_before>package decoders

import (
	"errors"
	"fmt"
	"log"
	"math"
	"regexp"
	"strconv"

	"github.com/seadsystem/Backend/DB/landingzone/constants"
)

type SeadPacket struct {
	Type      byte
	Location  byte
	Timestamp float64
	Period    float64
	Count     int
	Data      float64
	Serial    int
}

var headerRegex *regexp.Regexp
var InvalidHeader = errors.New("Invalid header.")
var InvalidPacket = errors.New("Invalid packet.")
var InvalidTime = errors.New("Invalid time.")

// init sets up stuff we need with proper error handling. If it isn't complicated or doesn't need error handling, it can probably just be assigned directly.
func init() {
	var err error
	headerRegex, err = regexp.Compile(constants.HEADER_REGEX)
	if err != nil {
		log.Panic("Regex compile error:", err)
	}
}

// DecodeHeader verifies that the header is in the correct format and extracts the serial number
func DecodeHeader(packet []byte) (serial int, err error) {
	serialStrings := headerRegex.FindSubmatch(packet)

	if serialStrings == nil || len(serialStrings) != 2 {
		err = InvalidHeader
		return
	}

	log.Printf("Header serial string: %s\n", string(serialStrings[1]))

	serial, err = strconv.Atoi(string(serialStrings[1]))
	return
}

// DecodePacket extracts the data sent from sensor
func DecodePacket(buffer []byte) (packet SeadPacket, err error) {
	for i := 0; i < len(buffer); {
		datatype := buffer[i]
		i++

		// Switch on the type of data sent in the packet
		switch {
		case datatype == 'T':
			// Type
			packet.Type = buffer[i]
			i++
		case datatype == 'l':
			// Location
			packet.Location = buffer[i]
			i++
		case datatype == 't':
			// Timestamp
			packet.Timestamp, err = asciiTimeToDouble(buffer[i : i+14])
			i += 14
		case datatype == 'P':
			// Period separator
			packet.Period, err = asciiTimeToDouble(buffer[i : i+14])
			i += 14
		case datatype == 'C':
			// Count
			packet.Count, err = Binary2int(buffer[i : i+2])
			i += 2
		case datatype == 'D':
			// Data
			// if count isn't set, return error
			// TODO finish parsing data
		case datatype == 'S':
			// Serial
			packet.Serial, err = strconv.Atoi(string(buffer[i : i+6]))
			i += 6
		case datatype == 'X':
			return
		default:
			err = InvalidPacket
		}

		if err != nil {
			return
		}
	}
	err = InvalidPacket
	return
}

func doubleToAsciiTime(double_time float64) string {
	// TODO: Check if this logic is correct or if we need to use http://golang.org/pkg/math/#Mod
	int_time := int(double_time)
	var days = math.Floor(double_time / (60 * 60 * 24))
	var hours = (int_time % (60 * 60 * 24)) / (60 * 60)
	var minutes = (int_time % (60 * 60)) / 60
	var seconds = (int_time % (60)) / 1
	var milliseconds = (int_time * 1000) % 1000
	var clock_time = (int_time * 12000) % 12

	return fmt.Sprintf("%03d%02d%02d%02d%03d%02d", days, hours, minutes, seconds, milliseconds, clock_time)
}

func asciiTimeToDouble(ascii_time []byte) (time float64, err error) {
	// Check time string format
	if len(ascii_time) != 16 {
		err = InvalidTime
	}
	_, err = strconv.Atoi(string(ascii_time))
	if err != nil {
		return
	}

	// Do the conversion now that we know it should work
	var ptr int = 0
	days, err := strconv.Atoi(string(ascii_time[ptr : ptr+3]))
	if err != nil {
		return
	}
	ptr += 3
	time += float64(60 * 60 * 24 * days)
	hours, err := strconv.Atoi(string(ascii_time[ptr : ptr+2]))
	if err != nil {
		return
	}
	ptr += 2
	time += float64(60 * 60 * hours)
	minutes, err := strconv.Atoi(string(ascii_time[ptr : ptr+2]))
	if err != nil {
		return
	}
	ptr += 2
	time += float64(60 * minutes)
	seconds, err := strconv.Atoi(string(ascii_time[ptr : ptr+2]))
	if err != nil {
		return
	}
	ptr += 2
	time += float64(seconds)
	milliseconds, err := strconv.Atoi(string(ascii_time[ptr : ptr+3]))
	if err != nil {
		return
	}
	ptr += 3
	time += float64(milliseconds) / 1000.0
	clock, err := strconv.Atoi(string(ascii_time[ptr : ptr+2]))
	if err != nil {
		return
	}
	ptr += 2
	time += float64(clock) / 12000.0
	return
}

// Every checks if every byte in a slice meets some criteria
func Every(data []byte, check func(byte) bool) bool {
	for _, element := range data {
		if !check(element) {
			return false
		}
	}
	return true
}

// Binary2int converts a byte array containing binary data into an int
func Binary2int(data []byte) (total int) {
	for index, element := range data {
		total += int(element)<<(index * 8)
	}
	return true
}
<commit_msg>Removed left over return value from new function.<commit_after>package decoders

import (
	"errors"
	"fmt"
	"log"
	"math"
	"regexp"
	"strconv"

	"github.com/seadsystem/Backend/DB/landingzone/constants"
)

type SeadPacket struct {
	Type      byte
	Location  byte
	Timestamp float64
	Period    float64
	Count     int
	Data      float64
	Serial    int
}

var headerRegex *regexp.Regexp
var InvalidHeader = errors.New("Invalid header.")
var InvalidPacket = errors.New("Invalid packet.")
var InvalidTime = errors.New("Invalid time.")

// init sets up stuff we need with proper error handling. If it isn't complicated or doesn't need error handling, it can probably just be assigned directly.
func init() {
	var err error
	headerRegex, err = regexp.Compile(constants.HEADER_REGEX)
	if err != nil {
		log.Panic("Regex compile error:", err)
	}
}

// DecodeHeader verifies that the header is in the correct format and extracts the serial number
func DecodeHeader(packet []byte) (serial int, err error) {
	serialStrings := headerRegex.FindSubmatch(packet)

	if serialStrings == nil || len(serialStrings) != 2 {
		err = InvalidHeader
		return
	}

	log.Printf("Header serial string: %s\n", string(serialStrings[1]))

	serial, err = strconv.Atoi(string(serialStrings[1]))
	return
}

// DecodePacket extracts the data sent from sensor
func DecodePacket(buffer []byte) (packet SeadPacket, err error) {
	for i := 0; i < len(buffer); {
		datatype := buffer[i]
		i++

		// Switch on the type of data sent in the packet
		switch {
		case datatype == 'T':
			// Type
			packet.Type = buffer[i]
			i++
		case datatype == 'l':
			// Location
			packet.Location = buffer[i]
			i++
		case datatype == 't':
			// Timestamp
			packet.Timestamp, err = asciiTimeToDouble(buffer[i : i+14])
			i += 14
		case datatype == 'P':
			// Period separator
			packet.Period, err = asciiTimeToDouble(buffer[i : i+14])
			i += 14
		case datatype == 'C':
			// Count
			packet.Count, err = Binary2int(buffer[i : i+2])
			i += 2
		case datatype == 'D':
			// Data
			// if count isn't set, return error
			// TODO finish parsing data
		case datatype == 'S':
			// Serial
			packet.Serial, err = strconv.Atoi(string(buffer[i : i+6]))
			i += 6
		case datatype == 'X':
			return
		default:
			err = InvalidPacket
		}

		if err != nil {
			return
		}
	}
	err = InvalidPacket
	return
}

func doubleToAsciiTime(double_time float64) string {
	// TODO: Check if this logic is correct or if we need to use http://golang.org/pkg/math/#Mod
	int_time := int(double_time)
	var days = math.Floor(double_time / (60 * 60 * 24))
	var hours = (int_time % (60 * 60 * 24)) / (60 * 60)
	var minutes = (int_time % (60 * 60)) / 60
	var seconds = (int_time % (60)) / 1
	var milliseconds = (int_time * 1000) % 1000
	var clock_time = (int_time * 12000) % 12

	return fmt.Sprintf("%03d%02d%02d%02d%03d%02d", days, hours, minutes, seconds, milliseconds, clock_time)
}

func asciiTimeToDouble(ascii_time []byte) (time float64, err error) {
	// Check time string format
	if len(ascii_time) != 16 {
		err = InvalidTime
	}
	_, err = strconv.Atoi(string(ascii_time))
	if err != nil {
		return
	}

	// Do the conversion now that we know it should work
	var ptr int = 0
	days, err := strconv.Atoi(string(ascii_time[ptr : ptr+3]))
	if err != nil {
		return
	}
	ptr += 3
	time += float64(60 * 60 * 24 * days)
	hours, err := strconv.Atoi(string(ascii_time[ptr : ptr+2]))
	if err != nil {
		return
	}
	ptr += 2
	time += float64(60 * 60 * hours)
	minutes, err := strconv.Atoi(string(ascii_time[ptr : ptr+2]))
	if err != nil {
		return
	}
	ptr += 2
	time += float64(60 * minutes)
	seconds, err := strconv.Atoi(string(ascii_time[ptr : ptr+2]))
	if err != nil {
		return
	}
	ptr += 2
	time += float64(seconds)
	milliseconds, err := strconv.Atoi(string(ascii_time[ptr : ptr+3]))
	if err != nil {
		return
	}
	ptr += 3
	time += float64(milliseconds) / 1000.0
	clock, err := strconv.Atoi(string(ascii_time[ptr : ptr+2]))
	if err != nil {
		return
	}
	ptr += 2
	time += float64(clock) / 12000.0
	return
}

// Every checks if every byte in a slice meets some criteria
func Every(data []byte, check func(byte) bool) bool {
	for _, element := range data {
		if !check(element) {
			return false
		}
	}
	return true
}

// Binary2int converts a byte array containing binary data into an int
func Binary2int(data []byte) (total int) {
	for index, element := range data {
		total += int(element)<<(index * 8)
	}
	return
}
<|endoftext|>


<commit_before>package emulator

import (
    "github.com/kierdavis/avr"
    "testing"
)

type decodeTest struct {
    word uint16
    inst avr.Instruction
}

var decodeTests = []decodeTest{
    decodeTest{0x1ef9, avr.ADC},
    decodeTest{0x1c23, avr.ADC},
    decodeTest{0x0efc, avr.ADD},
    decodeTest{0x0f6d, avr.ADD},
    decodeTest{0x96c7, avr.ADIW},
    decodeTest{0x96b4, avr.ADIW},
    decodeTest{0x2013, avr.AND},
    decodeTest{0x23ed, avr.AND},
    decodeTest{0x7dfc, avr.ANDI},
    decodeTest{0x70f6, avr.ANDI},
    decodeTest{0x9415, avr.ASR},
    decodeTest{0x95b5, avr.ASR},
    decodeTest{0x9498, avr.BCLR},
    decodeTest{0x94a8, avr.BCLR},
    decodeTest{0xf802, avr.BLD},
    decodeTest{0xf8f6, avr.BLD},
    decodeTest{0xf456, avr.BRBC},
    decodeTest{0xf5e3, avr.BRBC},
    decodeTest{0xf254, avr.BRBS},
    decodeTest{0xf179, avr.BRBS},
    decodeTest{0x9598, avr.BREAK},
    decodeTest{0x9418, avr.BSET},
    decodeTest{0x9468, avr.BSET},
    decodeTest{0xfb14, avr.BST},
    decodeTest{0xfbe7, avr.BST},
    decodeTest{0x94de, avr.CALL},
    decodeTest{0x95ef, avr.CALL},
    decodeTest{0x981c, avr.CBI},
    decodeTest{0x98b4, avr.CBI},
    decodeTest{0x9400, avr.COM},
    decodeTest{0x9530, avr.COM},
    decodeTest{0x170a, avr.CP},
    decodeTest{0x146f, avr.CP},
    decodeTest{0x060d, avr.CPC},
    decodeTest{0x0796, avr.CPC},
    decodeTest{0x3e57, avr.CPI},
    decodeTest{0x3ead, avr.CPI},
    decodeTest{0x1310, avr.CPSE},
    decodeTest{0x121e, avr.CPSE},
    decodeTest{0x944a, avr.DEC},
    decodeTest{0x94fa, avr.DEC},
    decodeTest{0x941b, avr.DES},
    decodeTest{0x94ab, avr.DES},
    decodeTest{0x9519, avr.EICALL},
    decodeTest{0x9419, avr.EIJMP},
    decodeTest{0x95D8, avr.ELPM_R0},
    decodeTest{0x9076, avr.ELPM},
    decodeTest{0x91a6, avr.ELPM},
    decodeTest{0x91f7, avr.ELPM_INC},
    decodeTest{0x9147, avr.ELPM_INC},
    decodeTest{0x242b, avr.EOR},
    decodeTest{0x24c1, avr.EOR},
    decodeTest{0x035d, avr.FMUL},
    decodeTest{0x0339, avr.FMUL},
    decodeTest{0x03f2, avr.FMULS},
    decodeTest{0x0380, avr.FMULS},
    decodeTest{0x03ce, avr.FMULSU},
    decodeTest{0x03ea, avr.FMULSU},
    decodeTest{0x9509, avr.ICALL},
    decodeTest{0x9409, avr.IJMP},
    decodeTest{0xb41d, avr.IN},
    decodeTest{0xb5c9, avr.IN},
    decodeTest{0x95f3, avr.INC},
    decodeTest{0x95d3, avr.INC},
    decodeTest{0x951d, avr.JMP},
    decodeTest{0x94ed, avr.JMP},
    decodeTest{0x9226, avr.LAC},
    decodeTest{0x93c6, avr.LAC},
    decodeTest{0x92d5, avr.LAS},
    decodeTest{0x9325, avr.LAS},
    decodeTest{0x9227, avr.LAT},
    decodeTest{0x93e7, avr.LAT},
    decodeTest{0x90cc, avr.LD_X},
    decodeTest{0x908c, avr.LD_X},
    decodeTest{0x904d, avr.LD_X_INC},
    decodeTest{0x91ad, avr.LD_X_INC},
    decodeTest{0x905e, avr.LD_X_DEC},
    decodeTest{0x90fe, avr.LD_X_DEC},
    decodeTest{0x80f8, avr.LDD_Y},
    decodeTest{0x8128, avr.LDD_Y},
    decodeTest{0x9119, avr.LD_Y_INC},
    decodeTest{0x9089, avr.LD_Y_INC},
    decodeTest{0x91da, avr.LD_Y_DEC},
    decodeTest{0x916a, avr.LD_Y_DEC},
    decodeTest{0xa938, avr.LDD_Y},
    decodeTest{0x80d9, avr.LDD_Y},
    decodeTest{0x8060, avr.LDD_Z},
    decodeTest{0x8130, avr.LDD_Z},
    decodeTest{0x9011, avr.LD_Z_INC},
    decodeTest{0x9031, avr.LD_Z_INC},
    decodeTest{0x90a2, avr.LD_Z_DEC},
    decodeTest{0x9102, avr.LD_Z_DEC},
    decodeTest{0xa9a7, avr.LDD_Z},
    decodeTest{0x8964, avr.LDD_Z},
    decodeTest{0xeb93, avr.LDI},
    decodeTest{0xeadd, avr.LDI},
}

func TestDecode(t *testing.T) {
    for _, test := range decodeTests {
        inst := Decode(test.word)
        if inst != test.inst {
            t.Errorf("Decode(0x%04x): expected '%s', got '%s'", test.word, test.inst, inst)
        }
    }
}

func BenchmarkDecode(b *testing.B) {
    // implement an xorshift RNG for speed
    x := uint16(0xabcd)
    for i := 0; i < b.N; i++ {
        x ^= x << 13
        x ^= x >> 9
        x ^= x << 7
        Decode(x)
    }
}
<commit_msg>Split Decode benchmark into random and non-random versions<commit_after>package emulator

import (
    "github.com/kierdavis/avr"
    "testing"
)

type decodeTest struct {
    word uint16
    inst avr.Instruction
}

var decodeTests = []decodeTest{
    decodeTest{0x1ef9, avr.ADC},
    decodeTest{0x1c23, avr.ADC},
    decodeTest{0x0efc, avr.ADD},
    decodeTest{0x0f6d, avr.ADD},
    decodeTest{0x96c7, avr.ADIW},
    decodeTest{0x96b4, avr.ADIW},
    decodeTest{0x2013, avr.AND},
    decodeTest{0x23ed, avr.AND},
    decodeTest{0x7dfc, avr.ANDI},
    decodeTest{0x70f6, avr.ANDI},
    decodeTest{0x9415, avr.ASR},
    decodeTest{0x95b5, avr.ASR},
    decodeTest{0x9498, avr.BCLR},
    decodeTest{0x94a8, avr.BCLR},
    decodeTest{0xf802, avr.BLD},
    decodeTest{0xf8f6, avr.BLD},
    decodeTest{0xf456, avr.BRBC},
    decodeTest{0xf5e3, avr.BRBC},
    decodeTest{0xf254, avr.BRBS},
    decodeTest{0xf179, avr.BRBS},
    decodeTest{0x9598, avr.BREAK},
    decodeTest{0x9418, avr.BSET},
    decodeTest{0x9468, avr.BSET},
    decodeTest{0xfb14, avr.BST},
    decodeTest{0xfbe7, avr.BST},
    decodeTest{0x94de, avr.CALL},
    decodeTest{0x95ef, avr.CALL},
    decodeTest{0x981c, avr.CBI},
    decodeTest{0x98b4, avr.CBI},
    decodeTest{0x9400, avr.COM},
    decodeTest{0x9530, avr.COM},
    decodeTest{0x170a, avr.CP},
    decodeTest{0x146f, avr.CP},
    decodeTest{0x060d, avr.CPC},
    decodeTest{0x0796, avr.CPC},
    decodeTest{0x3e57, avr.CPI},
    decodeTest{0x3ead, avr.CPI},
    decodeTest{0x1310, avr.CPSE},
    decodeTest{0x121e, avr.CPSE},
    decodeTest{0x944a, avr.DEC},
    decodeTest{0x94fa, avr.DEC},
    decodeTest{0x941b, avr.DES},
    decodeTest{0x94ab, avr.DES},
    decodeTest{0x9519, avr.EICALL},
    decodeTest{0x9419, avr.EIJMP},
    decodeTest{0x95D8, avr.ELPM_R0},
    decodeTest{0x9076, avr.ELPM},
    decodeTest{0x91a6, avr.ELPM},
    decodeTest{0x91f7, avr.ELPM_INC},
    decodeTest{0x9147, avr.ELPM_INC},
    decodeTest{0x242b, avr.EOR},
    decodeTest{0x24c1, avr.EOR},
    decodeTest{0x035d, avr.FMUL},
    decodeTest{0x0339, avr.FMUL},
    decodeTest{0x03f2, avr.FMULS},
    decodeTest{0x0380, avr.FMULS},
    decodeTest{0x03ce, avr.FMULSU},
    decodeTest{0x03ea, avr.FMULSU},
    decodeTest{0x9509, avr.ICALL},
    decodeTest{0x9409, avr.IJMP},
    decodeTest{0xb41d, avr.IN},
    decodeTest{0xb5c9, avr.IN},
    decodeTest{0x95f3, avr.INC},
    decodeTest{0x95d3, avr.INC},
    decodeTest{0x951d, avr.JMP},
    decodeTest{0x94ed, avr.JMP},
    decodeTest{0x9226, avr.LAC},
    decodeTest{0x93c6, avr.LAC},
    decodeTest{0x92d5, avr.LAS},
    decodeTest{0x9325, avr.LAS},
    decodeTest{0x9227, avr.LAT},
    decodeTest{0x93e7, avr.LAT},
    decodeTest{0x90cc, avr.LD_X},
    decodeTest{0x908c, avr.LD_X},
    decodeTest{0x904d, avr.LD_X_INC},
    decodeTest{0x91ad, avr.LD_X_INC},
    decodeTest{0x905e, avr.LD_X_DEC},
    decodeTest{0x90fe, avr.LD_X_DEC},
    decodeTest{0x80f8, avr.LDD_Y},
    decodeTest{0x8128, avr.LDD_Y},
    decodeTest{0x9119, avr.LD_Y_INC},
    decodeTest{0x9089, avr.LD_Y_INC},
    decodeTest{0x91da, avr.LD_Y_DEC},
    decodeTest{0x916a, avr.LD_Y_DEC},
    decodeTest{0xa938, avr.LDD_Y},
    decodeTest{0x80d9, avr.LDD_Y},
    decodeTest{0x8060, avr.LDD_Z},
    decodeTest{0x8130, avr.LDD_Z},
    decodeTest{0x9011, avr.LD_Z_INC},
    decodeTest{0x9031, avr.LD_Z_INC},
    decodeTest{0x90a2, avr.LD_Z_DEC},
    decodeTest{0x9102, avr.LD_Z_DEC},
    decodeTest{0xa9a7, avr.LDD_Z},
    decodeTest{0x8964, avr.LDD_Z},
    decodeTest{0xeb93, avr.LDI},
    decodeTest{0xeadd, avr.LDI},
}

func TestDecode(t *testing.T) {
    for _, test := range decodeTests {
        inst := Decode(test.word)
        if inst != test.inst {
            t.Errorf("Decode(0x%04x): expected '%s', got '%s'", test.word, test.inst, inst)
        }
    }
}

func BenchmarkDecode951d(b *testing.B) {
    for i := 0; i < b.N; i++ {
        Decode(0x951d)
    }
}

func BenchmarkDecodeRandom(b *testing.B) {
    // implement an xorshift RNG for speed
    x := uint16(0xabcd)
    for i := 0; i < b.N; i++ {
        x ^= x << 13
        x ^= x >> 9
        x ^= x << 7
        Decode(x)
    }
}
<|endoftext|>


<commit_before>package warserver

import (
    "encoding/binary"
    "github.com/gorilla/websocket"
    "net"
)

type connection interface {
    Read() ([]byte, error)
    Write(msg []byte) error
    Close()
}

type websocketConn struct {
    ws *websocket.Conn
}

func (c *websocketConn) Read() ([]byte, error) {
    _, msg, err := c.ws.ReadMessage()
    return msg, err
}

func (c *websocketConn) Write(msg []byte) error {
    return c.ws.WriteMessage(websocket.TextMessage, msg)
}

func (c *websocketConn) Close() {
    c.ws.Close()
}

type socketConn struct {
    sock net.Conn
}

func (c *socketConn) Read() ([]byte, error) {
    // the 32 bit initial message dictates the size of the message
    sizeBuf := make([]byte, 4)
    _, err := c.sock.Read(sizeBuf)
    if err != nil {
        return nil, err
    }
    size, _ := binary.Varint(sizeBuf)
    msgBuf := make([]byte, size)
    _, err = c.sock.Read(msgBuf)
    return msgBuf, err
}

func (c *socketConn) Write(msg []byte) error {
    n, err := c.sock.Write(msg)
    for num_sent := len(msg) - n; num_sent > 0; n, err = c.sock.Write(msg) {
        num_sent -= n
        if err != nil {
            return err
        }
    }
    return err
}

func (c *socketConn) Close() {
    c.sock.Close()
}
<commit_msg>Untested Write functions, yay!<commit_after>package warserver

import (
    "encoding/binary"
    "github.com/gorilla/websocket"
    "net"
)

type connection interface {
    Read() ([]byte, error)
    Write(msg []byte) error
    Close()
}

type websocketConn struct {
    ws *websocket.Conn
}

func (c *websocketConn) Read() ([]byte, error) {
    _, msg, err := c.ws.ReadMessage()
    return msg, err
}

func (c *websocketConn) Write(msg []byte) error {
    return c.ws.WriteMessage(websocket.TextMessage, msg)
}

func (c *websocketConn) Close() {
    c.ws.Close()
}

type socketConn struct {
    sock net.Conn
}

func (c *socketConn) Read() ([]byte, error) {
    // the 32 bit initial message dictates the size of the message
    sizeBuf := make([]byte, 4)
    _, err := c.sock.Read(sizeBuf)
    if err != nil {
        return nil, err
    }
    size, _ := binary.Varint(sizeBuf)
    msgBuf := make([]byte, size)
    _, err = c.sock.Read(msgBuf)
    return msgBuf, err
}

func (c *socketConn) Write(msg []byte) error {
    lenBuf := make([]byte, 4)
    msgLen := len(msg)
    binary.PutUvarint(lenBuf, uint64(msgLen))
    err := c.fullWrite(lenBuf)
    if err != nil {
        return err
    }
    return c.fullWrite(msg)
}

func (c *socketConn) fullWrite(msg []byte) error {
    msgLen := len(msg)
    n, err := c.sock.Write(msg)
    for num_sent := 0; num_sent < msgLen; n, err = c.sock.Write(msg) {
        num_sent += n
        msg = msg[n:]
        if err != nil {
            return err
        }
    }
    return err
}

func (c *socketConn) Close() {
    c.sock.Close()
}
<|endoftext|>


<commit_before>// Copyright 2016 The nvim-go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package autocmd

import (
	"time"

	"github.com/zchee/nvim-go/src/config"
	"github.com/zchee/nvim-go/src/nvimutil"
)

// BufReadPre gets user config variables and assign to global variable when autocmd BufReadPre.
func (a *Autocmd) BufReadPre(cfg *config.Config) {
	defer nvimutil.Profile(a.ctx, time.Now(), "BufReadPre")
}
<commit_msg>autocmd/bufreadpre: change args to empty struct pointer<commit_after>// Copyright 2016 The nvim-go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package autocmd

import (
	"time"

	"github.com/zchee/nvim-go/src/nvimutil"
)

type bufReadPreEval struct {}

// BufReadPre gets user config variables and assign to global variable when autocmd BufReadPre.
func (a *Autocmd) BufReadPre(eval *bufReadPreEval) {
	defer nvimutil.Profile(a.ctx, time.Now(), "BufReadPre")
}
<|endoftext|>