<commit_before>package io

import (
	"bytes"
	"io"
	"io/ioutil"
	"math/rand"
	"strings"
	"testing"

	mdag "github.com/ipfs/go-ipfs/merkledag"
	"github.com/ipfs/go-ipfs/unixfs"

	context "context"

	testu "github.com/ipfs/go-ipfs/unixfs/test"
)

func TestBasicRead(t *testing.T) {
	dserv := testu.GetDAGServ()
	inbuf, node := testu.GetRandomNode(t, dserv, 1024, testu.UseProtoBufLeaves)
	ctx, closer := context.WithCancel(context.Background())
	defer closer()

	reader, err := NewDagReader(ctx, node, dserv)
	if err != nil {
		t.Fatal(err)
	}

	outbuf, err := ioutil.ReadAll(reader)
	if err != nil {
		t.Fatal(err)
	}

	err = testu.ArrComp(inbuf, outbuf)
	if err != nil {
		t.Fatal(err)
	}
}

func TestSeekAndRead(t *testing.T) {
	dserv := testu.GetDAGServ()
	inbuf := make([]byte, 256)
	for i := 0; i <= 255; i++ {
		inbuf[i] = byte(i)
	}

	node := testu.GetNode(t, dserv, inbuf, testu.UseProtoBufLeaves)
	ctx, closer := context.WithCancel(context.Background())
	defer closer()

	reader, err := NewDagReader(ctx, node, dserv)
	if err != nil {
		t.Fatal(err)
	}

	for i := 255; i >= 0; i-- {
		reader.Seek(int64(i), io.SeekStart)

		if reader.Offset() != int64(i) {
			t.Fatal("expected offset to be increased by one after read")
		}

		out := readByte(t, reader)

		if int(out) != i {
			t.Fatalf("read %d at index %d, expected %d", out, i, i)
		}

		if reader.Offset() != int64(i+1) {
			t.Fatal("expected offset to be increased by one after read")
		}
	}
}

func TestSeekAndReadLarge(t *testing.T) {
	dserv := testu.GetDAGServ()
	inbuf := make([]byte, 20000)
	rand.Read(inbuf)

	node := testu.GetNode(t, dserv, inbuf, testu.UseProtoBufLeaves)
	ctx, closer := context.WithCancel(context.Background())
	defer closer()

	reader, err := NewDagReader(ctx, node, dserv)
	if err != nil {
		t.Fatal(err)
	}

	_, err = reader.Seek(10000, io.SeekStart)
	if err != nil {
		t.Fatal(err)
	}

	buf := make([]byte, 100)
	_, err = io.ReadFull(reader, buf)
	if err != nil {
		t.Fatal(err)
	}

	if !bytes.Equal(buf, inbuf[10000:10100]) {
		t.Fatal("seeked read failed")
	}

	pbdr := reader.(*PBDagReader)
	var count int
	for i, p := range pbdr.promises {
		if i > 20 && i < 30 {
			if p == nil {
				t.Fatal("expected index to be not nil: ", i)
			}
			count++
		} else {
			if p != nil {
				t.Fatal("expected index to be nil: ", i)
			}
		}
	}
	// -1 because we read some and it cleared one
	if count != preloadSize-1 {
		t.Fatalf("expected %d preloaded promises, got %d", preloadSize-1, count)
	}
}

func TestRelativeSeek(t *testing.T) {
	dserv := testu.GetDAGServ()
	ctx, closer := context.WithCancel(context.Background())
	defer closer()

	inbuf := make([]byte, 1024)

	for i := 0; i < 256; i++ {
		inbuf[i*4] = byte(i)
	}

	inbuf[1023] = 1 // force the reader to be 1024 bytes
	node := testu.GetNode(t, dserv, inbuf, testu.UseProtoBufLeaves)

	reader, err := NewDagReader(ctx, node, dserv)
	if err != nil {
		t.Fatal(err)
	}

	for i := 0; i < 256; i++ {
		if reader.Offset() != int64(i*4) {
			t.Fatalf("offset should be %d, was %d", i*4, reader.Offset())
		}
		out := readByte(t, reader)
		if int(out) != i {
			t.Fatalf("expected to read: %d at %d, read %d", i, reader.Offset()-1, out)
		}
		if i != 255 {
			_, err := reader.Seek(3, io.SeekCurrent)
			if err != nil {
				t.Fatal(err)
			}
		}
	}

	_, err = reader.Seek(4, io.SeekEnd)
	if err != nil {
		t.Fatal(err)
	}

	for i := 0; i < 256; i++ {
		if reader.Offset() != int64(1020-i*4) {
			t.Fatalf("offset should be %d, was %d", 1020-i*4, reader.Offset())
		}
		out := readByte(t, reader)
		if int(out) != 255-i {
			t.Fatalf("expected to read: %d at %d, read %d", 255-i, reader.Offset()-1, out)
		}
		reader.Seek(-5, io.SeekCurrent) // seek 4 bytes but we read one byte every time so 5 bytes
	}

}

func TestTypeFailures(t *testing.T) {
	dserv := testu.GetDAGServ()
	ctx, closer := context.WithCancel(context.Background())
	defer closer()

	node := unixfs.EmptyDirNode()
	if _, err := NewDagReader(ctx, node, dserv); err != ErrIsDir {
		t.Fatalf("excepted to get %v, got %v", ErrIsDir, err)
	}

	data, err := unixfs.SymlinkData("/somelink")
	if err != nil {
		t.Fatal(err)
	}
	node = mdag.NodeWithData(data)

	if _, err := NewDagReader(ctx, node, dserv); err != ErrCantReadSymlinks {
		t.Fatalf("excepted to get %v, got %v", ErrCantReadSymlinks, err)
	}
}

func TestBadPBData(t *testing.T) {
	dserv := testu.GetDAGServ()
	ctx, closer := context.WithCancel(context.Background())
	defer closer()

	node := mdag.NodeWithData([]byte{42})
	_, err := NewDagReader(ctx, node, dserv)
	if err == nil {
		t.Fatal("excepted error, got nil")
	}
}

func TestMetadataNode(t *testing.T) {
	ctx, closer := context.WithCancel(context.Background())
	defer closer()

	dserv := testu.GetDAGServ()
	rdata, rnode := testu.GetRandomNode(t, dserv, 512, testu.UseProtoBufLeaves)
	err := dserv.Add(ctx, rnode)
	if err != nil {
		t.Fatal(err)
	}

	data, err := unixfs.BytesForMetadata(&unixfs.Metadata{
		MimeType: "text",
		Size:     125,
	})
	if err != nil {
		t.Fatal(err)
	}
	node := mdag.NodeWithData(data)

	_, err = NewDagReader(ctx, node, dserv)
	if err == nil {
		t.Fatal("expected an error")
	}
	if !strings.Contains(err.Error(), "incorrectly formatted") {
		t.Fatal("expected different error")
	}

	node.AddNodeLink("", rnode)

	reader, err := NewDagReader(ctx, node, dserv)
	if err != nil {
		t.Fatal(err)
	}
	readdata, err := ioutil.ReadAll(reader)
	if err != nil {
		t.Fatal(err)
	}
	if err := testu.ArrComp(rdata, readdata); err != nil {
		t.Fatal(err)
	}
}

func TestWriteTo(t *testing.T) {
	dserv := testu.GetDAGServ()
	inbuf, node := testu.GetRandomNode(t, dserv, 1024, testu.UseProtoBufLeaves)
	ctx, closer := context.WithCancel(context.Background())
	defer closer()

	reader, err := NewDagReader(ctx, node, dserv)
	if err != nil {
		t.Fatal(err)
	}

	outbuf := new(bytes.Buffer)
	reader.WriteTo(outbuf)

	err = testu.ArrComp(inbuf, outbuf.Bytes())
	if err != nil {
		t.Fatal(err)
	}

}

func TestReaderSzie(t *testing.T) {
	dserv := testu.GetDAGServ()
	size := int64(1024)
	_, node := testu.GetRandomNode(t, dserv, size, testu.UseProtoBufLeaves)
	ctx, closer := context.WithCancel(context.Background())
	defer closer()

	reader, err := NewDagReader(ctx, node, dserv)
	if err != nil {
		t.Fatal(err)
	}

	if reader.Size() != uint64(size) {
		t.Fatal("wrong reader size")
	}
}

func readByte(t testing.TB, reader DagReader) byte {
	out := make([]byte, 1)
	c, err := reader.Read(out)

	if c != 1 {
		t.Fatal("reader should have read just one byte")
	}
	if err != nil {
		t.Fatal(err)
	}

	return out[0]
}
<commit_msg>test dag reader context cancellation<commit_after>package io

import (
	"bytes"
	"io"
	"io/ioutil"
	"math/rand"
	"strings"
	"testing"

	mdag "github.com/ipfs/go-ipfs/merkledag"
	"github.com/ipfs/go-ipfs/unixfs"

	context "context"

	testu "github.com/ipfs/go-ipfs/unixfs/test"
)

func TestBasicRead(t *testing.T) {
	dserv := testu.GetDAGServ()
	inbuf, node := testu.GetRandomNode(t, dserv, 1024, testu.UseProtoBufLeaves)
	ctx, closer := context.WithCancel(context.Background())
	defer closer()

	reader, err := NewDagReader(ctx, node, dserv)
	if err != nil {
		t.Fatal(err)
	}

	outbuf, err := ioutil.ReadAll(reader)
	if err != nil {
		t.Fatal(err)
	}

	err = testu.ArrComp(inbuf, outbuf)
	if err != nil {
		t.Fatal(err)
	}
}

func TestSeekAndRead(t *testing.T) {
	dserv := testu.GetDAGServ()
	inbuf := make([]byte, 256)
	for i := 0; i <= 255; i++ {
		inbuf[i] = byte(i)
	}

	node := testu.GetNode(t, dserv, inbuf, testu.UseProtoBufLeaves)
	ctx, closer := context.WithCancel(context.Background())
	defer closer()

	reader, err := NewDagReader(ctx, node, dserv)
	if err != nil {
		t.Fatal(err)
	}

	for i := 255; i >= 0; i-- {
		reader.Seek(int64(i), io.SeekStart)

		if reader.Offset() != int64(i) {
			t.Fatal("expected offset to be increased by one after read")
		}

		out := readByte(t, reader)

		if int(out) != i {
			t.Fatalf("read %d at index %d, expected %d", out, i, i)
		}

		if reader.Offset() != int64(i+1) {
			t.Fatal("expected offset to be increased by one after read")
		}
	}
}

func TestSeekAndReadLarge(t *testing.T) {
	dserv := testu.GetDAGServ()
	inbuf := make([]byte, 20000)
	rand.Read(inbuf)

	node := testu.GetNode(t, dserv, inbuf, testu.UseProtoBufLeaves)
	ctx, closer := context.WithCancel(context.Background())
	defer closer()

	reader, err := NewDagReader(ctx, node, dserv)
	if err != nil {
		t.Fatal(err)
	}

	_, err = reader.Seek(10000, io.SeekStart)
	if err != nil {
		t.Fatal(err)
	}

	buf := make([]byte, 100)
	_, err = io.ReadFull(reader, buf)
	if err != nil {
		t.Fatal(err)
	}

	if !bytes.Equal(buf, inbuf[10000:10100]) {
		t.Fatal("seeked read failed")
	}

	pbdr := reader.(*PBDagReader)
	var count int
	for i, p := range pbdr.promises {
		if i > 20 && i < 30 {
			if p == nil {
				t.Fatal("expected index to be not nil: ", i)
			}
			count++
		} else {
			if p != nil {
				t.Fatal("expected index to be nil: ", i)
			}
		}
	}
	// -1 because we read some and it cleared one
	if count != preloadSize-1 {
		t.Fatalf("expected %d preloaded promises, got %d", preloadSize-1, count)
	}
}

func TestReadAndCancel(t *testing.T) {
	dserv := testu.GetDAGServ()
	inbuf := make([]byte, 20000)
	rand.Read(inbuf)

	node := testu.GetNode(t, dserv, inbuf, testu.UseProtoBufLeaves)
	ctx, closer := context.WithCancel(context.Background())
	defer closer()

	reader, err := NewDagReader(ctx, node, dserv)
	if err != nil {
		t.Fatal(err)
	}

	ctx, cancel := context.WithCancel(context.Background())
	buf := make([]byte, 100)
	_, err = reader.CtxReadFull(ctx, buf)
	if err != nil {
		t.Fatal(err)
	}
	if !bytes.Equal(buf, inbuf[0:100]) {
		t.Fatal("read failed")
	}
	cancel()

	b, err := ioutil.ReadAll(reader)
	if err != nil {
		t.Fatal(err)
	}

	if !bytes.Equal(inbuf[100:], b) {
		t.Fatal("buffers not equal")
	}
}

func TestRelativeSeek(t *testing.T) {
	dserv := testu.GetDAGServ()
	ctx, closer := context.WithCancel(context.Background())
	defer closer()

	inbuf := make([]byte, 1024)

	for i := 0; i < 256; i++ {
		inbuf[i*4] = byte(i)
	}

	inbuf[1023] = 1 // force the reader to be 1024 bytes
	node := testu.GetNode(t, dserv, inbuf, testu.UseProtoBufLeaves)

	reader, err := NewDagReader(ctx, node, dserv)
	if err != nil {
		t.Fatal(err)
	}

	for i := 0; i < 256; i++ {
		if reader.Offset() != int64(i*4) {
			t.Fatalf("offset should be %d, was %d", i*4, reader.Offset())
		}
		out := readByte(t, reader)
		if int(out) != i {
			t.Fatalf("expected to read: %d at %d, read %d", i, reader.Offset()-1, out)
		}
		if i != 255 {
			_, err := reader.Seek(3, io.SeekCurrent)
			if err != nil {
				t.Fatal(err)
			}
		}
	}

	_, err = reader.Seek(4, io.SeekEnd)
	if err != nil {
		t.Fatal(err)
	}

	for i := 0; i < 256; i++ {
		if reader.Offset() != int64(1020-i*4) {
			t.Fatalf("offset should be %d, was %d", 1020-i*4, reader.Offset())
		}
		out := readByte(t, reader)
		if int(out) != 255-i {
			t.Fatalf("expected to read: %d at %d, read %d", 255-i, reader.Offset()-1, out)
		}
		reader.Seek(-5, io.SeekCurrent) // seek 4 bytes but we read one byte every time so 5 bytes
	}

}

func TestTypeFailures(t *testing.T) {
	dserv := testu.GetDAGServ()
	ctx, closer := context.WithCancel(context.Background())
	defer closer()

	node := unixfs.EmptyDirNode()
	if _, err := NewDagReader(ctx, node, dserv); err != ErrIsDir {
		t.Fatalf("excepted to get %v, got %v", ErrIsDir, err)
	}

	data, err := unixfs.SymlinkData("/somelink")
	if err != nil {
		t.Fatal(err)
	}
	node = mdag.NodeWithData(data)

	if _, err := NewDagReader(ctx, node, dserv); err != ErrCantReadSymlinks {
		t.Fatalf("excepted to get %v, got %v", ErrCantReadSymlinks, err)
	}
}

func TestBadPBData(t *testing.T) {
	dserv := testu.GetDAGServ()
	ctx, closer := context.WithCancel(context.Background())
	defer closer()

	node := mdag.NodeWithData([]byte{42})
	_, err := NewDagReader(ctx, node, dserv)
	if err == nil {
		t.Fatal("excepted error, got nil")
	}
}

func TestMetadataNode(t *testing.T) {
	ctx, closer := context.WithCancel(context.Background())
	defer closer()

	dserv := testu.GetDAGServ()
	rdata, rnode := testu.GetRandomNode(t, dserv, 512, testu.UseProtoBufLeaves)
	err := dserv.Add(ctx, rnode)
	if err != nil {
		t.Fatal(err)
	}

	data, err := unixfs.BytesForMetadata(&unixfs.Metadata{
		MimeType: "text",
		Size:     125,
	})
	if err != nil {
		t.Fatal(err)
	}
	node := mdag.NodeWithData(data)

	_, err = NewDagReader(ctx, node, dserv)
	if err == nil {
		t.Fatal("expected an error")
	}
	if !strings.Contains(err.Error(), "incorrectly formatted") {
		t.Fatal("expected different error")
	}

	node.AddNodeLink("", rnode)

	reader, err := NewDagReader(ctx, node, dserv)
	if err != nil {
		t.Fatal(err)
	}
	readdata, err := ioutil.ReadAll(reader)
	if err != nil {
		t.Fatal(err)
	}
	if err := testu.ArrComp(rdata, readdata); err != nil {
		t.Fatal(err)
	}
}

func TestWriteTo(t *testing.T) {
	dserv := testu.GetDAGServ()
	inbuf, node := testu.GetRandomNode(t, dserv, 1024, testu.UseProtoBufLeaves)
	ctx, closer := context.WithCancel(context.Background())
	defer closer()

	reader, err := NewDagReader(ctx, node, dserv)
	if err != nil {
		t.Fatal(err)
	}

	outbuf := new(bytes.Buffer)
	reader.WriteTo(outbuf)

	err = testu.ArrComp(inbuf, outbuf.Bytes())
	if err != nil {
		t.Fatal(err)
	}

}

func TestReaderSzie(t *testing.T) {
	dserv := testu.GetDAGServ()
	size := int64(1024)
	_, node := testu.GetRandomNode(t, dserv, size, testu.UseProtoBufLeaves)
	ctx, closer := context.WithCancel(context.Background())
	defer closer()

	reader, err := NewDagReader(ctx, node, dserv)
	if err != nil {
		t.Fatal(err)
	}

	if reader.Size() != uint64(size) {
		t.Fatal("wrong reader size")
	}
}

func readByte(t testing.TB, reader DagReader) byte {
	out := make([]byte, 1)
	c, err := reader.Read(out)

	if c != 1 {
		t.Fatal("reader should have read just one byte")
	}
	if err != nil {
		t.Fatal(err)
	}

	return out[0]
}
<|endoftext|>


<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"os"
	"sync"
	"time"

	"github.com/coreos/coreos-cloudinit/config"
	"github.com/coreos/coreos-cloudinit/config/validate"
	"github.com/coreos/coreos-cloudinit/datasource"
	"github.com/coreos/coreos-cloudinit/datasource/configdrive"
	"github.com/coreos/coreos-cloudinit/datasource/file"
	"github.com/coreos/coreos-cloudinit/datasource/metadata/cloudsigma"
	"github.com/coreos/coreos-cloudinit/datasource/metadata/digitalocean"
	"github.com/coreos/coreos-cloudinit/datasource/metadata/ec2"
	"github.com/coreos/coreos-cloudinit/datasource/proc_cmdline"
	"github.com/coreos/coreos-cloudinit/datasource/url"
	"github.com/coreos/coreos-cloudinit/datasource/waagent"
	"github.com/coreos/coreos-cloudinit/initialize"
	"github.com/coreos/coreos-cloudinit/network"
	"github.com/coreos/coreos-cloudinit/pkg"
	"github.com/coreos/coreos-cloudinit/system"
)

const (
	version               = "1.3.3+git"
	datasourceInterval    = 100 * time.Millisecond
	datasourceMaxInterval = 30 * time.Second
	datasourceTimeout     = 5 * time.Minute
)

var (
	flags = struct {
		printVersion  bool
		ignoreFailure bool
		sources       struct {
			file                        string
			configDrive                 string
			waagent                     string
			metadataService             bool
			ec2MetadataService          string
			cloudSigmaMetadataService   bool
			digitalOceanMetadataService string
			url                         string
			procCmdLine                 bool
		}
		convertNetconf string
		workspace      string
		sshKeyName     string
		oem            string
		validate       bool
	}{}
)

func init() {
	flag.BoolVar(&flags.printVersion, "version", false, "Print the version and exit")
	flag.BoolVar(&flags.ignoreFailure, "ignore-failure", false, "Exits with 0 status in the event of malformed input from user-data")
	flag.StringVar(&flags.sources.file, "from-file", "", "Read user-data from provided file")
	flag.StringVar(&flags.sources.configDrive, "from-configdrive", "", "Read data from provided cloud-drive directory")
	flag.StringVar(&flags.sources.waagent, "from-waagent", "", "Read data from provided waagent directory")
	flag.BoolVar(&flags.sources.metadataService, "from-metadata-service", false, "[DEPRECATED - Use -from-ec2-metadata] Download data from metadata service")
	flag.StringVar(&flags.sources.ec2MetadataService, "from-ec2-metadata", "", "Download EC2 data from the provided url")
	flag.BoolVar(&flags.sources.cloudSigmaMetadataService, "from-cloudsigma-metadata", false, "Download data from CloudSigma server context")
	flag.StringVar(&flags.sources.digitalOceanMetadataService, "from-digitalocean-metadata", "", "Download DigitalOcean data from the provided url")
	flag.StringVar(&flags.sources.url, "from-url", "", "Download user-data from provided url")
	flag.BoolVar(&flags.sources.procCmdLine, "from-proc-cmdline", false, fmt.Sprintf("Parse %s for '%s=<url>', using the cloud-config served by an HTTP GET to <url>", proc_cmdline.ProcCmdlineLocation, proc_cmdline.ProcCmdlineCloudConfigFlag))
	flag.StringVar(&flags.oem, "oem", "", "Use the settings specific to the provided OEM")
	flag.StringVar(&flags.convertNetconf, "convert-netconf", "", "Read the network config provided in cloud-drive and translate it from the specified format into networkd unit files")
	flag.StringVar(&flags.workspace, "workspace", "/var/lib/coreos-cloudinit", "Base directory coreos-cloudinit should use to store data")
	flag.StringVar(&flags.sshKeyName, "ssh-key-name", initialize.DefaultSSHKeyName, "Add SSH keys to the system with the given name")
	flag.BoolVar(&flags.validate, "validate", false, "[EXPERIMENTAL] Validate the user-data but do not apply it to the system")
}

type oemConfig map[string]string

var (
	oemConfigs = map[string]oemConfig{
		"digitalocean": oemConfig{
			"from-digitalocean-metadata": "http://169.254.169.254/",
			"convert-netconf":            "digitalocean",
		},
		"ec2-compat": oemConfig{
			"from-ec2-metadata": "http://169.254.169.254/",
			"from-configdrive":  "/media/configdrive",
		},
		"rackspace-onmetal": oemConfig{
			"from-configdrive": "/media/configdrive",
			"convert-netconf":  "debian",
		},
		"azure": oemConfig{
			"from-waagent": "/var/lib/waagent",
		},
		"cloudsigma": oemConfig{
			"from-cloudsigma-metadata": "true",
		},
	}
)

func main() {
	failure := false

	flag.Parse()

	if c, ok := oemConfigs[flags.oem]; ok {
		for k, v := range c {
			flag.Set(k, v)
		}
	} else if flags.oem != "" {
		oems := make([]string, 0, len(oemConfigs))
		for k := range oemConfigs {
			oems = append(oems, k)
		}
		fmt.Printf("Invalid option to --oem: %q. Supported options: %q\n", flags.oem, oems)
		os.Exit(2)
	}

	if flags.printVersion == true {
		fmt.Printf("coreos-cloudinit version %s\n", version)
		os.Exit(0)
	}

	switch flags.convertNetconf {
	case "":
	case "debian":
	case "digitalocean":
	default:
		fmt.Printf("Invalid option to -convert-netconf: '%s'. Supported options: 'debian, digitalocean'\n", flags.convertNetconf)
		os.Exit(2)
	}

	dss := getDatasources()
	if len(dss) == 0 {
		fmt.Println("Provide at least one of --from-file, --from-configdrive, --from-ec2-metadata, --from-cloudsigma-metadata, --from-url or --from-proc-cmdline")
		os.Exit(2)
	}

	ds := selectDatasource(dss)
	if ds == nil {
		fmt.Println("No datasources available in time")
		os.Exit(1)
	}

	fmt.Printf("Fetching user-data from datasource of type %q\n", ds.Type())
	userdataBytes, err := ds.FetchUserdata()
	if err != nil {
		fmt.Printf("Failed fetching user-data from datasource: %v\nContinuing...\n", err)
		failure = true
	}

	if report, err := validate.Validate(userdataBytes); err == nil {
		ret := 0
		for _, e := range report.Entries() {
			fmt.Println(e)
			ret = 1
		}
		if flags.validate {
			os.Exit(ret)
		}
	} else {
		fmt.Printf("Failed while validating user_data (%q)\n", err)
		if flags.validate {
			os.Exit(1)
		}
	}

	fmt.Printf("Fetching meta-data from datasource of type %q\n", ds.Type())
	metadata, err := ds.FetchMetadata()
	if err != nil {
		fmt.Printf("Failed fetching meta-data from datasource: %v\n", err)
		os.Exit(1)
	}

	// Apply environment to user-data
	env := initialize.NewEnvironment("/", ds.ConfigRoot(), flags.workspace, flags.sshKeyName, metadata)
	userdata := env.Apply(string(userdataBytes))

	var ccu *config.CloudConfig
	var script *config.Script
	if ud, err := initialize.ParseUserData(userdata); err != nil {
		fmt.Printf("Failed to parse user-data: %v\nContinuing...\n", err)
		failure = true
	} else {
		switch t := ud.(type) {
		case *config.CloudConfig:
			ccu = t
		case *config.Script:
			script = t
		}
	}

	fmt.Println("Merging cloud-config from meta-data and user-data")
	cc := mergeConfigs(ccu, metadata)

	var ifaces []network.InterfaceGenerator
	if flags.convertNetconf != "" {
		var err error
		switch flags.convertNetconf {
		case "debian":
			ifaces, err = network.ProcessDebianNetconf(metadata.NetworkConfig)
		case "digitalocean":
			ifaces, err = network.ProcessDigitalOceanNetconf(metadata.NetworkConfig)
		default:
			err = fmt.Errorf("Unsupported network config format %q", flags.convertNetconf)
		}
		if err != nil {
			fmt.Printf("Failed to generate interfaces: %v\n", err)
			os.Exit(1)
		}
	}

	if err = initialize.Apply(cc, ifaces, env); err != nil {
		fmt.Printf("Failed to apply cloud-config: %v\n", err)
		os.Exit(1)
	}

	if script != nil {
		if err = runScript(*script, env); err != nil {
			fmt.Printf("Failed to run script: %v\n", err)
			os.Exit(1)
		}
	}

	if failure && !flags.ignoreFailure {
		os.Exit(1)
	}
}

// mergeConfigs merges certain options from md (meta-data from the datasource)
// onto cc (a CloudConfig derived from user-data), if they are not already set
// on cc (i.e. user-data always takes precedence)
func mergeConfigs(cc *config.CloudConfig, md datasource.Metadata) (out config.CloudConfig) {
	if cc != nil {
		out = *cc
	}

	if md.Hostname != "" {
		if out.Hostname != "" {
			fmt.Printf("Warning: user-data hostname (%s) overrides metadata hostname (%s)\n", out.Hostname, md.Hostname)
		} else {
			out.Hostname = md.Hostname
		}
	}
	for _, key := range md.SSHPublicKeys {
		out.SSHAuthorizedKeys = append(out.SSHAuthorizedKeys, key)
	}
	return
}

// getDatasources creates a slice of possible Datasources for cloudinit based
// on the different source command-line flags.
func getDatasources() []datasource.Datasource {
	dss := make([]datasource.Datasource, 0, 5)
	if flags.sources.file != "" {
		dss = append(dss, file.NewDatasource(flags.sources.file))
	}
	if flags.sources.url != "" {
		dss = append(dss, url.NewDatasource(flags.sources.url))
	}
	if flags.sources.configDrive != "" {
		dss = append(dss, configdrive.NewDatasource(flags.sources.configDrive))
	}
	if flags.sources.metadataService {
		dss = append(dss, ec2.NewDatasource(ec2.DefaultAddress))
	}
	if flags.sources.ec2MetadataService != "" {
		dss = append(dss, ec2.NewDatasource(flags.sources.ec2MetadataService))
	}
	if flags.sources.cloudSigmaMetadataService {
		dss = append(dss, cloudsigma.NewServerContextService())
	}
	if flags.sources.digitalOceanMetadataService != "" {
		dss = append(dss, digitalocean.NewDatasource(flags.sources.digitalOceanMetadataService))
	}
	if flags.sources.waagent != "" {
		dss = append(dss, waagent.NewDatasource(flags.sources.waagent))
	}
	if flags.sources.procCmdLine {
		dss = append(dss, proc_cmdline.NewDatasource())
	}
	return dss
}

// selectDatasource attempts to choose a valid Datasource to use based on its
// current availability. The first Datasource to report to be available is
// returned. Datasources will be retried if possible if they are not
// immediately available. If all Datasources are permanently unavailable or
// datasourceTimeout is reached before one becomes available, nil is returned.
func selectDatasource(sources []datasource.Datasource) datasource.Datasource {
	ds := make(chan datasource.Datasource)
	stop := make(chan struct{})
	var wg sync.WaitGroup

	for _, s := range sources {
		wg.Add(1)
		go func(s datasource.Datasource) {
			defer wg.Done()

			duration := datasourceInterval
			for {
				fmt.Printf("Checking availability of %q\n", s.Type())
				if s.IsAvailable() {
					ds <- s
					return
				} else if !s.AvailabilityChanges() {
					return
				}
				select {
				case <-stop:
					return
				case <-time.After(duration):
					duration = pkg.ExpBackoff(duration, datasourceMaxInterval)
				}
			}
		}(s)
	}

	done := make(chan struct{})
	go func() {
		wg.Wait()
		close(done)
	}()

	var s datasource.Datasource
	select {
	case s = <-ds:
	case <-done:
	case <-time.After(datasourceTimeout):
	}

	close(stop)
	return s
}

// TODO(jonboulle): this should probably be refactored and moved into a different module
func runScript(script config.Script, env *initialize.Environment) error {
	err := initialize.PrepWorkspace(env.Workspace())
	if err != nil {
		fmt.Printf("Failed preparing workspace: %v\n", err)
		return err
	}
	path, err := initialize.PersistScriptInWorkspace(script, env.Workspace())
	if err == nil {
		var name string
		name, err = system.ExecuteScript(path)
		initialize.PersistUnitNameInWorkspace(name, env.Workspace())
	}
	return err
}
<commit_msg>coreos-cloudinit: bump to v1.3.4<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"os"
	"sync"
	"time"

	"github.com/coreos/coreos-cloudinit/config"
	"github.com/coreos/coreos-cloudinit/config/validate"
	"github.com/coreos/coreos-cloudinit/datasource"
	"github.com/coreos/coreos-cloudinit/datasource/configdrive"
	"github.com/coreos/coreos-cloudinit/datasource/file"
	"github.com/coreos/coreos-cloudinit/datasource/metadata/cloudsigma"
	"github.com/coreos/coreos-cloudinit/datasource/metadata/digitalocean"
	"github.com/coreos/coreos-cloudinit/datasource/metadata/ec2"
	"github.com/coreos/coreos-cloudinit/datasource/proc_cmdline"
	"github.com/coreos/coreos-cloudinit/datasource/url"
	"github.com/coreos/coreos-cloudinit/datasource/waagent"
	"github.com/coreos/coreos-cloudinit/initialize"
	"github.com/coreos/coreos-cloudinit/network"
	"github.com/coreos/coreos-cloudinit/pkg"
	"github.com/coreos/coreos-cloudinit/system"
)

const (
	version               = "1.3.4"
	datasourceInterval    = 100 * time.Millisecond
	datasourceMaxInterval = 30 * time.Second
	datasourceTimeout     = 5 * time.Minute
)

var (
	flags = struct {
		printVersion  bool
		ignoreFailure bool
		sources       struct {
			file                        string
			configDrive                 string
			waagent                     string
			metadataService             bool
			ec2MetadataService          string
			cloudSigmaMetadataService   bool
			digitalOceanMetadataService string
			url                         string
			procCmdLine                 bool
		}
		convertNetconf string
		workspace      string
		sshKeyName     string
		oem            string
		validate       bool
	}{}
)

func init() {
	flag.BoolVar(&flags.printVersion, "version", false, "Print the version and exit")
	flag.BoolVar(&flags.ignoreFailure, "ignore-failure", false, "Exits with 0 status in the event of malformed input from user-data")
	flag.StringVar(&flags.sources.file, "from-file", "", "Read user-data from provided file")
	flag.StringVar(&flags.sources.configDrive, "from-configdrive", "", "Read data from provided cloud-drive directory")
	flag.StringVar(&flags.sources.waagent, "from-waagent", "", "Read data from provided waagent directory")
	flag.BoolVar(&flags.sources.metadataService, "from-metadata-service", false, "[DEPRECATED - Use -from-ec2-metadata] Download data from metadata service")
	flag.StringVar(&flags.sources.ec2MetadataService, "from-ec2-metadata", "", "Download EC2 data from the provided url")
	flag.BoolVar(&flags.sources.cloudSigmaMetadataService, "from-cloudsigma-metadata", false, "Download data from CloudSigma server context")
	flag.StringVar(&flags.sources.digitalOceanMetadataService, "from-digitalocean-metadata", "", "Download DigitalOcean data from the provided url")
	flag.StringVar(&flags.sources.url, "from-url", "", "Download user-data from provided url")
	flag.BoolVar(&flags.sources.procCmdLine, "from-proc-cmdline", false, fmt.Sprintf("Parse %s for '%s=<url>', using the cloud-config served by an HTTP GET to <url>", proc_cmdline.ProcCmdlineLocation, proc_cmdline.ProcCmdlineCloudConfigFlag))
	flag.StringVar(&flags.oem, "oem", "", "Use the settings specific to the provided OEM")
	flag.StringVar(&flags.convertNetconf, "convert-netconf", "", "Read the network config provided in cloud-drive and translate it from the specified format into networkd unit files")
	flag.StringVar(&flags.workspace, "workspace", "/var/lib/coreos-cloudinit", "Base directory coreos-cloudinit should use to store data")
	flag.StringVar(&flags.sshKeyName, "ssh-key-name", initialize.DefaultSSHKeyName, "Add SSH keys to the system with the given name")
	flag.BoolVar(&flags.validate, "validate", false, "[EXPERIMENTAL] Validate the user-data but do not apply it to the system")
}

type oemConfig map[string]string

var (
	oemConfigs = map[string]oemConfig{
		"digitalocean": oemConfig{
			"from-digitalocean-metadata": "http://169.254.169.254/",
			"convert-netconf":            "digitalocean",
		},
		"ec2-compat": oemConfig{
			"from-ec2-metadata": "http://169.254.169.254/",
			"from-configdrive":  "/media/configdrive",
		},
		"rackspace-onmetal": oemConfig{
			"from-configdrive": "/media/configdrive",
			"convert-netconf":  "debian",
		},
		"azure": oemConfig{
			"from-waagent": "/var/lib/waagent",
		},
		"cloudsigma": oemConfig{
			"from-cloudsigma-metadata": "true",
		},
	}
)

func main() {
	failure := false

	flag.Parse()

	if c, ok := oemConfigs[flags.oem]; ok {
		for k, v := range c {
			flag.Set(k, v)
		}
	} else if flags.oem != "" {
		oems := make([]string, 0, len(oemConfigs))
		for k := range oemConfigs {
			oems = append(oems, k)
		}
		fmt.Printf("Invalid option to --oem: %q. Supported options: %q\n", flags.oem, oems)
		os.Exit(2)
	}

	if flags.printVersion == true {
		fmt.Printf("coreos-cloudinit version %s\n", version)
		os.Exit(0)
	}

	switch flags.convertNetconf {
	case "":
	case "debian":
	case "digitalocean":
	default:
		fmt.Printf("Invalid option to -convert-netconf: '%s'. Supported options: 'debian, digitalocean'\n", flags.convertNetconf)
		os.Exit(2)
	}

	dss := getDatasources()
	if len(dss) == 0 {
		fmt.Println("Provide at least one of --from-file, --from-configdrive, --from-ec2-metadata, --from-cloudsigma-metadata, --from-url or --from-proc-cmdline")
		os.Exit(2)
	}

	ds := selectDatasource(dss)
	if ds == nil {
		fmt.Println("No datasources available in time")
		os.Exit(1)
	}

	fmt.Printf("Fetching user-data from datasource of type %q\n", ds.Type())
	userdataBytes, err := ds.FetchUserdata()
	if err != nil {
		fmt.Printf("Failed fetching user-data from datasource: %v\nContinuing...\n", err)
		failure = true
	}

	if report, err := validate.Validate(userdataBytes); err == nil {
		ret := 0
		for _, e := range report.Entries() {
			fmt.Println(e)
			ret = 1
		}
		if flags.validate {
			os.Exit(ret)
		}
	} else {
		fmt.Printf("Failed while validating user_data (%q)\n", err)
		if flags.validate {
			os.Exit(1)
		}
	}

	fmt.Printf("Fetching meta-data from datasource of type %q\n", ds.Type())
	metadata, err := ds.FetchMetadata()
	if err != nil {
		fmt.Printf("Failed fetching meta-data from datasource: %v\n", err)
		os.Exit(1)
	}

	// Apply environment to user-data
	env := initialize.NewEnvironment("/", ds.ConfigRoot(), flags.workspace, flags.sshKeyName, metadata)
	userdata := env.Apply(string(userdataBytes))

	var ccu *config.CloudConfig
	var script *config.Script
	if ud, err := initialize.ParseUserData(userdata); err != nil {
		fmt.Printf("Failed to parse user-data: %v\nContinuing...\n", err)
		failure = true
	} else {
		switch t := ud.(type) {
		case *config.CloudConfig:
			ccu = t
		case *config.Script:
			script = t
		}
	}

	fmt.Println("Merging cloud-config from meta-data and user-data")
	cc := mergeConfigs(ccu, metadata)

	var ifaces []network.InterfaceGenerator
	if flags.convertNetconf != "" {
		var err error
		switch flags.convertNetconf {
		case "debian":
			ifaces, err = network.ProcessDebianNetconf(metadata.NetworkConfig)
		case "digitalocean":
			ifaces, err = network.ProcessDigitalOceanNetconf(metadata.NetworkConfig)
		default:
			err = fmt.Errorf("Unsupported network config format %q", flags.convertNetconf)
		}
		if err != nil {
			fmt.Printf("Failed to generate interfaces: %v\n", err)
			os.Exit(1)
		}
	}

	if err = initialize.Apply(cc, ifaces, env); err != nil {
		fmt.Printf("Failed to apply cloud-config: %v\n", err)
		os.Exit(1)
	}

	if script != nil {
		if err = runScript(*script, env); err != nil {
			fmt.Printf("Failed to run script: %v\n", err)
			os.Exit(1)
		}
	}

	if failure && !flags.ignoreFailure {
		os.Exit(1)
	}
}

// mergeConfigs merges certain options from md (meta-data from the datasource)
// onto cc (a CloudConfig derived from user-data), if they are not already set
// on cc (i.e. user-data always takes precedence)
func mergeConfigs(cc *config.CloudConfig, md datasource.Metadata) (out config.CloudConfig) {
	if cc != nil {
		out = *cc
	}

	if md.Hostname != "" {
		if out.Hostname != "" {
			fmt.Printf("Warning: user-data hostname (%s) overrides metadata hostname (%s)\n", out.Hostname, md.Hostname)
		} else {
			out.Hostname = md.Hostname
		}
	}
	for _, key := range md.SSHPublicKeys {
		out.SSHAuthorizedKeys = append(out.SSHAuthorizedKeys, key)
	}
	return
}

// getDatasources creates a slice of possible Datasources for cloudinit based
// on the different source command-line flags.
func getDatasources() []datasource.Datasource {
	dss := make([]datasource.Datasource, 0, 5)
	if flags.sources.file != "" {
		dss = append(dss, file.NewDatasource(flags.sources.file))
	}
	if flags.sources.url != "" {
		dss = append(dss, url.NewDatasource(flags.sources.url))
	}
	if flags.sources.configDrive != "" {
		dss = append(dss, configdrive.NewDatasource(flags.sources.configDrive))
	}
	if flags.sources.metadataService {
		dss = append(dss, ec2.NewDatasource(ec2.DefaultAddress))
	}
	if flags.sources.ec2MetadataService != "" {
		dss = append(dss, ec2.NewDatasource(flags.sources.ec2MetadataService))
	}
	if flags.sources.cloudSigmaMetadataService {
		dss = append(dss, cloudsigma.NewServerContextService())
	}
	if flags.sources.digitalOceanMetadataService != "" {
		dss = append(dss, digitalocean.NewDatasource(flags.sources.digitalOceanMetadataService))
	}
	if flags.sources.waagent != "" {
		dss = append(dss, waagent.NewDatasource(flags.sources.waagent))
	}
	if flags.sources.procCmdLine {
		dss = append(dss, proc_cmdline.NewDatasource())
	}
	return dss
}

// selectDatasource attempts to choose a valid Datasource to use based on its
// current availability. The first Datasource to report to be available is
// returned. Datasources will be retried if possible if they are not
// immediately available. If all Datasources are permanently unavailable or
// datasourceTimeout is reached before one becomes available, nil is returned.
func selectDatasource(sources []datasource.Datasource) datasource.Datasource {
	ds := make(chan datasource.Datasource)
	stop := make(chan struct{})
	var wg sync.WaitGroup

	for _, s := range sources {
		wg.Add(1)
		go func(s datasource.Datasource) {
			defer wg.Done()

			duration := datasourceInterval
			for {
				fmt.Printf("Checking availability of %q\n", s.Type())
				if s.IsAvailable() {
					ds <- s
					return
				} else if !s.AvailabilityChanges() {
					return
				}
				select {
				case <-stop:
					return
				case <-time.After(duration):
					duration = pkg.ExpBackoff(duration, datasourceMaxInterval)
				}
			}
		}(s)
	}

	done := make(chan struct{})
	go func() {
		wg.Wait()
		close(done)
	}()

	var s datasource.Datasource
	select {
	case s = <-ds:
	case <-done:
	case <-time.After(datasourceTimeout):
	}

	close(stop)
	return s
}

// TODO(jonboulle): this should probably be refactored and moved into a different module
func runScript(script config.Script, env *initialize.Environment) error {
	err := initialize.PrepWorkspace(env.Workspace())
	if err != nil {
		fmt.Printf("Failed preparing workspace: %v\n", err)
		return err
	}
	path, err := initialize.PersistScriptInWorkspace(script, env.Workspace())
	if err == nil {
		var name string
		name, err = system.ExecuteScript(path)
		initialize.PersistUnitNameInWorkspace(name, env.Workspace())
	}
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/coreos/coreos-cloudinit/datasource"
	"github.com/coreos/coreos-cloudinit/initialize"
	"github.com/coreos/coreos-cloudinit/system"
)

const version = "0.2.0"

func main() {
	var printVersion bool
	flag.BoolVar(&printVersion, "version", false, "Print the version and exit")

	var ignoreFailure bool
	flag.BoolVar(&ignoreFailure, "ignore-failure", false, "Exits with 0 status in the event of malformed input from user-data")

	var file string
	flag.StringVar(&file, "from-file", "", "Read user-data from provided file")

	var url string
	flag.StringVar(&url, "from-url", "", "Download user-data from provided url")

	var workspace string
	flag.StringVar(&workspace, "workspace", "/var/lib/coreos-cloudinit", "Base directory coreos-cloudinit should use to store data")

	var sshKeyName string
	flag.StringVar(&sshKeyName, "ssh-key-name", initialize.DefaultSSHKeyName, "Add SSH keys to the system with the given name")

	flag.Parse()

	if printVersion == true {
		fmt.Printf("coreos-cloudinit version %s\n", version)
		os.Exit(0)
	}

	if file != "" && url != "" {
		fmt.Println("Provide one of --from-file or --from-url")
		os.Exit(1)
	}

	var ds datasource.Datasource
	if file != "" {
		ds = datasource.NewLocalFile(file)
	} else if url != "" {
		ds = datasource.NewMetadataService(url)
	} else {
		fmt.Println("Provide one of --from-file or --from-url")
		os.Exit(1)
	}

	log.Printf("Fetching user-data from datasource of type %q", ds.Type())
	userdata, err := ds.Fetch()
	if err != nil {
		log.Printf("Failed fetching user-data from datasource: %v", err)
		if ignoreFailure {
			os.Exit(0)
		} else {
			os.Exit(1)
		}
	}

	if len(userdata) == 0 {
		log.Printf("No user data to handle, exiting.")
		os.Exit(0)
	}

	parsed, err := ParseUserData(userdata)
	if err != nil {
		log.Printf("Failed parsing user-data: %v", err)
		if ignoreFailure {
			os.Exit(0)
		} else {
			os.Exit(1)
		}
	}

	env := initialize.NewEnvironment("/", workspace)
	err = initialize.PrepWorkspace(env.Workspace())
	if err != nil {
		log.Fatalf("Failed preparing workspace: %v", err)
	}

	switch t := parsed.(type) {
	case initialize.CloudConfig:
		err = initialize.Apply(t, env)
	case system.Script:
		var path string
		path, err = initialize.PersistScriptInWorkspace(t, env.Workspace())
		if err == nil {
			var name string
			name, err = system.ExecuteScript(path)
			initialize.PersistUnitNameInWorkspace(name, workspace)
		}
	}

	if err != nil {
		log.Fatalf("Failed resolving user-data: %v", err)
	}
}

func ParseUserData(contents []byte) (interface{}, error) {
	bytereader := bytes.NewReader(contents)
	bufreader := bufio.NewReader(bytereader)
	header, _ := bufreader.ReadString('\n')

	if strings.HasPrefix(header, "#!") {
		log.Printf("Parsing user-data as script")
		return system.Script(contents), nil

	} else if header == "#cloud-config\n" {
		log.Printf("Parsing user-data as cloud-config")
		cfg, err := initialize.NewCloudConfig(contents)
		if err != nil {
			log.Fatal(err.Error())
		}
		return *cfg, nil
	} else {
		return nil, fmt.Errorf("Unrecognized user-data header: %s", header)
	}
}
<commit_msg>chore(release): Bump version to v0.2.0+git<commit_after>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/coreos/coreos-cloudinit/datasource"
	"github.com/coreos/coreos-cloudinit/initialize"
	"github.com/coreos/coreos-cloudinit/system"
)

const version = "0.2.0+git"

func main() {
	var printVersion bool
	flag.BoolVar(&printVersion, "version", false, "Print the version and exit")

	var ignoreFailure bool
	flag.BoolVar(&ignoreFailure, "ignore-failure", false, "Exits with 0 status in the event of malformed input from user-data")

	var file string
	flag.StringVar(&file, "from-file", "", "Read user-data from provided file")

	var url string
	flag.StringVar(&url, "from-url", "", "Download user-data from provided url")

	var workspace string
	flag.StringVar(&workspace, "workspace", "/var/lib/coreos-cloudinit", "Base directory coreos-cloudinit should use to store data")

	var sshKeyName string
	flag.StringVar(&sshKeyName, "ssh-key-name", initialize.DefaultSSHKeyName, "Add SSH keys to the system with the given name")

	flag.Parse()

	if printVersion == true {
		fmt.Printf("coreos-cloudinit version %s\n", version)
		os.Exit(0)
	}

	if file != "" && url != "" {
		fmt.Println("Provide one of --from-file or --from-url")
		os.Exit(1)
	}

	var ds datasource.Datasource
	if file != "" {
		ds = datasource.NewLocalFile(file)
	} else if url != "" {
		ds = datasource.NewMetadataService(url)
	} else {
		fmt.Println("Provide one of --from-file or --from-url")
		os.Exit(1)
	}

	log.Printf("Fetching user-data from datasource of type %q", ds.Type())
	userdata, err := ds.Fetch()
	if err != nil {
		log.Printf("Failed fetching user-data from datasource: %v", err)
		if ignoreFailure {
			os.Exit(0)
		} else {
			os.Exit(1)
		}
	}

	if len(userdata) == 0 {
		log.Printf("No user data to handle, exiting.")
		os.Exit(0)
	}

	parsed, err := ParseUserData(userdata)
	if err != nil {
		log.Printf("Failed parsing user-data: %v", err)
		if ignoreFailure {
			os.Exit(0)
		} else {
			os.Exit(1)
		}
	}

	env := initialize.NewEnvironment("/", workspace)
	err = initialize.PrepWorkspace(env.Workspace())
	if err != nil {
		log.Fatalf("Failed preparing workspace: %v", err)
	}

	switch t := parsed.(type) {
	case initialize.CloudConfig:
		err = initialize.Apply(t, env)
	case system.Script:
		var path string
		path, err = initialize.PersistScriptInWorkspace(t, env.Workspace())
		if err == nil {
			var name string
			name, err = system.ExecuteScript(path)
			initialize.PersistUnitNameInWorkspace(name, workspace)
		}
	}

	if err != nil {
		log.Fatalf("Failed resolving user-data: %v", err)
	}
}

func ParseUserData(contents []byte) (interface{}, error) {
	bytereader := bytes.NewReader(contents)
	bufreader := bufio.NewReader(bytereader)
	header, _ := bufreader.ReadString('\n')

	if strings.HasPrefix(header, "#!") {
		log.Printf("Parsing user-data as script")
		return system.Script(contents), nil

	} else if header == "#cloud-config\n" {
		log.Printf("Parsing user-data as cloud-config")
		cfg, err := initialize.NewCloudConfig(contents)
		if err != nil {
			log.Fatal(err.Error())
		}
		return *cfg, nil
	} else {
		return nil, fmt.Errorf("Unrecognized user-data header: %s", header)
	}
}
<|endoftext|>


<commit_before>//  Copyright 2019 Google Inc. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

// GCE VM image export tool
package main

import (
	"context"
	"flag"
	"fmt"
	"log"
	"os"

	"github.com/GoogleCloudPlatform/compute-image-tools/cli_tools/common/utils/compute"
	"github.com/GoogleCloudPlatform/compute-image-tools/cli_tools/common/utils/daisy"
	"github.com/GoogleCloudPlatform/compute-image-tools/cli_tools/common/utils/logging"
	"github.com/GoogleCloudPlatform/compute-image-tools/cli_tools/common/utils/param"
	"github.com/GoogleCloudPlatform/compute-image-tools/cli_tools/common/utils/path"
	"github.com/GoogleCloudPlatform/compute-image-tools/cli_tools/common/utils/storage"
	"github.com/GoogleCloudPlatform/compute-image-tools/cli_tools/common/utils/validation"
	"github.com/GoogleCloudPlatform/compute-image-tools/cli_tools/daisy_common"
	"github.com/GoogleCloudPlatform/compute-image-tools/daisy"
)

const (
	workflowDir              = "daisy_workflows/export/"
	exportWorkflow           = workflowDir + "image_export.wf.json"
	exportAndConvertWorkflow = workflowDir + "image_export_ext.wf.json"
	clientIDFlagKey          = "client_id"
	destinationURIFlagKey    = "destination_uri"
	sourceImageFlagKey       = "source_image"
)

var (
	clientID             = flag.String(clientIDFlagKey, "", "Identifies the client of the importer, e.g. `gcloud` or `pantheon`.")
	destinationURI       = flag.String(destinationURIFlagKey, "", "The Google Cloud Storage URI destination for the exported virtual disk file. For example: gs://my-bucket/my-exported-image.vmdk.")
	sourceImage          = flag.String(sourceImageFlagKey, "", "Compute Engine image from which to export")
	format               = flag.String("format", "", "Specify the format to export to, such as vmdk, vhdx, vpc, or qcow2.")
	project              = flag.String("project", "", "Project to run in, overrides what is set in workflow.")
	network              = flag.String("network", "", "Name of the network in your project to use for the image import. The network must have access to Google Cloud Storage. If not specified, the network named default is used.")
	subnet               = flag.String("subnet", "", "Name of the subnetwork in your project to use for the image import. If	the network resource is in legacy mode, do not provide this property. If the network is in auto subnet mode, providing the subnetwork is optional. If the network is in custom subnet mode, then this field should be specified. Zone should be specified if this field is specified.")
	zone                 = flag.String("zone", "", "Zone of the image to import. The zone in which to do the work of importing the image. Overrides the default compute/zone property value for this command invocation.")
	timeout              = flag.String("timeout", "", "Maximum time a build can last before it is failed as TIMEOUT. For example, specifying 2h will fail the process after 2 hours. See $ gcloud topic datetimes for information on duration formats.")
	scratchBucketGcsPath = flag.String("scratch_bucket_gcs_path", "", "GCS scratch bucket to use, overrides what is set in workflow.")
	oauth                = flag.String("oauth", "", "path to oauth json file, overrides what is set in workflow.")
	ce                   = flag.String("compute_endpoint_override", "", "API endpoint to override default.")
	gcsLogsDisabled      = flag.Bool("disable_gcs_logging", false, "do not stream logs to GCS.")
	cloudLogsDisabled    = flag.Bool("disable_cloud_logging", false, "do not stream logs to Cloud Logging.")
	stdoutLogsDisabled   = flag.Bool("disable_stdout_logging", false, "do not display individual workflow logs on stdout.")
	labels               = flag.String("labels", "", "List of label KEY=VALUE pairs to add. Keys must start with a lowercase character and contain only hyphens (-), underscores (_), lowercase characters, and numbers. Values must contain only hyphens (-), underscores (_), lowercase characters, and numbers.")

	region  = new(string)
	buildID = os.Getenv("BUILD_ID")

	userLabels            map[string]string
	currentExecutablePath *string
)

func init() {
	currentExecutablePathStr := string(os.Args[0])
	currentExecutablePath = &currentExecutablePathStr
}

func validateAndParseFlags() error {
	flag.Parse()

	if err := validationutils.ValidateStringFlagNotEmpty(*clientID, clientIDFlagKey); err != nil {
		return err
	}
	if err := validationutils.ValidateStringFlagNotEmpty(*destinationURI, destinationURIFlagKey); err != nil {
		return err
	}
	if err := validationutils.ValidateStringFlagNotEmpty(*sourceImage, sourceImageFlagKey); err != nil {
		return err
	}

	if *labels != "" {
		var err error
		userLabels, err = paramutils.ParseKeyValues(*labels)
		if err != nil {
			return err
		}
	}

	return nil
}

func getWorkflowPath() string {
	if *format == "" {
		return pathutils.ToWorkingDir(exportWorkflow, *currentExecutablePath)
	}

	return pathutils.ToWorkingDir(exportAndConvertWorkflow, *currentExecutablePath)
}

func buildDaisyVars() map[string]string {
	varMap := map[string]string{}

	varMap["destination"] = *destinationURI

	varMap["source_image"] = *sourceImage

	if *format != "" {
		varMap["format"] = *format
	}
	if *subnet != "" {
		varMap["export_subnet"] = fmt.Sprintf("regions/%v/subnetworks/%v", *region, *subnet)
		// When subnet is set, we need to grant a value to network to avoid fallback to default
		if *network == "" {
			varMap["import_network"] = ""
		}
	}
	if *network != "" {
		varMap["export_network"] = fmt.Sprintf("global/networks/%v", *network)
	}
	return varMap
}

func runExport(ctx context.Context) error {
	exportWorkflowPath := getWorkflowPath()
	varMap := buildDaisyVars()
	workflow, err := daisycommon.ParseWorkflow(exportWorkflowPath, varMap,
		*project, *zone, *scratchBucketGcsPath, *oauth, *timeout, *ce, *gcsLogsDisabled,
		*cloudLogsDisabled, *stdoutLogsDisabled)
	if err != nil {
		return err
	}

	workflowModifier := func(w *daisy.Workflow) {
		rl := &daisyutils.ResourceLabeler{
			BuildID: buildID, UserLabels: userLabels, BuildIDLabelKey: "gce-image-export-build-id",
			InstanceLabelKeyRetriever: func(instance *daisy.Instance) string {
				return "gce-image-export-tmp"
			},
			DiskLabelKeyRetriever: func(disk *daisy.Disk) string {
				return "gce-image-export-tmp"
			},
			ImageLabelKeyRetriever: func(image *daisy.Image) string {
				return "gce-image-export"
			}}
		rl.LabelResources(w)
	}

	return workflow.RunWithModifiers(ctx, nil, workflowModifier)
}

func main() {
	if err := validateAndParseFlags(); err != nil {
		log.Fatalf(err.Error())
	}

	ctx := context.Background()
	metadataGCE := &computeutils.MetadataGCE{}
	storageClient, err := storageutils.NewStorageClient(
		ctx, logging.NewLogger("[image-export]"), oauth)
	if err != nil {
		log.Fatalf("error creating storage client %v", err)
	}
	defer storageClient.Close()

	scratchBucketCreator := storageutils.NewScratchBucketCreator(ctx, storageClient)
	zoneRetriever, err := storageutils.NewZoneRetriever(metadataGCE, paramutils.CreateComputeClient(&ctx, *oauth, *ce))
	if err != nil {
		log.Fatalf(err.Error())
	}

	err = paramutils.PopulateMissingParameters(project, zone, region, scratchBucketGcsPath,
		*destinationURI, metadataGCE, scratchBucketCreator, zoneRetriever, storageClient)
	if err != nil {
		log.Fatalf(err.Error())
	}

	if err := runExport(ctx); err != nil {
		log.Fatalf(err.Error())
	}
}
<commit_msg>fix export_network param name (#856)<commit_after>//  Copyright 2019 Google Inc. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

// GCE VM image export tool
package main

import (
	"context"
	"flag"
	"fmt"
	"log"
	"os"

	"github.com/GoogleCloudPlatform/compute-image-tools/cli_tools/common/utils/compute"
	"github.com/GoogleCloudPlatform/compute-image-tools/cli_tools/common/utils/daisy"
	"github.com/GoogleCloudPlatform/compute-image-tools/cli_tools/common/utils/logging"
	"github.com/GoogleCloudPlatform/compute-image-tools/cli_tools/common/utils/param"
	"github.com/GoogleCloudPlatform/compute-image-tools/cli_tools/common/utils/path"
	"github.com/GoogleCloudPlatform/compute-image-tools/cli_tools/common/utils/storage"
	"github.com/GoogleCloudPlatform/compute-image-tools/cli_tools/common/utils/validation"
	"github.com/GoogleCloudPlatform/compute-image-tools/cli_tools/daisy_common"
	"github.com/GoogleCloudPlatform/compute-image-tools/daisy"
)

const (
	workflowDir              = "daisy_workflows/export/"
	exportWorkflow           = workflowDir + "image_export.wf.json"
	exportAndConvertWorkflow = workflowDir + "image_export_ext.wf.json"
	clientIDFlagKey          = "client_id"
	destinationURIFlagKey    = "destination_uri"
	sourceImageFlagKey       = "source_image"
)

var (
	clientID             = flag.String(clientIDFlagKey, "", "Identifies the client of the importer, e.g. `gcloud` or `pantheon`.")
	destinationURI       = flag.String(destinationURIFlagKey, "", "The Google Cloud Storage URI destination for the exported virtual disk file. For example: gs://my-bucket/my-exported-image.vmdk.")
	sourceImage          = flag.String(sourceImageFlagKey, "", "Compute Engine image from which to export")
	format               = flag.String("format", "", "Specify the format to export to, such as vmdk, vhdx, vpc, or qcow2.")
	project              = flag.String("project", "", "Project to run in, overrides what is set in workflow.")
	network              = flag.String("network", "", "Name of the network in your project to use for the image import. The network must have access to Google Cloud Storage. If not specified, the network named default is used.")
	subnet               = flag.String("subnet", "", "Name of the subnetwork in your project to use for the image import. If	the network resource is in legacy mode, do not provide this property. If the network is in auto subnet mode, providing the subnetwork is optional. If the network is in custom subnet mode, then this field should be specified. Zone should be specified if this field is specified.")
	zone                 = flag.String("zone", "", "Zone of the image to import. The zone in which to do the work of importing the image. Overrides the default compute/zone property value for this command invocation.")
	timeout              = flag.String("timeout", "", "Maximum time a build can last before it is failed as TIMEOUT. For example, specifying 2h will fail the process after 2 hours. See $ gcloud topic datetimes for information on duration formats.")
	scratchBucketGcsPath = flag.String("scratch_bucket_gcs_path", "", "GCS scratch bucket to use, overrides what is set in workflow.")
	oauth                = flag.String("oauth", "", "path to oauth json file, overrides what is set in workflow.")
	ce                   = flag.String("compute_endpoint_override", "", "API endpoint to override default.")
	gcsLogsDisabled      = flag.Bool("disable_gcs_logging", false, "do not stream logs to GCS.")
	cloudLogsDisabled    = flag.Bool("disable_cloud_logging", false, "do not stream logs to Cloud Logging.")
	stdoutLogsDisabled   = flag.Bool("disable_stdout_logging", false, "do not display individual workflow logs on stdout.")
	labels               = flag.String("labels", "", "List of label KEY=VALUE pairs to add. Keys must start with a lowercase character and contain only hyphens (-), underscores (_), lowercase characters, and numbers. Values must contain only hyphens (-), underscores (_), lowercase characters, and numbers.")

	region  = new(string)
	buildID = os.Getenv("BUILD_ID")

	userLabels            map[string]string
	currentExecutablePath *string
)

func init() {
	currentExecutablePathStr := string(os.Args[0])
	currentExecutablePath = &currentExecutablePathStr
}

func validateAndParseFlags() error {
	flag.Parse()

	if err := validationutils.ValidateStringFlagNotEmpty(*clientID, clientIDFlagKey); err != nil {
		return err
	}
	if err := validationutils.ValidateStringFlagNotEmpty(*destinationURI, destinationURIFlagKey); err != nil {
		return err
	}
	if err := validationutils.ValidateStringFlagNotEmpty(*sourceImage, sourceImageFlagKey); err != nil {
		return err
	}

	if *labels != "" {
		var err error
		userLabels, err = paramutils.ParseKeyValues(*labels)
		if err != nil {
			return err
		}
	}

	return nil
}

func getWorkflowPath() string {
	if *format == "" {
		return pathutils.ToWorkingDir(exportWorkflow, *currentExecutablePath)
	}

	return pathutils.ToWorkingDir(exportAndConvertWorkflow, *currentExecutablePath)
}

func buildDaisyVars() map[string]string {
	varMap := map[string]string{}

	varMap["destination"] = *destinationURI

	varMap["source_image"] = *sourceImage

	if *format != "" {
		varMap["format"] = *format
	}
	if *subnet != "" {
		varMap["export_subnet"] = fmt.Sprintf("regions/%v/subnetworks/%v", *region, *subnet)
		// When subnet is set, we need to grant a value to network to avoid fallback to default
		if *network == "" {
			varMap["export_network"] = ""
		}
	}
	if *network != "" {
		varMap["export_network"] = fmt.Sprintf("global/networks/%v", *network)
	}
	return varMap
}

func runExport(ctx context.Context) error {
	exportWorkflowPath := getWorkflowPath()
	varMap := buildDaisyVars()
	workflow, err := daisycommon.ParseWorkflow(exportWorkflowPath, varMap,
		*project, *zone, *scratchBucketGcsPath, *oauth, *timeout, *ce, *gcsLogsDisabled,
		*cloudLogsDisabled, *stdoutLogsDisabled)
	if err != nil {
		return err
	}

	workflowModifier := func(w *daisy.Workflow) {
		rl := &daisyutils.ResourceLabeler{
			BuildID: buildID, UserLabels: userLabels, BuildIDLabelKey: "gce-image-export-build-id",
			InstanceLabelKeyRetriever: func(instance *daisy.Instance) string {
				return "gce-image-export-tmp"
			},
			DiskLabelKeyRetriever: func(disk *daisy.Disk) string {
				return "gce-image-export-tmp"
			},
			ImageLabelKeyRetriever: func(image *daisy.Image) string {
				return "gce-image-export"
			}}
		rl.LabelResources(w)
	}

	return workflow.RunWithModifiers(ctx, nil, workflowModifier)
}

func main() {
	if err := validateAndParseFlags(); err != nil {
		log.Fatalf(err.Error())
	}

	ctx := context.Background()
	metadataGCE := &computeutils.MetadataGCE{}
	storageClient, err := storageutils.NewStorageClient(
		ctx, logging.NewLogger("[image-export]"), oauth)
	if err != nil {
		log.Fatalf("error creating storage client %v", err)
	}
	defer storageClient.Close()

	scratchBucketCreator := storageutils.NewScratchBucketCreator(ctx, storageClient)
	zoneRetriever, err := storageutils.NewZoneRetriever(metadataGCE, paramutils.CreateComputeClient(&ctx, *oauth, *ce))
	if err != nil {
		log.Fatalf(err.Error())
	}

	err = paramutils.PopulateMissingParameters(project, zone, region, scratchBucketGcsPath,
		*destinationURI, metadataGCE, scratchBucketCreator, zoneRetriever, storageClient)
	if err != nil {
		log.Fatalf(err.Error())
	}

	if err := runExport(ctx); err != nil {
		log.Fatalf(err.Error())
	}
}
<|endoftext|>


<commit_before>package state

import (
	"testing"
	"time"
)

func TestTombstoneGC_invalid(t *testing.T) {
	_, err := NewTombstoneGC(0, 0)
	if err == nil {
		t.Fatalf("should fail")
	}

	_, err = NewTombstoneGC(time.Second, 0)
	if err == nil {
		t.Fatalf("should fail")
	}

	_, err = NewTombstoneGC(0, time.Second)
	if err == nil {
		t.Fatalf("should fail")
	}
}

func TestTombstoneGC(t *testing.T) {
	ttl := 20 * time.Millisecond
	gran := 5 * time.Millisecond
	gc, err := NewTombstoneGC(ttl, gran)
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	gc.SetEnabled(true)

	if gc.PendingExpiration() {
		t.Fatalf("should not be pending")
	}

	start := time.Now()
	gc.Hint(100)

	if !gc.PendingExpiration() {
		t.Fatalf("should be pending")
	}

	select {
	case index := <-gc.ExpireCh():
		end := time.Now()
		if end.Sub(start) < ttl {
			t.Fatalf("expired early")
		}
		if index != 100 {
			t.Fatalf("bad index: %d", index)
		}

	case <-time.After(ttl * 2):
		t.Fatalf("should get expiration")
	}

	start2 := time.Now()
	gc.Hint(120)
	gc.Hint(125)

	if !gc.PendingExpiration() {
		t.Fatalf("should be pending")
	}

	select {
	case index := <-gc.ExpireCh():
		end := time.Now()
		if end.Sub(start2) < ttl {
			t.Fatalf("expired early")
		}
		if index != 125 {
			t.Fatalf("bad index: %d", index)
		}

	case <-time.After(ttl * 2):
		t.Fatalf("should get expiration")
	}
}

func TestTombstoneGC_Expire(t *testing.T) {
	ttl := 10 * time.Millisecond
	gran := 5 * time.Millisecond
	gc, err := NewTombstoneGC(ttl, gran)
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	gc.SetEnabled(true)

	if gc.PendingExpiration() {
		t.Fatalf("should not be pending")
	}

	gc.Hint(100)
	gc.SetEnabled(false)

	if gc.PendingExpiration() {
		t.Fatalf("should not be pending")
	}

	select {
	case <-gc.ExpireCh():
		t.Fatalf("should be reset")
	case <-time.After(ttl * 2):
	}
}
<commit_msg>Skips the tombstone GC test in Travis for now.<commit_after>package state

import (
	"os"
	"testing"
	"time"
)

func TestTombstoneGC_invalid(t *testing.T) {
	_, err := NewTombstoneGC(0, 0)
	if err == nil {
		t.Fatalf("should fail")
	}

	_, err = NewTombstoneGC(time.Second, 0)
	if err == nil {
		t.Fatalf("should fail")
	}

	_, err = NewTombstoneGC(0, time.Second)
	if err == nil {
		t.Fatalf("should fail")
	}
}

func TestTombstoneGC(t *testing.T) {
	if os.Getenv("TRAVIS") == "true" {
		t.Skip("GC test is flaky on travis-ci (see #3670)")
	}

	ttl := 20 * time.Millisecond
	gran := 5 * time.Millisecond
	gc, err := NewTombstoneGC(ttl, gran)
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	gc.SetEnabled(true)

	if gc.PendingExpiration() {
		t.Fatalf("should not be pending")
	}

	start := time.Now()
	gc.Hint(100)

	if !gc.PendingExpiration() {
		t.Fatalf("should be pending")
	}

	select {
	case index := <-gc.ExpireCh():
		end := time.Now()
		if end.Sub(start) < ttl {
			t.Fatalf("expired early")
		}
		if index != 100 {
			t.Fatalf("bad index: %d", index)
		}

	case <-time.After(ttl * 2):
		t.Fatalf("should get expiration")
	}

	start2 := time.Now()
	gc.Hint(120)
	gc.Hint(125)

	if !gc.PendingExpiration() {
		t.Fatalf("should be pending")
	}

	select {
	case index := <-gc.ExpireCh():
		end := time.Now()
		if end.Sub(start2) < ttl {
			t.Fatalf("expired early")
		}
		if index != 125 {
			t.Fatalf("bad index: %d", index)
		}

	case <-time.After(ttl * 2):
		t.Fatalf("should get expiration")
	}
}

func TestTombstoneGC_Expire(t *testing.T) {
	ttl := 10 * time.Millisecond
	gran := 5 * time.Millisecond
	gc, err := NewTombstoneGC(ttl, gran)
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	gc.SetEnabled(true)

	if gc.PendingExpiration() {
		t.Fatalf("should not be pending")
	}

	gc.Hint(100)
	gc.SetEnabled(false)

	if gc.PendingExpiration() {
		t.Fatalf("should not be pending")
	}

	select {
	case <-gc.ExpireCh():
		t.Fatalf("should be reset")
	case <-time.After(ttl * 2):
	}
}
<|endoftext|>


<commit_before>package api

import (
	"socialapi/workers/integration/webhook"
	"socialapi/workers/integration/webhook/services"
)

type PrepareRequest struct {
	Data  *services.ServiceInput
	Token string
	Name  string
	// Username is used for sending messages
	// to given user's bot channel
	Username string
}

func (p *PrepareRequest) validate() error {
	if p.Token == "" {
		return ErrTokenNotSet
	}

	if p.Name == "" {
		return ErrNameNotSet
	}

	return nil
}

func (p *PrepareRequest) verify() (*webhook.Integration, error) {
	i := webhook.NewIntegration()
	err := i.ByName(p.Name)
	if err != nil {
		return nil, err
	}

	return i, nil
}
<commit_msg>webhook: add groupName field to PrepareRequest<commit_after>package api

import (
	"socialapi/workers/integration/webhook"
	"socialapi/workers/integration/webhook/services"
)

type PrepareRequest struct {
	Data      *services.ServiceInput
	Token     string
	Name      string
	GroupName string
	// Username is used for sending messages
	// to given user's bot channel
	Username string
}

func (p *PrepareRequest) validate() error {
	if p.Token == "" {
		return ErrTokenNotSet
	}

	if p.Name == "" {
		return ErrNameNotSet
	}

	return nil
}

func (p *PrepareRequest) verify() (*webhook.Integration, error) {
	i := webhook.NewIntegration()
	err := i.ByName(p.Name)
	if err != nil {
		return nil, err
	}

	return i, nil
}
<|endoftext|>


<commit_before>//go:generate go-bindata -nocompress voices/dmr voices/dstar voices/p25

package main

import (
	"io"
	"log"
	"flag"
	"net"
	"strings"
	"encoding/binary"
	"bytes"
	"path/filepath"
	"os"
	"time"
	"fmt"
)

func getFilePathForCodePair(modemMode spkModemMode, codePair string) string {
	var dir string

	switch (modemMode) {
		case SPK_MODEM_MODE_DMR: dir = "voices/dmr/"
		case SPK_MODEM_MODE_DSTAR: dir = "voices/dstar/"
		case SPK_MODEM_MODE_C4FM: dir = "voices/dmr/"
		case SPK_MODEM_MODE_C4FM_HALF_DEVIATION: dir = "voices/dmr/"
		case SPK_MODEM_MODE_NXDN: dir = "voices/dmr/"
		case SPK_MODEM_MODE_P25: dir = "voices/p25/"
	}
	if dir == "" {
		return dir
	}

	for filePath, _ := range _bindata {
		if filepath.Dir(filePath) != filepath.Clean(dir) || filepath.Ext(filePath) != ".ambe" {
			continue
		}

		fileName := filepath.Base(filePath)
		// The requested code pair is stored in the first two characters of the filename.
		var fileCodePair = string(fileName[0]) + string(fileName[1])
		if fileCodePair == codePair {
			return filePath
		}
	}
	return ""
}

func sendAMBEAnswer(udpConn *net.UDPConn, toAddr *net.UDPAddr, res *spkAMBEResponsePacket) {
	var buf bytes.Buffer
	binary.Write(&buf, binary.BigEndian, res)
	writtenBytes, err := udpConn.WriteToUDP(buf.Bytes(), toAddr)
	if writtenBytes != SPK_AMBE_RESPONSE_PACKET_SIZE || err != nil {
		log.Printf("warning: can't send udp packet to %s\n", toAddr.String())
	}

	if res.PacketType != SPK_PACKET_TYPE_RESPONSE_TERMINATOR {
		res.SeqNum++
		time.Sleep(time.Duration(res.FrameCount) * 20 * time.Millisecond)
	}
}

func sendIMBEAnswer(udpConn *net.UDPConn, toAddr *net.UDPAddr, res *spkIMBEResponsePacket) {
	var buf bytes.Buffer
	binary.Write(&buf, binary.BigEndian, res)
	writtenBytes, err := udpConn.WriteToUDP(buf.Bytes(), toAddr)
	if writtenBytes != SPK_IMBE_RESPONSE_PACKET_SIZE || err != nil {
		log.Printf("warning: can't send udp packet to %s\n", toAddr.String())
	}

	if res.PacketType != SPK_PACKET_TYPE_RESPONSE_TERMINATOR {
		res.SeqNum++
		time.Sleep(time.Duration(res.FrameCount) * 20 * time.Millisecond)
	}
}

func StartSendAnswer(udpConn *net.UDPConn, toAddr net.UDPAddr, rp *spkRequestPacket) {
	defer RequestRemove(rp.SessionID, &toAddr)

	codeStr := strings.TrimRight(string(rp.CodeStr[:]), "\x00")

	// If the client is requesting a connect announce to a Homebrew server, we try to query a BM status from
	// the server's BM HTTP API to get linked talkgroups and reflector.
	bmGetClientDataFinished := make(chan bool)
	defer close(bmGetClientDataFinished)
	bmGetClientDataRunning := false
	var bmGetClientDataResult bmClientData
	var serverData bmServerData
	if rp.ConnectorID == SPK_CONNECTOR_ID_HOMEBREW &&
		(rp.AnnounceType == SPK_ANNOUNCE_TYPE_CONNECTED || rp.AnnounceType == SPK_ANNOUNCE_TYPE_CONNECTED_BRANDMEISTER_SHORTENED ||
			rp.AnnounceType == SPK_ANNOUNCE_TYPE_CONNECTOR_STATUS) {

		serverIP := fmt.Sprintf("%d.%d.%d.%d", rp.AnnounceTypeData[0] >> 24, (rp.AnnounceTypeData[0] >> 16) & 0xff,
			(rp.AnnounceTypeData[0] >> 8) & 0xff, rp.AnnounceTypeData[0] & 0xff)

		var ok bool
		if serverData, ok = BMGetServerDataForServerIP(serverIP); ok {
			clientId := rp.AnnounceTypeData[1]

			log.Printf("getting bm client data srv:%s cid:%d", serverIP, clientId)
			bmGetClientDataRunning = true
			go BMGetClientData(clientId, &bmGetClientDataResult, bmGetClientDataFinished)
		}
	}

	var res spkResponsePacket
	switch (rp.ModemMode) {
		default:
			copy(res.AMBE.Magic[:], SPK_PACKET_MAGIC)
			res.AMBE.PacketType = SPK_PACKET_TYPE_AMBE_RESPONSE
			res.AMBE.SessionID = rp.SessionID
			break
		case SPK_MODEM_MODE_P25:
			copy(res.IMBE.Magic[:], SPK_PACKET_MAGIC)
			res.IMBE.PacketType = SPK_PACKET_TYPE_IMBE_RESPONSE
			res.IMBE.SessionID = rp.SessionID
			break
	}

	// Stepping through each code char pair.
	for codeStrPos := 0; codeStrPos < len(codeStr); codeStrPos += 2 {
		if bmGetClientDataRunning {
			select {
				case finished := <- bmGetClientDataFinished:
					if finished && codeStrPos < 4 {
						codeStr = strings.Replace(codeStr, "HBSV", BMGenerateCodeStrFromClientData(&bmGetClientDataResult, &serverData,
							rp.AnnounceType == SPK_ANNOUNCE_TYPE_CONNECTED_BRANDMEISTER_SHORTENED), 1)
						log.Printf("code str modified for %s to %s", toAddr.String(), codeStr)
						bmGetClientDataRunning = false
					}
				default:
					break
			}
		}

		if codeStrPos+2 > len(codeStr) {
			log.Println("warning: last code pair is broken")
			break
		}

		var codePair = codeStr[codeStrPos:codeStrPos+2]

		filePath := getFilePathForCodePair(rp.ModemMode, codePair)
		if filePath == "" {
			log.Printf("warning: file not found for modem mode %d code pair \"%s\", skipping\n", rp.ModemMode, codePair)
			continue
		}

		data, err := Asset(filePath)
		if err != nil {
			log.Printf("warning: can't find \"%s\", skipping\n", filePath)
			continue
		}

		log.Printf("playing %s to %s\n", filePath, toAddr.String())

		reader := bytes.NewReader(data)
		var fileFinished = false

		// Filling up frames from the file.
		for !fileFinished {
			switch (rp.ModemMode) {
				default:
					for ; res.AMBE.FrameCount < 3; res.AMBE.FrameCount++ {
						readBytes, err := reader.Read(res.AMBE.Frames[res.AMBE.FrameCount][:])
						if err != nil || readBytes != 9 {
							fileFinished = true
							break
						}
					}

					// Flushing if needed.
					if res.AMBE.FrameCount == 3 {
						sendAMBEAnswer(udpConn, &toAddr, &res.AMBE)
						res.AMBE.FrameCount = 0
					}
					break
				case SPK_MODEM_MODE_P25:
					for ; res.IMBE.FrameCount < 3; res.IMBE.FrameCount++ {
						readBytes, err := reader.Read(res.IMBE.Frames[res.IMBE.FrameCount][:])
						if err != nil || readBytes != 18 {
							fileFinished = true
							break
						}
					}

					// Flushing if needed.
					if res.IMBE.FrameCount == 3 {
						sendIMBEAnswer(udpConn, &toAddr, &res.IMBE)
						res.IMBE.FrameCount = 0
					}
					break
			}
		}
	}

	switch (rp.ModemMode) {
		default:
			res.AMBE.PacketType = SPK_PACKET_TYPE_RESPONSE_TERMINATOR
			sendAMBEAnswer(udpConn, &toAddr, &res.AMBE)
			break;
		case SPK_MODEM_MODE_P25:
			res.IMBE.PacketType = SPK_PACKET_TYPE_RESPONSE_TERMINATOR
			sendIMBEAnswer(udpConn, &toAddr, &res.IMBE)
			break
	}

	if bmGetClientDataRunning {
		<- bmGetClientDataFinished
	}

	log.Printf("playing to %s finished\n", toAddr.String())
}

func processPacket(udpConn *net.UDPConn, fromAddr *net.UDPAddr, buffer []byte, readBytes int) {
	var packetType = buffer[7]

	switch (packetType) {
		default:
			log.Printf("ignoring packet with type 0x%.2x\n", packetType)
		case SPK_PACKET_TYPE_REQUEST:
			if readBytes != SPK_REQUEST_PACKET_SIZE {
				log.Printf("ignoring packet with size %d\n", readBytes)
				return
			}

			// Reading the packet payload to our request struct.
			readBuf := bytes.NewReader(buffer)
			var rp spkRequestPacket
			err := binary.Read(readBuf, binary.BigEndian, &rp)
			if err != nil {
				log.Println("ignoring packet, binary parse error: ", err)
				return
			}

			switch (rp.ModemMode) {
				case SPK_MODEM_MODE_DMR: break
				case SPK_MODEM_MODE_DSTAR: break
				case SPK_MODEM_MODE_C4FM: break
				case SPK_MODEM_MODE_C4FM_HALF_DEVIATION: break
				case SPK_MODEM_MODE_NXDN: break
				case SPK_MODEM_MODE_P25: break
				default:
					log.Printf("ignoring packet, invalid modem mode %.2x\n", rp.ModemMode)
					return
			}

			rp.CodeStr[SPK_ANNOUNCE_DATA_MAX_LENGTH-1] = 0

			if RequestIsAdded(rp.SessionID, fromAddr) {
				//log.Printf("ignoring packet, request already under processing with sid:0x%.8x\n", rp.SessionID)
				return
			}
			RequestAdd(rp.SessionID, fromAddr)

			atStr, atdStr := decodeAnnounceTypeAndDataToStr(rp.AnnounceType, rp.AnnounceTypeData)
			log.Printf("sending \"%s\" to %s (sid:0x%.8x t:%s con:%s at:%s %s)\n",
				strings.TrimRight(string(rp.CodeStr[:]), "\x00"), fromAddr.String(), rp.SessionID, getModemModeNameStr(rp.ModemMode),
				getConnectorIdNameStr(rp.ConnectorID), atStr, atdStr);
			go StartSendAnswer(udpConn, *fromAddr, &rp)
	}
}

func main() {
	var bindIp = "0.0.0.0"
	var bindPort = 65200

	flag.IntVar(&bindPort, "p", bindPort, "bind to port")
	flag.StringVar(&bindIp, "i", bindIp, "bind to ip address")
	flag.Parse()

	logFile, err := os.OpenFile("spk-srv.log", os.O_APPEND | os.O_WRONLY | os.O_CREATE, 0600)
	if err != nil {
		log.Println("warning: can't open spk-srv.log for writing: ", err)
	} else {
		defer logFile.Close()
		log.SetOutput(io.MultiWriter(os.Stdout, logFile))
	}

	log.Printf("binding to %s:%d\n", bindIp, bindPort)
	udpConn, err := net.ListenUDP("udp4", &net.UDPAddr{ net.ParseIP(bindIp), bindPort, "" })
	if err != nil {
		log.Fatal(err)
	}
	defer udpConn.Close()

	go BMProcess()

	log.Println("starting listening loop")
	buffer := make([]byte, 128)
	for {
		readBytes, fromAddr, err := udpConn.ReadFromUDP(buffer)
		if err != nil {
			log.Fatal(err)
		}

		// Did we read at least magic + version number of bytes? Does packet magic match?
		if readBytes >= 7 && strings.Compare(SPK_PACKET_MAGIC, string(buffer[:6])) == 0 {
			log.Printf("got %d byte packet from %s\n", readBytes, fromAddr.String())

			switch (buffer[6]) {
				default:
					log.Printf("ignoring packet with version 0x%.2x\n", buffer[6])
				case 0:
					processPacket(udpConn, fromAddr, buffer, readBytes)
			}
		}
	}
}
<commit_msg>Add flag to turn on logging to file (disabled by default)<commit_after>//go:generate go-bindata -nocompress voices/dmr voices/dstar voices/p25

package main

import (
	"io"
	"log"
	"flag"
	"net"
	"strings"
	"encoding/binary"
	"bytes"
	"path/filepath"
	"os"
	"time"
	"fmt"
)

func getFilePathForCodePair(modemMode spkModemMode, codePair string) string {
	var dir string

	switch (modemMode) {
		case SPK_MODEM_MODE_DMR: dir = "voices/dmr/"
		case SPK_MODEM_MODE_DSTAR: dir = "voices/dstar/"
		case SPK_MODEM_MODE_C4FM: dir = "voices/dmr/"
		case SPK_MODEM_MODE_C4FM_HALF_DEVIATION: dir = "voices/dmr/"
		case SPK_MODEM_MODE_NXDN: dir = "voices/dmr/"
		case SPK_MODEM_MODE_P25: dir = "voices/p25/"
	}
	if dir == "" {
		return dir
	}

	for filePath, _ := range _bindata {
		if filepath.Dir(filePath) != filepath.Clean(dir) || filepath.Ext(filePath) != ".ambe" {
			continue
		}

		fileName := filepath.Base(filePath)
		// The requested code pair is stored in the first two characters of the filename.
		var fileCodePair = string(fileName[0]) + string(fileName[1])
		if fileCodePair == codePair {
			return filePath
		}
	}
	return ""
}

func sendAMBEAnswer(udpConn *net.UDPConn, toAddr *net.UDPAddr, res *spkAMBEResponsePacket) {
	var buf bytes.Buffer
	binary.Write(&buf, binary.BigEndian, res)
	writtenBytes, err := udpConn.WriteToUDP(buf.Bytes(), toAddr)
	if writtenBytes != SPK_AMBE_RESPONSE_PACKET_SIZE || err != nil {
		log.Printf("warning: can't send udp packet to %s\n", toAddr.String())
	}

	if res.PacketType != SPK_PACKET_TYPE_RESPONSE_TERMINATOR {
		res.SeqNum++
		time.Sleep(time.Duration(res.FrameCount) * 20 * time.Millisecond)
	}
}

func sendIMBEAnswer(udpConn *net.UDPConn, toAddr *net.UDPAddr, res *spkIMBEResponsePacket) {
	var buf bytes.Buffer
	binary.Write(&buf, binary.BigEndian, res)
	writtenBytes, err := udpConn.WriteToUDP(buf.Bytes(), toAddr)
	if writtenBytes != SPK_IMBE_RESPONSE_PACKET_SIZE || err != nil {
		log.Printf("warning: can't send udp packet to %s\n", toAddr.String())
	}

	if res.PacketType != SPK_PACKET_TYPE_RESPONSE_TERMINATOR {
		res.SeqNum++
		time.Sleep(time.Duration(res.FrameCount) * 20 * time.Millisecond)
	}
}

func StartSendAnswer(udpConn *net.UDPConn, toAddr net.UDPAddr, rp *spkRequestPacket) {
	defer RequestRemove(rp.SessionID, &toAddr)

	codeStr := strings.TrimRight(string(rp.CodeStr[:]), "\x00")

	// If the client is requesting a connect announce to a Homebrew server, we try to query a BM status from
	// the server's BM HTTP API to get linked talkgroups and reflector.
	bmGetClientDataFinished := make(chan bool)
	defer close(bmGetClientDataFinished)
	bmGetClientDataRunning := false
	var bmGetClientDataResult bmClientData
	var serverData bmServerData
	if rp.ConnectorID == SPK_CONNECTOR_ID_HOMEBREW &&
		(rp.AnnounceType == SPK_ANNOUNCE_TYPE_CONNECTED || rp.AnnounceType == SPK_ANNOUNCE_TYPE_CONNECTED_BRANDMEISTER_SHORTENED ||
			rp.AnnounceType == SPK_ANNOUNCE_TYPE_CONNECTOR_STATUS) {

		serverIP := fmt.Sprintf("%d.%d.%d.%d", rp.AnnounceTypeData[0] >> 24, (rp.AnnounceTypeData[0] >> 16) & 0xff,
			(rp.AnnounceTypeData[0] >> 8) & 0xff, rp.AnnounceTypeData[0] & 0xff)

		var ok bool
		if serverData, ok = BMGetServerDataForServerIP(serverIP); ok {
			clientId := rp.AnnounceTypeData[1]

			log.Printf("getting bm client data srv:%s cid:%d", serverIP, clientId)
			bmGetClientDataRunning = true
			go BMGetClientData(clientId, &bmGetClientDataResult, bmGetClientDataFinished)
		}
	}

	var res spkResponsePacket
	switch (rp.ModemMode) {
		default:
			copy(res.AMBE.Magic[:], SPK_PACKET_MAGIC)
			res.AMBE.PacketType = SPK_PACKET_TYPE_AMBE_RESPONSE
			res.AMBE.SessionID = rp.SessionID
			break
		case SPK_MODEM_MODE_P25:
			copy(res.IMBE.Magic[:], SPK_PACKET_MAGIC)
			res.IMBE.PacketType = SPK_PACKET_TYPE_IMBE_RESPONSE
			res.IMBE.SessionID = rp.SessionID
			break
	}

	// Stepping through each code char pair.
	for codeStrPos := 0; codeStrPos < len(codeStr); codeStrPos += 2 {
		if bmGetClientDataRunning {
			select {
				case finished := <- bmGetClientDataFinished:
					if finished && codeStrPos < 4 {
						codeStr = strings.Replace(codeStr, "HBSV", BMGenerateCodeStrFromClientData(&bmGetClientDataResult, &serverData,
							rp.AnnounceType == SPK_ANNOUNCE_TYPE_CONNECTED_BRANDMEISTER_SHORTENED), 1)
						log.Printf("code str modified for %s to %s", toAddr.String(), codeStr)
						bmGetClientDataRunning = false
					}
				default:
					break
			}
		}

		if codeStrPos+2 > len(codeStr) {
			log.Println("warning: last code pair is broken")
			break
		}

		var codePair = codeStr[codeStrPos:codeStrPos+2]

		filePath := getFilePathForCodePair(rp.ModemMode, codePair)
		if filePath == "" {
			log.Printf("warning: file not found for modem mode %d code pair \"%s\", skipping\n", rp.ModemMode, codePair)
			continue
		}

		data, err := Asset(filePath)
		if err != nil {
			log.Printf("warning: can't find \"%s\", skipping\n", filePath)
			continue
		}

		log.Printf("playing %s to %s\n", filePath, toAddr.String())

		reader := bytes.NewReader(data)
		var fileFinished = false

		// Filling up frames from the file.
		for !fileFinished {
			switch (rp.ModemMode) {
				default:
					for ; res.AMBE.FrameCount < 3; res.AMBE.FrameCount++ {
						readBytes, err := reader.Read(res.AMBE.Frames[res.AMBE.FrameCount][:])
						if err != nil || readBytes != 9 {
							fileFinished = true
							break
						}
					}

					// Flushing if needed.
					if res.AMBE.FrameCount == 3 {
						sendAMBEAnswer(udpConn, &toAddr, &res.AMBE)
						res.AMBE.FrameCount = 0
					}
					break
				case SPK_MODEM_MODE_P25:
					for ; res.IMBE.FrameCount < 3; res.IMBE.FrameCount++ {
						readBytes, err := reader.Read(res.IMBE.Frames[res.IMBE.FrameCount][:])
						if err != nil || readBytes != 18 {
							fileFinished = true
							break
						}
					}

					// Flushing if needed.
					if res.IMBE.FrameCount == 3 {
						sendIMBEAnswer(udpConn, &toAddr, &res.IMBE)
						res.IMBE.FrameCount = 0
					}
					break
			}
		}
	}

	switch (rp.ModemMode) {
		default:
			res.AMBE.PacketType = SPK_PACKET_TYPE_RESPONSE_TERMINATOR
			sendAMBEAnswer(udpConn, &toAddr, &res.AMBE)
			break;
		case SPK_MODEM_MODE_P25:
			res.IMBE.PacketType = SPK_PACKET_TYPE_RESPONSE_TERMINATOR
			sendIMBEAnswer(udpConn, &toAddr, &res.IMBE)
			break
	}

	if bmGetClientDataRunning {
		<- bmGetClientDataFinished
	}

	log.Printf("playing to %s finished\n", toAddr.String())
}

func processPacket(udpConn *net.UDPConn, fromAddr *net.UDPAddr, buffer []byte, readBytes int) {
	var packetType = buffer[7]

	switch (packetType) {
		default:
			log.Printf("ignoring packet with type 0x%.2x\n", packetType)
		case SPK_PACKET_TYPE_REQUEST:
			if readBytes != SPK_REQUEST_PACKET_SIZE {
				log.Printf("ignoring packet with size %d\n", readBytes)
				return
			}

			// Reading the packet payload to our request struct.
			readBuf := bytes.NewReader(buffer)
			var rp spkRequestPacket
			err := binary.Read(readBuf, binary.BigEndian, &rp)
			if err != nil {
				log.Println("ignoring packet, binary parse error: ", err)
				return
			}

			switch (rp.ModemMode) {
				case SPK_MODEM_MODE_DMR: break
				case SPK_MODEM_MODE_DSTAR: break
				case SPK_MODEM_MODE_C4FM: break
				case SPK_MODEM_MODE_C4FM_HALF_DEVIATION: break
				case SPK_MODEM_MODE_NXDN: break
				case SPK_MODEM_MODE_P25: break
				default:
					log.Printf("ignoring packet, invalid modem mode %.2x\n", rp.ModemMode)
					return
			}

			rp.CodeStr[SPK_ANNOUNCE_DATA_MAX_LENGTH-1] = 0

			if RequestIsAdded(rp.SessionID, fromAddr) {
				//log.Printf("ignoring packet, request already under processing with sid:0x%.8x\n", rp.SessionID)
				return
			}
			RequestAdd(rp.SessionID, fromAddr)

			atStr, atdStr := decodeAnnounceTypeAndDataToStr(rp.AnnounceType, rp.AnnounceTypeData)
			log.Printf("sending \"%s\" to %s (sid:0x%.8x t:%s con:%s at:%s %s)\n",
				strings.TrimRight(string(rp.CodeStr[:]), "\x00"), fromAddr.String(), rp.SessionID, getModemModeNameStr(rp.ModemMode),
				getConnectorIdNameStr(rp.ConnectorID), atStr, atdStr);
			go StartSendAnswer(udpConn, *fromAddr, &rp)
	}
}

func main() {
	var bindIp = "0.0.0.0"
	var bindPort = 65200
	var logToFile bool

	flag.IntVar(&bindPort, "p", bindPort, "bind to port")
	flag.StringVar(&bindIp, "i", bindIp, "bind to ip address")
	flag.BoolVar(&logToFile, "f", false, "log to file spk-srv.log")
	flag.Parse()

	if logToFile {
		logFile, err := os.OpenFile("spk-srv.log", os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
		if err != nil {
			log.Println("warning: can't open spk-srv.log for writing: ", err)
		} else {
			defer logFile.Close()
			log.SetOutput(io.MultiWriter(os.Stdout, logFile))
		}
	}

	log.Printf("binding to %s:%d\n", bindIp, bindPort)
	udpConn, err := net.ListenUDP("udp4", &net.UDPAddr{ net.ParseIP(bindIp), bindPort, "" })
	if err != nil {
		log.Fatal(err)
	}
	defer udpConn.Close()

	go BMProcess()

	log.Println("starting listening loop")
	buffer := make([]byte, 128)
	for {
		readBytes, fromAddr, err := udpConn.ReadFromUDP(buffer)
		if err != nil {
			log.Fatal(err)
		}

		// Did we read at least magic + version number of bytes? Does packet magic match?
		if readBytes >= 7 && strings.Compare(SPK_PACKET_MAGIC, string(buffer[:6])) == 0 {
			log.Printf("got %d byte packet from %s\n", readBytes, fromAddr.String())

			switch (buffer[6]) {
				default:
					log.Printf("ignoring packet with version 0x%.2x\n", buffer[6])
				case 0:
					processPacket(udpConn, fromAddr, buffer, readBytes)
			}
		}
	}
}
<|endoftext|>


<commit_before>package config

import (
	"flag"
	"os"
	"os/user"
	"strconv"
)

// Serving Flags
var serveDirectoryFlag = flag.String("serve.directory", "", "The directory on the local filesystem to serve.")
var servePathFlag = flag.String("serve.path", "/", "The HTTP/HTTPS serving root path for the hosted filesystem directory.")
var configFileFlag = flag.String("config.file", "", "YAML formatted configuration file. (overrides flag values)")
var verboseFlag = flag.Bool("verbose", false, "Print out extra information.")

// HTTP Flags
var httpPortFlag *int

// HTTPS Flags
var httpsPortFlag *int
var privateKeyFilePathFlag = flag.String("https.privatekey", "rsa.pem", "Certificate to host HTTPS with.")

// HTTPS Certificate Flags
var certificateFilePathFlag = flag.String("https.certificate.path", "cert.pem", "Certificate to host HTTPS with.")
var certHostsFlag = flag.String("https.certificate.hosts", "", "Comma-separated hostnames and IPs to generate a certificate for.")
var validDurationFlag = flag.Int("https.certificate.duration", 5475, "Certificate valid duration.")
var actAsCertificateAuthorityFlag = flag.Bool("https.certificate.authority", false, "(Experimental) Generate a root cert as a Certificate Authority")
var onlyGenerateCertFlag = flag.Bool("https.certificate.onlygenerate", false, "Only generate a self-signed certificate for the server.")

// Monitoring Flags
var metricsFlag = flag.Bool("metrics.enabled", true, "Enables server metrics for monitoring.")
var metricsPathFlag = flag.String("metrics.path", "/metrics", "The URL path for exporting server metrics for Prometheus monitoring.")

func init() {
	defaultPortInt := 8080
	defaultSecurePortInt := 8443
	currentUser, err := user.Current()
	if err == nil {
		if currentUser.Uid == "0" {
			defaultPortInt = 80
			defaultSecurePortInt = 443
		}
	}

	defaultPort := os.Getenv("PORT")
	if defaultPort != "" {
		port, err := strconv.Atoi(defaultPort)
		if err == nil {
			defaultPortInt = port
		}
	}

	httpPortFlag = flag.Int("http.port", defaultPortInt, "Port to run HTTP server.")
	httpsPortFlag = flag.Int("https.port", defaultSecurePortInt, "Port to run HTTPS server.")
}

func loadFromFlags() *Config {
	return &Config{
		Verbose:           *verboseFlag,
		Directory:         *serveDirectoryFlag,
		ServePath:         *servePathFlag,
		ConfigurationFile: *configFileFlag,
		Http: Http{
			Port: *httpPortFlag,
		},
		Https: Https{
			Port: *httpsPortFlag,
			Certificate: Certificate{
				PrivateKeyFilePath:        *privateKeyFilePathFlag,
				CertificateFilePath:       *certificateFilePathFlag,
				CertificateHosts:          *certHostsFlag,
				CertificateValidDuration:  *validDurationFlag,
				ActAsCertificateAuthority: *actAsCertificateAuthorityFlag,
				OnlyGenerateCertificate:   *onlyGenerateCertFlag,
				ForceOverwrite:            *onlyGenerateCertFlag, // No flag available yet.
			},
		},
		Metrics: Metrics{
			Enabled: *metricsFlag,
			Path:    *metricsPathFlag,
		},
	}
}
<commit_msg>Fix flag paths<commit_after>package config

import (
	"flag"
	"os"
	"os/user"
	"strconv"
)

// Serving Flags
var serveDirectoryFlag = flag.String("directory", "", "The directory on the local filesystem to serve.")
var servePathFlag = flag.String("servepath", "/", "The HTTP/HTTPS serving root path for the hosted filesystem directory.")
var configFileFlag = flag.String("configfile", "", "YAML formatted configuration file. (overrides flag values)")
var verboseFlag = flag.Bool("verbose", false, "Print out extra information.")

// HTTP Flags
var httpPortFlag *int

// HTTPS Flags
var httpsPortFlag *int
var privateKeyFilePathFlag = flag.String("https.privatekey", "rsa.pem", "Certificate to host HTTPS with.")

// HTTPS Certificate Flags
var certificateFilePathFlag = flag.String("https.certificate.path", "cert.pem", "Certificate to host HTTPS with.")
var certHostsFlag = flag.String("https.certificate.hosts", "", "Comma-separated hostnames and IPs to generate a certificate for.")
var validDurationFlag = flag.Int("https.certificate.duration", 5475, "Certificate valid duration.")
var actAsCertificateAuthorityFlag = flag.Bool("https.certificate.authority", false, "(Experimental) Generate a root cert as a Certificate Authority")
var onlyGenerateCertFlag = flag.Bool("https.certificate.onlygenerate", false, "Only generate a self-signed certificate for the server.")

// Monitoring Flags
var metricsFlag = flag.Bool("metrics.enabled", true, "Enables server metrics for monitoring.")
var metricsPathFlag = flag.String("metrics.path", "/metrics", "The URL path for exporting server metrics for Prometheus monitoring.")

func init() {
	defaultPortInt := 8080
	defaultSecurePortInt := 8443
	currentUser, err := user.Current()
	if err == nil {
		if currentUser.Uid == "0" {
			defaultPortInt = 80
			defaultSecurePortInt = 443
		}
	}

	defaultPort := os.Getenv("PORT")
	if defaultPort != "" {
		port, err := strconv.Atoi(defaultPort)
		if err == nil {
			defaultPortInt = port
		}
	}

	httpPortFlag = flag.Int("http.port", defaultPortInt, "Port to run HTTP server.")
	httpsPortFlag = flag.Int("https.port", defaultSecurePortInt, "Port to run HTTPS server.")
}

func loadFromFlags() *Config {
	return &Config{
		Verbose:           *verboseFlag,
		Directory:         *serveDirectoryFlag,
		ServePath:         *servePathFlag,
		ConfigurationFile: *configFileFlag,
		Http: Http{
			Port: *httpPortFlag,
		},
		Https: Https{
			Port: *httpsPortFlag,
			Certificate: Certificate{
				PrivateKeyFilePath:        *privateKeyFilePathFlag,
				CertificateFilePath:       *certificateFilePathFlag,
				CertificateHosts:          *certHostsFlag,
				CertificateValidDuration:  *validDurationFlag,
				ActAsCertificateAuthority: *actAsCertificateAuthorityFlag,
				OnlyGenerateCertificate:   *onlyGenerateCertFlag,
				ForceOverwrite:            *onlyGenerateCertFlag, // No flag available yet.
			},
		},
		Metrics: Metrics{
			Enabled: *metricsFlag,
			Path:    *metricsPathFlag,
		},
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	"github.com/hyperledger/fabric/core/chaincode/shim"
	"github.com/op/go-logging"
)

type SimpleChaincode struct {
}

var myLogger = logging.MustGetLogger("asset_mgm")

func main() {
	fmt.Println("started logging in main()")
	err := shim.Start(new(SimpleChaincode))
	if err != nil {
		fmt.Printf("failed in function main(), error is %v", err)
	}
}

func (t *SimpleChaincode) Init(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	fmt.Println("started logging in Init()")
	myLogger.Debug("init chaincode")
	return nil, nil
}

func (t *SimpleChaincode) Invoke(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	fmt.Println("started logging in Invoke()")
	myLogger.Debug("start Invoke")
	return nil, nil
}

func (t *SimpleChaincode) Query(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	fmt.Println("started logging in Query()")
	myLogger.Debug("started logging in Query()")
	return nil, nil
}
<commit_msg>remove op/logging code<commit_after>package main

import (
	"fmt"

	"github.com/hyperledger/fabric/core/chaincode/shim"
)

type SimpleChaincode struct {
}

func main() {
	fmt.Println("started logging in main()")
	err := shim.Start(new(SimpleChaincode))
	if err != nil {
		fmt.Printf("failed in function main(), error is %v", err)
	}
}

func (t *SimpleChaincode) Init(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	fmt.Println("started logging in Init()")

	return nil, nil
}

func (t *SimpleChaincode) Invoke(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	fmt.Println("started logging in Invoke()")

	return nil, nil
}

func (t *SimpleChaincode) Query(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	fmt.Println("started logging in Query()")

	return nil, nil
}
<|endoftext|>


<commit_before>package manta

import (
	"sort"

	"github.com/dotabuff/manta/dota"
)

// A message that has been read from an outerMessage but not yet processed.
type pendingMessage struct {
	tick uint32
	t    int32
	buf  []byte
}

// Calculates the priority of the message. Lower is more important.
func (m *pendingMessage) priority() int {
	switch m.t {
	case
		// These messages provide context needed for the rest of the tick
		// and should have the highest priority.
		int32(dota.NET_Messages_net_Tick),
		int32(dota.SVC_Messages_svc_CreateStringTable),
		int32(dota.SVC_Messages_svc_UpdateStringTable),
		int32(dota.NET_Messages_net_SpawnGroup_Load):
		return -10

	case
		// These messages benefit from having context but may also need to
		// provide context in terms of delta updates.
		int32(dota.SVC_Messages_svc_PacketEntities):
		return 5

	case
		// These messages benefit from having as much context as possible and
		// should have the lowest priority.
		int32(dota.EBaseGameEvents_GE_Source1LegacyGameEvent):
		return 10
	}

	return 0
}

// Provides a sortable structure for storing messages in the same packet.
type pendingMessages []*pendingMessage

func (ms pendingMessages) Len() int      { return len(ms) }
func (ms pendingMessages) Swap(i, j int) { ms[i], ms[j] = ms[j], ms[i] }
func (ms pendingMessages) Less(i, j int) bool {
	if ms[i].tick > ms[j].tick {
		return false
	}
	if ms[i].tick < ms[j].tick {
		return true
	}
	return ms[i].priority() < ms[j].priority()
}

// Internal parser for callback OnCDemoPacket, responsible for extracting
// multiple inner packets from a single CDemoPacket. This is the main structure
// that contains all other data types in the demo file.
func (p *Parser) onCDemoPacket(m *dota.CDemoPacket) error {
	// Create a slice to store pending mesages. Messages are read first as
	// pending messages then sorted before dispatch.
	ms := make(pendingMessages, 0, 10)

	// Read all messages from the buffer. Messages are packed serially as
	// {type, size, data}. We keep reading until until less than a byte remains.
	r := newReader(m.GetData())
	for r.remBytes() > 0 {
		t := int32(r.readUBitVar())
		size := r.readVarUint32()
		buf := r.readBytes(size)
		ms = append(ms, &pendingMessage{p.Tick, t, buf})
	}

	// Sort messages to ensure dependencies are met. For example, we need to
	// process string tables before game events that may reference them.
	sort.Sort(ms)

	// Dispatch messages in order, returning on handler error.
	for _, m := range ms {
		if err := p.Callbacks.callByPacketType(m.t, m.buf); err != nil {
			return err
		}
	}

	return nil
}

// Internal parser for callback OnCDemoFullPacket.
func (p *Parser) onCDemoFullPacket(m *dota.CDemoFullPacket) error {
	// Per Valve docs, parse the CDemoStringTables first.
	if m.StringTable != nil {
		if err := p.onCDemoStringTables(m.GetStringTable()); err != nil {
			return err
		}
	}

	// Then the CDemoPacket.
	if m.Packet != nil {
		if err := p.onCDemoPacket(m.GetPacket()); err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>tweak alloc<commit_after>package manta

import (
	"sort"

	"github.com/dotabuff/manta/dota"
)

// A message that has been read from an outerMessage but not yet processed.
type pendingMessage struct {
	tick uint32
	t    int32
	buf  []byte
}

// Calculates the priority of the message. Lower is more important.
func (m *pendingMessage) priority() int {
	switch m.t {
	case
		// These messages provide context needed for the rest of the tick
		// and should have the highest priority.
		int32(dota.NET_Messages_net_Tick),
		int32(dota.SVC_Messages_svc_CreateStringTable),
		int32(dota.SVC_Messages_svc_UpdateStringTable),
		int32(dota.NET_Messages_net_SpawnGroup_Load):
		return -10

	case
		// These messages benefit from having context but may also need to
		// provide context in terms of delta updates.
		int32(dota.SVC_Messages_svc_PacketEntities):
		return 5

	case
		// These messages benefit from having as much context as possible and
		// should have the lowest priority.
		int32(dota.EBaseGameEvents_GE_Source1LegacyGameEvent):
		return 10
	}

	return 0
}

// Provides a sortable structure for storing messages in the same packet.
type pendingMessages []*pendingMessage

func (ms pendingMessages) Len() int      { return len(ms) }
func (ms pendingMessages) Swap(i, j int) { ms[i], ms[j] = ms[j], ms[i] }
func (ms pendingMessages) Less(i, j int) bool {
	if ms[i].tick > ms[j].tick {
		return false
	}
	if ms[i].tick < ms[j].tick {
		return true
	}
	return ms[i].priority() < ms[j].priority()
}

// Internal parser for callback OnCDemoPacket, responsible for extracting
// multiple inner packets from a single CDemoPacket. This is the main structure
// that contains all other data types in the demo file.
func (p *Parser) onCDemoPacket(m *dota.CDemoPacket) error {
	// Create a slice to store pending mesages. Messages are read first as
	// pending messages then sorted before dispatch.
	ms := make(pendingMessages, 0, 2)

	// Read all messages from the buffer. Messages are packed serially as
	// {type, size, data}. We keep reading until until less than a byte remains.
	r := newReader(m.GetData())
	for r.remBytes() > 0 {
		t := int32(r.readUBitVar())
		size := r.readVarUint32()
		buf := r.readBytes(size)
		ms = append(ms, &pendingMessage{p.Tick, t, buf})
	}

	// Sort messages to ensure dependencies are met. For example, we need to
	// process string tables before game events that may reference them.
	sort.Sort(ms)

	// Dispatch messages in order, returning on handler error.
	for _, m := range ms {
		if err := p.Callbacks.callByPacketType(m.t, m.buf); err != nil {
			return err
		}
	}

	return nil
}

// Internal parser for callback OnCDemoFullPacket.
func (p *Parser) onCDemoFullPacket(m *dota.CDemoFullPacket) error {
	// Per Valve docs, parse the CDemoStringTables first.
	if m.StringTable != nil {
		if err := p.onCDemoStringTables(m.GetStringTable()); err != nil {
			return err
		}
	}

	// Then the CDemoPacket.
	if m.Packet != nil {
		if err := p.onCDemoPacket(m.GetPacket()); err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>