<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package namespace defines an interface for resolving and managing names.
package namespace

import (
	"time"

	"v.io/v23/context"
	"v.io/v23/naming"
	"v.io/v23/security/access"
)

// T is the interface for namespace operations; it is the client side library
// for the MountTable service
type T interface {
	// Mount the server object address under the object name, expiring after
	// the ttl. ttl of zero implies an implementation-specific high value
	// (essentially, forever).
	Mount(ctx *context.T, name, server string, ttl time.Duration, opts ...naming.NamespaceOpt) error

	// Unmount the server object address from the object name, or if server
	// is empty, unmount all server OAs from the object name.
	Unmount(ctx *context.T, name, server string, opts ...naming.NamespaceOpt) error

	// Delete the name from a mount table.  If the name has any children in its mount table,
	// it (and its chidren) will only be removed if deleteSubtree is true.
	Delete(ctx *context.T, name string, deleteSubtree bool, opts ...naming.NamespaceOpt) error

	// Resolve the object name into its mounted servers.
	Resolve(ctx *context.T, name string, opts ...naming.NamespaceOpt) (entry *naming.MountEntry, err error)

	// ResolveToMountTable resolves the object name into the mounttables
	// directly responsible for the name.
	ResolveToMountTable(ctx *context.T, name string, opts ...naming.NamespaceOpt) (entry *naming.MountEntry, err error)

	// FlushCacheEntry flushes resolution information cached for the name.  If
	// anything was flushed it returns true.
	FlushCacheEntry(name string) bool

	// CacheCtl sets controls and returns the current control values.
	CacheCtl(ctls ...naming.CacheCtl) []naming.CacheCtl

	// Glob returns MountEntry's whose name matches the pattern and GlobError's
	// for any piece of the space that can't be traversed.
	//
	// Two special patterns:
	//   prefix/... means all names below prefix.
	//   prefix/*** is like prefix/... but doesn't traverse into non-mounttable servers.
	//
	// Example:
	//	rc, err := ns.Glob(ctx, pattern)
	//	if err != nil {
	//		boom(t, "Glob(%s): %s", pattern, err)
	//	}
	//	for s := range rc {
	//		switch v := s.(type) {
	//		case *naming.MountEntry:
	//			fmt.Printf("%s: %v\n", v.Name, v.Servers)
	//		case *naming.GlobError:
	//			fmt.Fprintf(stderr, "%s can't be traversed: %s\n", v.Name, v.Error)
	//		}
	//	}
	Glob(ctx *context.T, pattern string, opts ...naming.NamespaceOpt) (chan interface{}, error)

	// SetRoots sets the roots that the local Namespace is
	// relative to. All relative names passed to the methods above
	// will be interpreted as relative to these roots. The roots
	// will be tried in the order that they are specified in the parameter
	// list for SetRoots. Calling SetRoots with no arguments will clear the
	// currently configured set of roots.
	SetRoots(roots ...string) error

	// Roots returns the currently configured roots. An empty slice is
	// returned if no roots are configured.
	Roots() []string

	// SetPermissions sets the AccessList in a node in a mount table.
	SetPermissions(ctx *context.T, name string, acl access.Permissions, version string, opts ...naming.NamespaceOpt) error

	// GetPermissions returns the AccessList in a node in a mount table.
	GetPermissions(ctx *context.T, name string, opts ...naming.NamespaceOpt) (acl access.Permissions, version string, err error)
}
<commit_msg>namespace/model: change Glob to return a chan of naming.GlobReply<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package namespace defines an interface for resolving and managing names.
package namespace

import (
	"time"

	"v.io/v23/context"
	"v.io/v23/naming"
	"v.io/v23/security/access"
)

// T is the interface for namespace operations; it is the client side library
// for the MountTable service
type T interface {
	// Mount the server object address under the object name, expiring after
	// the ttl. ttl of zero implies an implementation-specific high value
	// (essentially, forever).
	Mount(ctx *context.T, name, server string, ttl time.Duration, opts ...naming.NamespaceOpt) error

	// Unmount the server object address from the object name, or if server
	// is empty, unmount all server OAs from the object name.
	Unmount(ctx *context.T, name, server string, opts ...naming.NamespaceOpt) error

	// Delete the name from a mount table.  If the name has any children in its mount table,
	// it (and its chidren) will only be removed if deleteSubtree is true.
	Delete(ctx *context.T, name string, deleteSubtree bool, opts ...naming.NamespaceOpt) error

	// Resolve the object name into its mounted servers.
	Resolve(ctx *context.T, name string, opts ...naming.NamespaceOpt) (entry *naming.MountEntry, err error)

	// ResolveToMountTable resolves the object name into the mounttables
	// directly responsible for the name.
	ResolveToMountTable(ctx *context.T, name string, opts ...naming.NamespaceOpt) (entry *naming.MountEntry, err error)

	// FlushCacheEntry flushes resolution information cached for the name.  If
	// anything was flushed it returns true.
	FlushCacheEntry(name string) bool

	// CacheCtl sets controls and returns the current control values.
	CacheCtl(ctls ...naming.CacheCtl) []naming.CacheCtl

	// Glob returns MountEntry's whose name matches the pattern and GlobError's
	// for any piece of the space that can't be traversed.
	//
	// Two special patterns:
	//   prefix/... means all names below prefix.
	//   prefix/*** is like prefix/... but doesn't traverse into non-mounttable servers.
	//
	// Example:
	//	rc, err := ns.Glob(ctx, pattern)
	//	if err != nil {
	//		boom(t, "Glob(%s): %s", pattern, err)
	//	}
	//	for s := range rc {
	//		switch v := s.(type) {
	//		case *naming.MountEntry:
	//			fmt.Printf("%s: %v\n", v.Name, v.Servers)
	//		case *naming.GlobError:
	//			fmt.Fprintf(stderr, "%s can't be traversed: %s\n", v.Name, v.Error)
	//		}
	//	}
	Glob(ctx *context.T, pattern string, opts ...naming.NamespaceOpt) (chan naming.GlobReply, error)

	// SetRoots sets the roots that the local Namespace is
	// relative to. All relative names passed to the methods above
	// will be interpreted as relative to these roots. The roots
	// will be tried in the order that they are specified in the parameter
	// list for SetRoots. Calling SetRoots with no arguments will clear the
	// currently configured set of roots.
	SetRoots(roots ...string) error

	// Roots returns the currently configured roots. An empty slice is
	// returned if no roots are configured.
	Roots() []string

	// SetPermissions sets the AccessList in a node in a mount table.
	SetPermissions(ctx *context.T, name string, acl access.Permissions, version string, opts ...naming.NamespaceOpt) error

	// GetPermissions returns the AccessList in a node in a mount table.
	GetPermissions(ctx *context.T, name string, opts ...naming.NamespaceOpt) (acl access.Permissions, version string, err error)
}
<|endoftext|>


<commit_before>package lz4_test

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"reflect"
	"strings"
	"testing"

	"github.com/pierrec/lz4/v4"
	"github.com/pierrec/lz4/v4/internal/lz4block"
)

func TestWriter(t *testing.T) {
	goldenFiles := []string{
		"testdata/e.txt",
		"testdata/gettysburg.txt",
		"testdata/Mark.Twain-Tom.Sawyer.txt",
		"testdata/Mark.Twain-Tom.Sawyer_long.txt",
		"testdata/pg1661.txt",
		"testdata/pi.txt",
		"testdata/random.data",
		"testdata/repeat.txt",
		"testdata/issue102.data",
	}

	for _, fname := range goldenFiles {
		for _, option := range []lz4.Option{
			lz4.ConcurrencyOption(1),
			lz4.BlockChecksumOption(true),
			lz4.SizeOption(123),
			lz4.ConcurrencyOption(4),
		} {
			label := fmt.Sprintf("%s/%s", fname, option)
			t.Run(label, func(t *testing.T) {
				fname := fname
				t.Parallel()

				raw, err := ioutil.ReadFile(fname)
				if err != nil {
					t.Fatal(err)
				}
				r := bytes.NewReader(raw)

				// Compress.
				zout := new(bytes.Buffer)
				zw := lz4.NewWriter(zout)
				if err := zw.Apply(option, lz4.CompressionLevelOption(lz4.Level1)); err != nil {
					t.Fatal(err)
				}
				_, err = io.Copy(zw, r)
				if err != nil {
					t.Fatal(err)
				}
				err = zw.Close()
				if err != nil {
					t.Fatal(err)
				}

				// Uncompress.
				out := new(bytes.Buffer)
				zr := lz4.NewReader(zout)
				n, err := io.Copy(out, zr)
				if err != nil {
					t.Fatal(err)
				}

				// The uncompressed data must be the same as the initial input.
				if got, want := int(n), len(raw); got != want {
					t.Errorf("invalid sizes: got %d; want %d", got, want)
				}

				if got, want := out.Bytes(), raw; !reflect.DeepEqual(got, want) {
					t.Fatal("uncompressed data does not match original")
				}

				if strings.Contains(option.String(), "SizeOption") {
					if got, want := zr.Size(), 123; got != want {
						t.Errorf("invalid sizes: got %d; want %d", got, want)
					}
				}
			})
		}
	}
}

func TestWriter_Reset(t *testing.T) {
	data := pg1661
	buf := new(bytes.Buffer)
	src := bytes.NewReader(data)
	zw := lz4.NewWriter(buf)

	// Partial write.
	_, _ = io.CopyN(zw, src, int64(len(data))/2)

	buf.Reset()
	src.Reset(data)
	zw.Reset(buf)
	zw.Reset(buf)
	// Another time to maybe trigger some edge case.
	if _, err := io.Copy(zw, src); err != nil {
		t.Fatal(err)
	}
	if err := zw.Close(); err != nil {
		t.Fatal(err)
	}
	// Cannot compare compressed outputs directly, so compare the uncompressed output.
	out := new(bytes.Buffer)
	if _, err := io.Copy(out, lz4.NewReader(buf)); err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(out.Bytes(), data) {
		t.Fatal("result does not match original")
	}
}

func TestIssue41(t *testing.T) {
	r, w := io.Pipe()
	zw := lz4.NewWriter(w)
	zr := lz4.NewReader(r)

	data := "x"
	go func() {
		_, _ = fmt.Fprint(zw, data)
		_ = zw.Close()
		_ = w.Close()
	}()
	var buf bytes.Buffer
	_, _ = buf.ReadFrom(zr)
	if got, want := buf.String(), data; got != want {
		t.Fatal("uncompressed data does not match original")
	}
}

func TestIssue43(t *testing.T) {
	r, w := io.Pipe()
	go func() {
		defer w.Close()

		f, err := os.Open("testdata/issue43.data")
		if err != nil {
			t.Fatal(err)
		}
		defer f.Close()

		zw := lz4.NewWriter(w)
		defer zw.Close()

		_, err = io.Copy(zw, f)
		if err != nil {
			t.Fatal(err)
		}
	}()
	_, err := io.Copy(ioutil.Discard, lz4.NewReader(r))
	if err != nil {
		t.Fatal(err)
	}
}

func TestIssue51(t *testing.T) {
	data, err := ioutil.ReadFile("testdata/issue51.data")
	if err != nil {
		t.Fatal(err)
	}

	zbuf := make([]byte, 8192)

	n, err := lz4block.CompressBlock(data, zbuf)
	if err != nil {
		t.Fatal(err)
	}
	zbuf = zbuf[:n]

	buf := make([]byte, 8192)
	n, err = lz4block.UncompressBlock(zbuf, buf)
	if err != nil {
		t.Fatal(err)
	}
	buf = buf[:n]
	if !bytes.Equal(data, buf) {
		t.Fatal("processed data does not match input")
	}
}

func TestIssue71(t *testing.T) {
	for _, tc := range []string{
		"abc",               // < mfLimit
		"abcdefghijklmnopq", // > mfLimit
	} {
		t.Run(tc, func(t *testing.T) {
			src := []byte(tc)
			bound := lz4block.CompressBlockBound(len(tc))

			// Small buffer.
			zSmall := make([]byte, bound-1)
			n, err := lz4block.CompressBlock(src, zSmall)
			if err != nil {
				t.Fatal(err)
			}
			if n != 0 {
				t.Fatal("should be incompressible")
			}

			// Large enough buffer.
			zLarge := make([]byte, bound)
			n, err = lz4block.CompressBlock(src, zLarge)
			if err != nil {
				t.Fatal(err)
			}
			if n == 0 {
				t.Fatal("should be compressible")
			}
		})
	}
}

func TestWriterLegacy(t *testing.T) {
	goldenFiles := []string{
		"testdata/vmlinux_LZ4_19377",
		"testdata/bzImage_lz4_isolated",
	}

	for _, fname := range goldenFiles {
		t.Run(fname, func(t *testing.T) {
			fname := fname
			t.Parallel()

			src, err := ioutil.ReadFile(fname)
			if err != nil {
				t.Fatal(err)
			}

			out := new(bytes.Buffer)
			zw := lz4.NewWriter(out)
			if err := zw.Apply(lz4.LegacyOption(true), lz4.CompressionLevelOption(lz4.Fast)); err != nil {
				t.Fatal(err)
			}
			if _, err := io.Copy(zw, bytes.NewReader(src)); err != nil {
				t.Fatal(err)
			}
			if err := zw.Close(); err != nil {
				t.Fatal(err)
			}

			out2 := new(bytes.Buffer)
			zr := lz4.NewReader(out)
			if _, err := io.Copy(out2, zr); err != nil {
				t.Fatal(err)
			}
			if !bytes.Equal(out2.Bytes(), src) {
				t.Fatal("uncompressed compressed output different from source")
			}
		})
	}
}
<commit_msg>Fix go vet warnings<commit_after>package lz4_test

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"reflect"
	"strings"
	"testing"

	"github.com/pierrec/lz4/v4"
	"github.com/pierrec/lz4/v4/internal/lz4block"
)

func TestWriter(t *testing.T) {
	goldenFiles := []string{
		"testdata/e.txt",
		"testdata/gettysburg.txt",
		"testdata/Mark.Twain-Tom.Sawyer.txt",
		"testdata/Mark.Twain-Tom.Sawyer_long.txt",
		"testdata/pg1661.txt",
		"testdata/pi.txt",
		"testdata/random.data",
		"testdata/repeat.txt",
		"testdata/issue102.data",
	}

	for _, fname := range goldenFiles {
		for _, option := range []lz4.Option{
			lz4.ConcurrencyOption(1),
			lz4.BlockChecksumOption(true),
			lz4.SizeOption(123),
			lz4.ConcurrencyOption(4),
		} {
			label := fmt.Sprintf("%s/%s", fname, option)
			t.Run(label, func(t *testing.T) {
				fname := fname
				t.Parallel()

				raw, err := ioutil.ReadFile(fname)
				if err != nil {
					t.Fatal(err)
				}
				r := bytes.NewReader(raw)

				// Compress.
				zout := new(bytes.Buffer)
				zw := lz4.NewWriter(zout)
				if err := zw.Apply(option, lz4.CompressionLevelOption(lz4.Level1)); err != nil {
					t.Fatal(err)
				}
				_, err = io.Copy(zw, r)
				if err != nil {
					t.Fatal(err)
				}
				err = zw.Close()
				if err != nil {
					t.Fatal(err)
				}

				// Uncompress.
				out := new(bytes.Buffer)
				zr := lz4.NewReader(zout)
				n, err := io.Copy(out, zr)
				if err != nil {
					t.Fatal(err)
				}

				// The uncompressed data must be the same as the initial input.
				if got, want := int(n), len(raw); got != want {
					t.Errorf("invalid sizes: got %d; want %d", got, want)
				}

				if got, want := out.Bytes(), raw; !reflect.DeepEqual(got, want) {
					t.Fatal("uncompressed data does not match original")
				}

				if strings.Contains(option.String(), "SizeOption") {
					if got, want := zr.Size(), 123; got != want {
						t.Errorf("invalid sizes: got %d; want %d", got, want)
					}
				}
			})
		}
	}
}

func TestWriter_Reset(t *testing.T) {
	data := pg1661
	buf := new(bytes.Buffer)
	src := bytes.NewReader(data)
	zw := lz4.NewWriter(buf)

	// Partial write.
	_, _ = io.CopyN(zw, src, int64(len(data))/2)

	buf.Reset()
	src.Reset(data)
	zw.Reset(buf)
	zw.Reset(buf)
	// Another time to maybe trigger some edge case.
	if _, err := io.Copy(zw, src); err != nil {
		t.Fatal(err)
	}
	if err := zw.Close(); err != nil {
		t.Fatal(err)
	}
	// Cannot compare compressed outputs directly, so compare the uncompressed output.
	out := new(bytes.Buffer)
	if _, err := io.Copy(out, lz4.NewReader(buf)); err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(out.Bytes(), data) {
		t.Fatal("result does not match original")
	}
}

func TestIssue41(t *testing.T) {
	r, w := io.Pipe()
	zw := lz4.NewWriter(w)
	zr := lz4.NewReader(r)

	data := "x"
	go func() {
		_, _ = fmt.Fprint(zw, data)
		_ = zw.Close()
		_ = w.Close()
	}()
	var buf bytes.Buffer
	_, _ = buf.ReadFrom(zr)
	if got, want := buf.String(), data; got != want {
		t.Fatal("uncompressed data does not match original")
	}
}

func TestIssue43(t *testing.T) {
	r, w := io.Pipe()
	go func() {
		defer w.Close()

		f, err := os.Open("testdata/issue43.data")
		if err != nil {
			panic(err)
		}
		defer f.Close()

		zw := lz4.NewWriter(w)
		defer zw.Close()

		_, err = io.Copy(zw, f)
		if err != nil {
			panic(err)
		}
	}()
	_, err := io.Copy(ioutil.Discard, lz4.NewReader(r))
	if err != nil {
		t.Fatal(err)
	}
}

func TestIssue51(t *testing.T) {
	data, err := ioutil.ReadFile("testdata/issue51.data")
	if err != nil {
		t.Fatal(err)
	}

	zbuf := make([]byte, 8192)

	n, err := lz4block.CompressBlock(data, zbuf)
	if err != nil {
		t.Fatal(err)
	}
	zbuf = zbuf[:n]

	buf := make([]byte, 8192)
	n, err = lz4block.UncompressBlock(zbuf, buf)
	if err != nil {
		t.Fatal(err)
	}
	buf = buf[:n]
	if !bytes.Equal(data, buf) {
		t.Fatal("processed data does not match input")
	}
}

func TestIssue71(t *testing.T) {
	for _, tc := range []string{
		"abc",               // < mfLimit
		"abcdefghijklmnopq", // > mfLimit
	} {
		t.Run(tc, func(t *testing.T) {
			src := []byte(tc)
			bound := lz4block.CompressBlockBound(len(tc))

			// Small buffer.
			zSmall := make([]byte, bound-1)
			n, err := lz4block.CompressBlock(src, zSmall)
			if err != nil {
				t.Fatal(err)
			}
			if n != 0 {
				t.Fatal("should be incompressible")
			}

			// Large enough buffer.
			zLarge := make([]byte, bound)
			n, err = lz4block.CompressBlock(src, zLarge)
			if err != nil {
				t.Fatal(err)
			}
			if n == 0 {
				t.Fatal("should be compressible")
			}
		})
	}
}

func TestWriterLegacy(t *testing.T) {
	goldenFiles := []string{
		"testdata/vmlinux_LZ4_19377",
		"testdata/bzImage_lz4_isolated",
	}

	for _, fname := range goldenFiles {
		t.Run(fname, func(t *testing.T) {
			fname := fname
			t.Parallel()

			src, err := ioutil.ReadFile(fname)
			if err != nil {
				t.Fatal(err)
			}

			out := new(bytes.Buffer)
			zw := lz4.NewWriter(out)
			if err := zw.Apply(lz4.LegacyOption(true), lz4.CompressionLevelOption(lz4.Fast)); err != nil {
				t.Fatal(err)
			}
			if _, err := io.Copy(zw, bytes.NewReader(src)); err != nil {
				t.Fatal(err)
			}
			if err := zw.Close(); err != nil {
				t.Fatal(err)
			}

			out2 := new(bytes.Buffer)
			zr := lz4.NewReader(out)
			if _, err := io.Copy(out2, zr); err != nil {
				t.Fatal(err)
			}
			if !bytes.Equal(out2.Bytes(), src) {
				t.Fatal("uncompressed compressed output different from source")
			}
		})
	}
}
<|endoftext|>


<commit_before>// This is a simple experiment to provide a bridge between
// a websocket capable client and a old http backend

package main

import (
	"net/http"
	"code.google.com/p/go.net/websocket"
)

type signal struct{}

type method func() (interface{}, error)

type brokerMessage struct {
	request method
	completed chan signal
	err error
	data interface{}
}

// Hold all websocket sessions that are running at this moment
type SessionBroker struct {
	list []*Session
	done chan signal
}

func NewSessionBroker() *SessionBroker {
	s := &SessionBroker{
		list: make([]*Session, 0),
	}
	go s.loop()
}

func (s *SessionBroker) CreateSession(cli websocket.Conn) (*Session, error) {
}

func (s *SessionBroker) createMessage(m method) *brokerMessage {
	ret := brokerMessage {
		request: m,
		completed: make(chan signal, 0),
		err: nil,
		data: nil,
	}
	return ret
}

func (s *SessionBroker) loop() {
loop:
	for {
		select {
		case <-s.done:
			break loop
		}
	}
}

// Session between the websocket and the backend
type Session struct {
	// Client id
	Cid uint64

	// Backend base url
	Backend *url.URL
}

var broker = NewSessionBroker()

func handleWebSocket(conn websocket.Conn) {
	session := broker.CreateSession(conn)
loop:
	for {
		select {
		case data := <-session.DataFromBackend:
			session.WriteClient(data)
		case data := <-session.DataFromClient:
			session.WriteBackend(data)
		case <-session.Done:
			defer session.CloseClient()
			defer session.CloseBackend()
			break loop
		}
	}
}

func main() {
	flag.Parse()
	if *help {
		flag.Usage()
		os.Exit(1)
	}
	http.Handle("/ws", websocket.Handler(handleWebSocket))

	err := http.ListenAndServe(*port, nil)
	if err != nil {
		log.Fatalf("Error: %v", err)
	}
}

<commit_msg>rename SessionBroker to Broker<commit_after>// This is a simple experiment to provide a bridge between
// a websocket capable client and a old http backend

package main

import (
	"net/http"
	"code.google.com/p/go.net/websocket"
)

type signal struct{}

type method func(*Broker) (interface{}, error)

type brokerMessage struct {
	request method
	completed chan signal
	err error
	data interface{}
}

// Hold all websocket sessions that are running at this moment
type Broker struct {
	list []*Session
	done chan signal
}

func NewBroker() *Broker {
	s := &Broker{
		list: make([]*Session, 0),
	}
	go s.loop()
}

func (s *Broker) CreateSession(cli websocket.Conn) (*Session, error) {
}

func (s *Broker) createMessage(m method) *brokerMessage {
	ret := brokerMessage {
		request: m,
		completed: make(chan signal, 0),
		err: nil,
		data: nil,
	}
	return ret
}

func (s *Broker) loop() {
loop:
	for {
		select {
		case <-s.done:
			break loop
		}
	}
}

// Session between the websocket and the backend
type Session struct {
	// Client id
	Cid uint64

	// Backend base url
	Backend *url.URL
}

var broker = NewBroker()

func handleWebSocket(conn websocket.Conn) {
	session := broker.CreateSession(conn)
loop:
	for {
		select {
		case data := <-session.DataFromBackend:
			session.WriteClient(data)
		case data := <-session.DataFromClient:
			session.WriteBackend(data)
		case <-session.Done:
			defer session.CloseClient()
			defer session.CloseBackend()
			break loop
		}
	}
}

func main() {
	flag.Parse()
	if *help {
		flag.Usage()
		os.Exit(1)
	}
	http.Handle("/ws", websocket.Handler(handleWebSocket))

	err := http.ListenAndServe(*port, nil)
	if err != nil {
		log.Fatalf("Error: %v", err)
	}
}

<|endoftext|>


<commit_before>package main

import (
	"context"
	"flag"
	"fmt"
	"log"
	"net/http"
	"runtime"
	"sync/atomic"
	"time"

	"github.com/lomik/graphite-clickhouse/config"
	"github.com/lomik/graphite-clickhouse/find"
	"github.com/lomik/graphite-clickhouse/render"
	"github.com/lomik/graphite-clickhouse/tagger"
	"github.com/lomik/zapwriter"
	"go.uber.org/zap"

	_ "net/http/pprof"
)

// Version of graphite-clickhouse
const Version = "0.6.1"

type LogResponseWriter struct {
	http.ResponseWriter
	status int
}

func (w *LogResponseWriter) WriteHeader(status int) {
	w.status = status
	w.ResponseWriter.WriteHeader(status)
}

func (w *LogResponseWriter) Status() int {
	if w.status == 0 {
		return http.StatusOK
	}
	return w.status
}

func WrapResponseWriter(w http.ResponseWriter) *LogResponseWriter {
	if wrapped, ok := w.(*LogResponseWriter); ok {
		return wrapped
	}
	return &LogResponseWriter{ResponseWriter: w}
}

func Handler(logger *zap.Logger, handler http.Handler) http.Handler {
	var requestCounter uint32
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		writer := WrapResponseWriter(w)

		requestID := r.Header.Get("X-Request-Id")
		if requestID == "" {
			requestID = fmt.Sprintf("%d", atomic.AddUint32(&requestCounter, 1))
		}

		logger := logger.With(zap.String("request_id", requestID))

		r = r.WithContext(context.WithValue(r.Context(), "logger", logger))

		start := time.Now()
		handler.ServeHTTP(w, r)
		d := time.Since(start)
		logger.Info("access",
			zap.Duration("time", d),
			zap.String("method", r.Method),
			zap.String("url", r.URL.String()),
			zap.String("peer", r.RemoteAddr),
			zap.Int("status", writer.Status()),
		)
	})
}

func main() {
	var err error

	/* CONFIG start */

	configFile := flag.String("config", "/etc/graphite-clickhouse/graphite-clickhouse.conf", "Filename of config")
	printDefaultConfig := flag.Bool("config-print-default", false, "Print default config")
	checkConfig := flag.Bool("check-config", false, "Check config and exit")
	tags := flag.Bool("tags", false, "Build tags table")

	printVersion := flag.Bool("version", false, "Print version")

	flag.Parse()

	if *printVersion {
		fmt.Print(Version)
		return
	}

	if *printDefaultConfig {
		if err = config.PrintDefaultConfig(); err != nil {
			log.Fatal(err)
		}
		return
	}

	cfg, err := config.ReadConfig(*configFile)
	if err != nil {
		log.Fatal(err)
	}

	// config parsed successfully. Exit in check-only mode
	if *checkConfig {
		return
	}

	if err = zapwriter.ApplyConfig(cfg.Logging); err != nil {
		log.Fatal(err)
	}

	runtime.GOMAXPROCS(cfg.Common.MaxCPU)

	/* CONFIG end */

	/* CONSOLE COMMANDS start */
	if *tags {
		if err := tagger.Make(cfg); err != nil {
			log.Fatal(err)
		}
		return
	}

	/* CONSOLE COMMANDS end */

	http.Handle("/metrics/find/", Handler(zapwriter.Default(), find.NewHandler(cfg)))
	http.Handle("/render/", Handler(zapwriter.Default(), render.NewHandler(cfg)))

	http.Handle("/", Handler(zapwriter.Default(), http.HandlerFunc(http.NotFound)))

	log.Fatal(http.ListenAndServe(cfg.Common.Listen, nil))
}
<commit_msg>version 0.6.2<commit_after>package main

import (
	"context"
	"flag"
	"fmt"
	"log"
	"net/http"
	"runtime"
	"sync/atomic"
	"time"

	"github.com/lomik/graphite-clickhouse/config"
	"github.com/lomik/graphite-clickhouse/find"
	"github.com/lomik/graphite-clickhouse/render"
	"github.com/lomik/graphite-clickhouse/tagger"
	"github.com/lomik/zapwriter"
	"go.uber.org/zap"

	_ "net/http/pprof"
)

// Version of graphite-clickhouse
const Version = "0.6.2"

type LogResponseWriter struct {
	http.ResponseWriter
	status int
}

func (w *LogResponseWriter) WriteHeader(status int) {
	w.status = status
	w.ResponseWriter.WriteHeader(status)
}

func (w *LogResponseWriter) Status() int {
	if w.status == 0 {
		return http.StatusOK
	}
	return w.status
}

func WrapResponseWriter(w http.ResponseWriter) *LogResponseWriter {
	if wrapped, ok := w.(*LogResponseWriter); ok {
		return wrapped
	}
	return &LogResponseWriter{ResponseWriter: w}
}

func Handler(logger *zap.Logger, handler http.Handler) http.Handler {
	var requestCounter uint32
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		writer := WrapResponseWriter(w)

		requestID := r.Header.Get("X-Request-Id")
		if requestID == "" {
			requestID = fmt.Sprintf("%d", atomic.AddUint32(&requestCounter, 1))
		}

		logger := logger.With(zap.String("request_id", requestID))

		r = r.WithContext(context.WithValue(r.Context(), "logger", logger))

		start := time.Now()
		handler.ServeHTTP(w, r)
		d := time.Since(start)
		logger.Info("access",
			zap.Duration("time", d),
			zap.String("method", r.Method),
			zap.String("url", r.URL.String()),
			zap.String("peer", r.RemoteAddr),
			zap.Int("status", writer.Status()),
		)
	})
}

func main() {
	var err error

	/* CONFIG start */

	configFile := flag.String("config", "/etc/graphite-clickhouse/graphite-clickhouse.conf", "Filename of config")
	printDefaultConfig := flag.Bool("config-print-default", false, "Print default config")
	checkConfig := flag.Bool("check-config", false, "Check config and exit")
	tags := flag.Bool("tags", false, "Build tags table")

	printVersion := flag.Bool("version", false, "Print version")

	flag.Parse()

	if *printVersion {
		fmt.Print(Version)
		return
	}

	if *printDefaultConfig {
		if err = config.PrintDefaultConfig(); err != nil {
			log.Fatal(err)
		}
		return
	}

	cfg, err := config.ReadConfig(*configFile)
	if err != nil {
		log.Fatal(err)
	}

	// config parsed successfully. Exit in check-only mode
	if *checkConfig {
		return
	}

	if err = zapwriter.ApplyConfig(cfg.Logging); err != nil {
		log.Fatal(err)
	}

	runtime.GOMAXPROCS(cfg.Common.MaxCPU)

	/* CONFIG end */

	/* CONSOLE COMMANDS start */
	if *tags {
		if err := tagger.Make(cfg); err != nil {
			log.Fatal(err)
		}
		return
	}

	/* CONSOLE COMMANDS end */

	http.Handle("/metrics/find/", Handler(zapwriter.Default(), find.NewHandler(cfg)))
	http.Handle("/render/", Handler(zapwriter.Default(), render.NewHandler(cfg)))

	http.Handle("/", Handler(zapwriter.Default(), http.HandlerFunc(http.NotFound)))

	log.Fatal(http.ListenAndServe(cfg.Common.Listen, nil))
}
<|endoftext|>


<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package keybase

import (
	"encoding/base64"
	"errors"
	"fmt"
	"net"
	"os"
	"path/filepath"
	"runtime/trace"
	"sync"

	"strings"

	"github.com/keybase/client/go/externals"
	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/logger"
	"github.com/keybase/client/go/protocol/keybase1"
	"github.com/keybase/client/go/pvlsource"
	"github.com/keybase/client/go/service"
	"github.com/keybase/client/go/teams"
	"github.com/keybase/go-framed-msgpack-rpc/rpc"
	"github.com/keybase/kbfs/fsrpc"
	"github.com/keybase/kbfs/libkbfs"
)

var kbCtx *libkb.GlobalContext
var conn net.Conn
var startOnce sync.Once
var logSendContext libkb.LogSendContext
var kbfsConfig libkbfs.Config

type ExternalDNSNSFetcher interface {
	GetServers() []byte
}

type dnsNSFetcher struct {
	externalFetcher ExternalDNSNSFetcher
}

func newDNSNSFetcher(d ExternalDNSNSFetcher) dnsNSFetcher {
	return dnsNSFetcher{
		externalFetcher: d,
	}
}

func (d dnsNSFetcher) processExternalResult(raw []byte) []string {
	return strings.Split(string(raw), ",")
}

func (d dnsNSFetcher) GetServers() []string {
	if d.externalFetcher != nil {
		return d.processExternalResult(d.externalFetcher.GetServers())
	}
	return getDNSServers()
}

var _ libkb.DNSNameServerFetcher = dnsNSFetcher{}

// InitOnce runs the Keybase services (only runs one time)
func InitOnce(homeDir string, logFile string, runModeStr string, accessGroupOverride bool,
	dnsNSFetcher ExternalDNSNSFetcher) {
	startOnce.Do(func() {
		if err := Init(homeDir, logFile, runModeStr, accessGroupOverride, dnsNSFetcher); err != nil {
			kbCtx.Log.Errorf("Init error: %s", err)
		}
	})
}

// Init runs the Keybase services
func Init(homeDir string, logFile string, runModeStr string, accessGroupOverride bool,
	externalDNSNSFetcher ExternalDNSNSFetcher) error {
	fmt.Println("Go: Initializing")
	if logFile != "" {
		fmt.Printf("Go: Using log: %s\n", logFile)
	}

	tname := filepath.Join(filepath.Base(logFile), "svctrace.out")
	f, err := os.Create(tname)
	if err != nil {
		return err
	}
	fmt.Printf("Go: starting trace %s\n", tname)
	trace.Start(f)
	sd := func() error {
		fmt.Printf("Go: stopping trace %s\n", tname)
		trace.Stop()
		return nil
	}
	libkb.G.PushShutdownHook(sd)

	dnsNSFetcher := newDNSNSFetcher(externalDNSNSFetcher)
	dnsServers := dnsNSFetcher.GetServers()
	for _, srv := range dnsServers {
		fmt.Printf("Go: DNS Server: %s\n", srv)
	}

	kbCtx = libkb.G
	kbCtx.Init()
	kbCtx.SetServices(externals.GetServices())
	pvlsource.NewPvlSourceAndInstall(kbCtx)
	teams.NewTeamLoaderAndInstall(kbCtx)
	usage := libkb.Usage{
		Config:    true,
		API:       true,
		KbKeyring: true,
	}
	runMode, err := libkb.StringToRunMode(runModeStr)
	if err != nil {
		return err
	}
	config := libkb.AppConfig{
		HomeDir:                     homeDir,
		LogFile:                     logFile,
		RunMode:                     runMode,
		Debug:                       true,
		LocalRPCDebug:               "",
		SecurityAccessGroupOverride: accessGroupOverride,
	}
	err = kbCtx.Configure(config, usage)
	if err != nil {
		return err
	}

	svc := service.NewService(kbCtx, false)
	err = svc.StartLoopbackServer()
	if err != nil {
		return err
	}
	kbCtx.SetService()
	uir := service.NewUIRouter(kbCtx)
	kbCtx.SetUIRouter(uir)
	kbCtx.SetDNSNameServerFetcher(dnsNSFetcher)
	svc.SetupCriticalSubServices()
	svc.RunBackgroundOperations(uir)

	serviceLog := config.GetLogFile()
	logs := libkb.Logs{
		Service: serviceLog,
	}

	logSendContext = libkb.LogSendContext{
		Contextified: libkb.NewContextified(kbCtx),
		Logs:         logs,
	}

	// open the connection
	err = Reset()
	if err != nil {
		return err
	}

	go func() {
		kbfsParams := libkbfs.DefaultInitParams(kbCtx)
		// Setting this flag will enable KBFS debug logging to alway be
		// true in a mobile setting. Kill this setting if too spammy.
		kbfsParams.Debug = true
		kbfsParams.Mode = libkbfs.InitMinimalString
		kbfsConfig, _ = libkbfs.Init(kbCtx, kbfsParams, serviceCn{}, func() {}, kbCtx.Log)
	}()

	return nil
}

type serviceCn struct {
	ctx *libkb.GlobalContext
}

func (s serviceCn) NewKeybaseService(config libkbfs.Config, params libkbfs.InitParams, ctx libkbfs.Context, log logger.Logger) (libkbfs.KeybaseService, error) {
	keybaseService := libkbfs.NewKeybaseDaemonRPC(config, ctx, log, true, nil)
	keybaseService.AddProtocols([]rpc.Protocol{
		keybase1.FsProtocol(fsrpc.NewFS(config, log)),
	})
	return keybaseService, nil
}

func (s serviceCn) NewCrypto(config libkbfs.Config, params libkbfs.InitParams, ctx libkbfs.Context, log logger.Logger) (libkbfs.Crypto, error) {
	return libkbfs.NewCryptoClientRPC(config, ctx), nil
}

// LogSend sends a log to Keybase
func LogSend(status string, feedback string, sendLogs bool, uiLogPath string) (string, error) {
	logSendContext.Logs.Desktop = uiLogPath
	return logSendContext.LogSend(status, feedback, sendLogs, 5*1024*1024)
}

// WriteB64 sends a base64 encoded msgpack rpc payload
func WriteB64(str string) error {
	data, err := base64.StdEncoding.DecodeString(str)
	if err != nil {
		return fmt.Errorf("Base64 decode error: %s; %s", err, str)
	}
	n, err := conn.Write(data)
	if err != nil {
		return fmt.Errorf("Write error: %s", err)
	}
	if n != len(data) {
		return errors.New("Did not write all the data")
	}
	return nil
}

const targetBufferSize = 50 * 1024

// bufferSize must be divisible by 3 to ensure that we don't split
// our b64 encode across a payload boundary if we go over our buffer
// size.
const bufferSize = targetBufferSize - (targetBufferSize % 3)

// buffer for the conn.Read
var buffer = make([]byte, bufferSize)

// ReadB64 is a blocking read for base64 encoded msgpack rpc data.
// It is called serially by the mobile run loops.
func ReadB64() (string, error) {
	n, err := conn.Read(buffer)
	if n > 0 && err == nil {
		str := base64.StdEncoding.EncodeToString(buffer[0:n])
		return str, nil
	}

	if err != nil {
		// Attempt to fix the connection
		Reset()
		return "", fmt.Errorf("Read error: %s", err)
	}

	return "", nil
}

// Reset resets the socket connection
func Reset() error {
	if conn != nil {
		conn.Close()
	}

	var err error
	conn, err = kbCtx.LoopbackListener.Dial()
	if err != nil {
		return fmt.Errorf("Socket error: %s", err)
	}
	return nil
}

// Version returns semantic version string
func Version() string {
	return libkb.VersionString()
}
<commit_msg>Add startTrace<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package keybase

import (
	"encoding/base64"
	"errors"
	"fmt"
	"net"
	"os"
	"path/filepath"
	"runtime/trace"
	"sync"
	"time"

	"strings"

	"github.com/keybase/client/go/externals"
	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/logger"
	"github.com/keybase/client/go/protocol/keybase1"
	"github.com/keybase/client/go/pvlsource"
	"github.com/keybase/client/go/service"
	"github.com/keybase/client/go/teams"
	"github.com/keybase/go-framed-msgpack-rpc/rpc"
	"github.com/keybase/kbfs/fsrpc"
	"github.com/keybase/kbfs/libkbfs"
)

var kbCtx *libkb.GlobalContext
var conn net.Conn
var startOnce sync.Once
var logSendContext libkb.LogSendContext
var kbfsConfig libkbfs.Config

type ExternalDNSNSFetcher interface {
	GetServers() []byte
}

type dnsNSFetcher struct {
	externalFetcher ExternalDNSNSFetcher
}

func newDNSNSFetcher(d ExternalDNSNSFetcher) dnsNSFetcher {
	return dnsNSFetcher{
		externalFetcher: d,
	}
}

func (d dnsNSFetcher) processExternalResult(raw []byte) []string {
	return strings.Split(string(raw), ",")
}

func (d dnsNSFetcher) GetServers() []string {
	if d.externalFetcher != nil {
		return d.processExternalResult(d.externalFetcher.GetServers())
	}
	return getDNSServers()
}

var _ libkb.DNSNameServerFetcher = dnsNSFetcher{}

// InitOnce runs the Keybase services (only runs one time)
func InitOnce(homeDir string, logFile string, runModeStr string, accessGroupOverride bool,
	dnsNSFetcher ExternalDNSNSFetcher) {
	startOnce.Do(func() {
		if err := Init(homeDir, logFile, runModeStr, accessGroupOverride, dnsNSFetcher); err != nil {
			kbCtx.Log.Errorf("Init error: %s", err)
		}
	})
}

// Init runs the Keybase services
func Init(homeDir string, logFile string, runModeStr string, accessGroupOverride bool,
	externalDNSNSFetcher ExternalDNSNSFetcher) error {
	fmt.Println("Go: Initializing")
	if logFile != "" {
		fmt.Printf("Go: Using log: %s\n", logFile)
	}

	startTrace()

	dnsNSFetcher := newDNSNSFetcher(externalDNSNSFetcher)
	dnsServers := dnsNSFetcher.GetServers()
	for _, srv := range dnsServers {
		fmt.Printf("Go: DNS Server: %s\n", srv)
	}

	kbCtx = libkb.G
	kbCtx.Init()
	kbCtx.SetServices(externals.GetServices())
	pvlsource.NewPvlSourceAndInstall(kbCtx)
	teams.NewTeamLoaderAndInstall(kbCtx)
	usage := libkb.Usage{
		Config:    true,
		API:       true,
		KbKeyring: true,
	}
	runMode, err := libkb.StringToRunMode(runModeStr)
	if err != nil {
		return err
	}
	config := libkb.AppConfig{
		HomeDir:                     homeDir,
		LogFile:                     logFile,
		RunMode:                     runMode,
		Debug:                       true,
		LocalRPCDebug:               "",
		SecurityAccessGroupOverride: accessGroupOverride,
	}
	err = kbCtx.Configure(config, usage)
	if err != nil {
		return err
	}

	svc := service.NewService(kbCtx, false)
	err = svc.StartLoopbackServer()
	if err != nil {
		return err
	}
	kbCtx.SetService()
	uir := service.NewUIRouter(kbCtx)
	kbCtx.SetUIRouter(uir)
	kbCtx.SetDNSNameServerFetcher(dnsNSFetcher)
	svc.SetupCriticalSubServices()
	svc.RunBackgroundOperations(uir)

	serviceLog := config.GetLogFile()
	logs := libkb.Logs{
		Service: serviceLog,
	}

	logSendContext = libkb.LogSendContext{
		Contextified: libkb.NewContextified(kbCtx),
		Logs:         logs,
	}

	// open the connection
	err = Reset()
	if err != nil {
		return err
	}

	go func() {
		kbfsParams := libkbfs.DefaultInitParams(kbCtx)
		// Setting this flag will enable KBFS debug logging to alway be
		// true in a mobile setting. Kill this setting if too spammy.
		kbfsParams.Debug = true
		kbfsParams.Mode = libkbfs.InitMinimalString
		kbfsConfig, _ = libkbfs.Init(kbCtx, kbfsParams, serviceCn{}, func() {}, kbCtx.Log)
	}()

	return nil
}

type serviceCn struct {
	ctx *libkb.GlobalContext
}

func (s serviceCn) NewKeybaseService(config libkbfs.Config, params libkbfs.InitParams, ctx libkbfs.Context, log logger.Logger) (libkbfs.KeybaseService, error) {
	keybaseService := libkbfs.NewKeybaseDaemonRPC(config, ctx, log, true, nil)
	keybaseService.AddProtocols([]rpc.Protocol{
		keybase1.FsProtocol(fsrpc.NewFS(config, log)),
	})
	return keybaseService, nil
}

func (s serviceCn) NewCrypto(config libkbfs.Config, params libkbfs.InitParams, ctx libkbfs.Context, log logger.Logger) (libkbfs.Crypto, error) {
	return libkbfs.NewCryptoClientRPC(config, ctx), nil
}

// LogSend sends a log to Keybase
func LogSend(status string, feedback string, sendLogs bool, uiLogPath string) (string, error) {
	logSendContext.Logs.Desktop = uiLogPath
	return logSendContext.LogSend(status, feedback, sendLogs, 5*1024*1024)
}

// WriteB64 sends a base64 encoded msgpack rpc payload
func WriteB64(str string) error {
	data, err := base64.StdEncoding.DecodeString(str)
	if err != nil {
		return fmt.Errorf("Base64 decode error: %s; %s", err, str)
	}
	n, err := conn.Write(data)
	if err != nil {
		return fmt.Errorf("Write error: %s", err)
	}
	if n != len(data) {
		return errors.New("Did not write all the data")
	}
	return nil
}

const targetBufferSize = 50 * 1024

// bufferSize must be divisible by 3 to ensure that we don't split
// our b64 encode across a payload boundary if we go over our buffer
// size.
const bufferSize = targetBufferSize - (targetBufferSize % 3)

// buffer for the conn.Read
var buffer = make([]byte, bufferSize)

// ReadB64 is a blocking read for base64 encoded msgpack rpc data.
// It is called serially by the mobile run loops.
func ReadB64() (string, error) {
	n, err := conn.Read(buffer)
	if n > 0 && err == nil {
		str := base64.StdEncoding.EncodeToString(buffer[0:n])
		return str, nil
	}

	if err != nil {
		// Attempt to fix the connection
		Reset()
		return "", fmt.Errorf("Read error: %s", err)
	}

	return "", nil
}

// Reset resets the socket connection
func Reset() error {
	if conn != nil {
		conn.Close()
	}

	var err error
	conn, err = kbCtx.LoopbackListener.Dial()
	if err != nil {
		return fmt.Errorf("Socket error: %s", err)
	}
	return nil
}

// Version returns semantic version string
func Version() string {
	return libkb.VersionString()
}

func startTrace() {
	if os.Getenv("KEYBASE_TRACE_MOBILE") != "1" {
		return
	}

	tname := filepath.Join(filepath.Dir(logFile), "svctrace.out")
	f, err := os.Create(tname)
	if err != nil {
		return err
	}
	fmt.Printf("Go: starting trace %s\n", tname)
	trace.Start(f)
	go func() {
		fmt.Printf("Go: sleeping 30s for trace\n")
		time.Sleep(30 * time.Second)
		fmt.Printf("Go: stopping trace %s\n", tname)
		trace.Stop()
		time.Sleep(5 * time.Second)
		fmt.Printf("Go: trace stopped\n")
	}()
}
<|endoftext|>


<commit_before>/*
   Copyright 2014 Outbrain Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package inst

import (
	"errors"
	"fmt"
	"github.com/outbrain/golib/log"
	"github.com/outbrain/orchestrator/go/config"
	"github.com/pmylund/go-cache"
	"net"
	"regexp"
	"strings"
	"time"
)

type HostnameResolve struct {
	hostname         string
	resolvedHostname string
}

func init() {
	if config.Config.ExpiryHostnameResolvesMinutes < 1 {
		config.Config.ExpiryHostnameResolvesMinutes = 1
	}
}

var hostnameResolvesLightweightCache = cache.New(time.Duration(config.Config.ExpiryHostnameResolvesMinutes)*time.Minute, time.Minute)
var hostnameResolvesLightweightCacheLoadedOnceFromDB bool = false

// GetCNAME resolves an IP or hostname into a normalized valid CNAME
func GetCNAME(hostname string) (string, error) {
	res, err := net.LookupCNAME(hostname)
	if err != nil {
		return hostname, err
	}
	res = strings.TrimRight(res, ".")
	return res, nil
}

func resolveHostname(hostname string) (string, error) {
	switch strings.ToLower(config.Config.HostnameResolveMethod) {
	case "none":
		return hostname, nil
	case "default":
		return hostname, nil
	case "cname":
		return GetCNAME(hostname)
	}
	return hostname, nil
}

// Attempt to resolve a hostname. This may returned a database cached hostname or otherwise
// it may resolve the hostname via CNAME
func ResolveHostname(hostname string) (string, error) {
	hostname = strings.TrimSpace(hostname)
	if hostname == "" {
		return hostname, errors.New("Will not resolve empty hostname")
	}
	// First go to lightweight cache
	if resolvedHostname, found := hostnameResolvesLightweightCache.Get(hostname); found {
		return resolvedHostname.(string), nil
	}

	if !hostnameResolvesLightweightCacheLoadedOnceFromDB {
		// A continuous-discovery will first make sure to load all resolves from DB.
		// However cli does not do so.
		// Anyway, it seems like the cache was not loaded from DB. Before doing real resolves,
		// let's try and get the resolved hostname from database.
		if resolvedHostname, err := ReadResolvedHostname(hostname); err == nil && resolvedHostname != "" {
			hostnameResolvesLightweightCache.Set(hostname, resolvedHostname, 0)
			return resolvedHostname, nil
		}
	}

	// Unfound: resolve!
	log.Debugf("Hostname unresolved yet: %s", hostname)
	resolvedHostname, err := resolveHostname(hostname)
	if config.Config.RejectHostnameResolvePattern != "" {
		// Reject, don't even cache
		if matched, _ := regexp.MatchString(config.Config.RejectHostnameResolvePattern, resolvedHostname); matched {
			return hostname, fmt.Errorf("Resolved hostname is rejected: %s", resolvedHostname)
		}
	}

	if err != nil {
		// Problem. What we'll do is cache the hostname for just one minute, so as to avoid flooding requests
		// on one hand, yet make it refresh shortly on the other hand. Anyway do not write to database.
		hostnameResolvesLightweightCache.Set(hostname, resolvedHostname, time.Minute)
		return hostname, err
	}
	// Good result! Cache it, also to DB
	log.Debugf("Cache hostname resolve %s as %s", hostname, resolvedHostname)
	UpdateResolvedHostname(hostname, resolvedHostname)
	return resolvedHostname, nil
}

// UpdateResolvedHostname will store the given resolved hostname in cache
// Returns false when the key already existed with same resolved value (similar
// to AFFECTED_ROWS() in mysql)
func UpdateResolvedHostname(hostname string, resolvedHostname string) bool {
	if existingResolvedHostname, found := hostnameResolvesLightweightCache.Get(hostname); found && (existingResolvedHostname == resolvedHostname) {
		return false
	}
	hostnameResolvesLightweightCache.Set(hostname, resolvedHostname, 0)
	if strings.ToLower(config.Config.HostnameResolveMethod) != "none" {
		WriteResolvedHostname(hostname, resolvedHostname)
	}
	return true
}

func LoadHostnameResolveCacheFromDatabase() error {
	allHostnamesResolves, err := readAllHostnameResolves()
	if err != nil {
		return err
	}
	for _, hostnameResolve := range allHostnamesResolves {
		hostnameResolvesLightweightCache.Set(hostnameResolve.hostname, hostnameResolve.resolvedHostname, 0)
	}
	hostnameResolvesLightweightCacheLoadedOnceFromDB = true
	return nil
}

func ResetHostnameResolveCache() error {
	err := deleteHostnameResolves()
	hostnameResolvesLightweightCache.Flush()
	hostnameResolvesLightweightCacheLoadedOnceFromDB = false
	return err
}

func HostnameResolveCache() (map[string]*cache.Item, error) {
	return hostnameResolvesLightweightCache.Items(), nil
}

func UnresolveHostname(instanceKey *InstanceKey) (InstanceKey, bool, error) {
	unresolvedHostname, err := readUnresolvedHostname(instanceKey.Hostname)
	if err != nil {
		return *instanceKey, false, log.Errore(err)
	}
	if unresolvedHostname == instanceKey.Hostname {
		// unchanged. Nothing to do
		return *instanceKey, false, nil
	}
	// We unresovled to a different hostname. We will now re-resolve to double-check!
	unresolvedKey := &InstanceKey{Hostname: unresolvedHostname, Port: instanceKey.Port}

	instance, err := ReadTopologyInstance(unresolvedKey)
	if err != nil {
		return *instanceKey, false, log.Errore(err)
	}
	if instance.Key.Hostname != instanceKey.Hostname {
		// Resolve(Unresolve(hostname)) != hostname ==> Bad; reject
		if *config.RuntimeCLIFlags.SkipUnresolveCheck {
			return *instanceKey, false, nil
		}
		return *instanceKey, false, log.Errorf("Error unresolving; hostname=%s, unresolved=%s, re-resolved=%s; mismatch. Skip/ignore with --skip-unresolve-check", instanceKey.Hostname, unresolvedKey.Hostname, instance.Key.Hostname)
	}
	return *unresolvedKey, true, nil
}

func RegisterHostnameUnresolve(instanceKey *InstanceKey, unresolvedHostname string) (err error) {
	return WriteHostnameUnresolve(instanceKey, unresolvedHostname)
}
<commit_msg>Not reading hostname from db is resolve config is "none"<commit_after>/*
   Copyright 2014 Outbrain Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package inst

import (
	"errors"
	"fmt"
	"github.com/outbrain/golib/log"
	"github.com/outbrain/orchestrator/go/config"
	"github.com/pmylund/go-cache"
	"net"
	"regexp"
	"strings"
	"time"
)

type HostnameResolve struct {
	hostname         string
	resolvedHostname string
}

func init() {
	if config.Config.ExpiryHostnameResolvesMinutes < 1 {
		config.Config.ExpiryHostnameResolvesMinutes = 1
	}
}

var hostnameResolvesLightweightCache = cache.New(time.Duration(config.Config.ExpiryHostnameResolvesMinutes)*time.Minute, time.Minute)
var hostnameResolvesLightweightCacheLoadedOnceFromDB bool = false

// GetCNAME resolves an IP or hostname into a normalized valid CNAME
func GetCNAME(hostname string) (string, error) {
	res, err := net.LookupCNAME(hostname)
	if err != nil {
		return hostname, err
	}
	res = strings.TrimRight(res, ".")
	return res, nil
}

func resolveHostname(hostname string) (string, error) {
	switch strings.ToLower(config.Config.HostnameResolveMethod) {
	case "none":
		return hostname, nil
	case "default":
		return hostname, nil
	case "cname":
		return GetCNAME(hostname)
	}
	return hostname, nil
}

// Attempt to resolve a hostname. This may returned a database cached hostname or otherwise
// it may resolve the hostname via CNAME
func ResolveHostname(hostname string) (string, error) {
	hostname = strings.TrimSpace(hostname)
	if hostname == "" {
		return hostname, errors.New("Will not resolve empty hostname")
	}
	// First go to lightweight cache
	if resolvedHostname, found := hostnameResolvesLightweightCache.Get(hostname); found {
		return resolvedHostname.(string), nil
	}

	if !hostnameResolvesLightweightCacheLoadedOnceFromDB {
		// A continuous-discovery will first make sure to load all resolves from DB.
		// However cli does not do so.
		// Anyway, it seems like the cache was not loaded from DB. Before doing real resolves,
		// let's try and get the resolved hostname from database.
		if strings.ToLower(config.Config.HostnameResolveMethod) != "none" {
			if resolvedHostname, err := ReadResolvedHostname(hostname); err == nil && resolvedHostname != "" {
				hostnameResolvesLightweightCache.Set(hostname, resolvedHostname, 0)
				return resolvedHostname, nil
			}
		}
	}

	// Unfound: resolve!
	log.Debugf("Hostname unresolved yet: %s", hostname)
	resolvedHostname, err := resolveHostname(hostname)
	if config.Config.RejectHostnameResolvePattern != "" {
		// Reject, don't even cache
		if matched, _ := regexp.MatchString(config.Config.RejectHostnameResolvePattern, resolvedHostname); matched {
			return hostname, fmt.Errorf("Resolved hostname is rejected: %s", resolvedHostname)
		}
	}

	if err != nil {
		// Problem. What we'll do is cache the hostname for just one minute, so as to avoid flooding requests
		// on one hand, yet make it refresh shortly on the other hand. Anyway do not write to database.
		hostnameResolvesLightweightCache.Set(hostname, resolvedHostname, time.Minute)
		return hostname, err
	}
	// Good result! Cache it, also to DB
	log.Debugf("Cache hostname resolve %s as %s", hostname, resolvedHostname)
	UpdateResolvedHostname(hostname, resolvedHostname)
	return resolvedHostname, nil
}

// UpdateResolvedHostname will store the given resolved hostname in cache
// Returns false when the key already existed with same resolved value (similar
// to AFFECTED_ROWS() in mysql)
func UpdateResolvedHostname(hostname string, resolvedHostname string) bool {
	if existingResolvedHostname, found := hostnameResolvesLightweightCache.Get(hostname); found && (existingResolvedHostname == resolvedHostname) {
		return false
	}
	hostnameResolvesLightweightCache.Set(hostname, resolvedHostname, 0)
	if strings.ToLower(config.Config.HostnameResolveMethod) != "none" {
		WriteResolvedHostname(hostname, resolvedHostname)
	}
	return true
}

func LoadHostnameResolveCacheFromDatabase() error {
	allHostnamesResolves, err := readAllHostnameResolves()
	if err != nil {
		return err
	}
	for _, hostnameResolve := range allHostnamesResolves {
		hostnameResolvesLightweightCache.Set(hostnameResolve.hostname, hostnameResolve.resolvedHostname, 0)
	}
	hostnameResolvesLightweightCacheLoadedOnceFromDB = true
	return nil
}

func ResetHostnameResolveCache() error {
	err := deleteHostnameResolves()
	hostnameResolvesLightweightCache.Flush()
	hostnameResolvesLightweightCacheLoadedOnceFromDB = false
	return err
}

func HostnameResolveCache() (map[string]*cache.Item, error) {
	return hostnameResolvesLightweightCache.Items(), nil
}

func UnresolveHostname(instanceKey *InstanceKey) (InstanceKey, bool, error) {
	unresolvedHostname, err := readUnresolvedHostname(instanceKey.Hostname)
	if err != nil {
		return *instanceKey, false, log.Errore(err)
	}
	if unresolvedHostname == instanceKey.Hostname {
		// unchanged. Nothing to do
		return *instanceKey, false, nil
	}
	// We unresovled to a different hostname. We will now re-resolve to double-check!
	unresolvedKey := &InstanceKey{Hostname: unresolvedHostname, Port: instanceKey.Port}

	instance, err := ReadTopologyInstance(unresolvedKey)
	if err != nil {
		return *instanceKey, false, log.Errore(err)
	}
	if instance.Key.Hostname != instanceKey.Hostname {
		// Resolve(Unresolve(hostname)) != hostname ==> Bad; reject
		if *config.RuntimeCLIFlags.SkipUnresolveCheck {
			return *instanceKey, false, nil
		}
		return *instanceKey, false, log.Errorf("Error unresolving; hostname=%s, unresolved=%s, re-resolved=%s; mismatch. Skip/ignore with --skip-unresolve-check", instanceKey.Hostname, unresolvedKey.Hostname, instance.Key.Hostname)
	}
	return *unresolvedKey, true, nil
}

func RegisterHostnameUnresolve(instanceKey *InstanceKey, unresolvedHostname string) (err error) {
	return WriteHostnameUnresolve(instanceKey, unresolvedHostname)
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"os/exec"
	"regexp"
	"strings"
	"sync"
	"time"

	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	pb "github.com/brotherlogic/gobuildslave/proto"
	pbgs "github.com/brotherlogic/goserver/proto"
	"github.com/brotherlogic/goserver/utils"
)

// RunJob - runs the job
func (s *Server) RunJob(ctx context.Context, req *pb.RunRequest) (*pb.RunResponse, error) {
	if !req.GetJob().GetBreakout() &&
		(s.Registry.Identifier == "clust6" ||
			s.Registry.Identifier == "clust3" ||
			s.Registry.Identifier == "clust7" ||
			s.Registry.Identifier == "clust8" ||
			s.Registry.Identifier == "clust4") {
		return &pb.RunResponse{}, status.Errorf(codes.FailedPrecondition, "we only run the basic set of jobs")
	}

	if req.GetBits() > 0 && s.Bits != int(req.GetBits()) {
		return &pb.RunResponse{}, status.Errorf(codes.FailedPrecondition, "Cannot run %v bits on this server", req.GetBits())
	}

	if !s.doesBuild && !req.Job.Breakout {
		return &pb.RunResponse{}, status.Errorf(codes.FailedPrecondition, "Refusing to build")
	}
	s.nMut.Lock()
	defer s.nMut.Unlock()
	if _, ok := s.njobs[req.GetJob().GetName()]; ok {
		return &pb.RunResponse{}, fmt.Errorf("Already running this job!")
	}

	if len(s.njobs) > s.maxJobs && !req.GetJob().GetBreakout() {
		return nil, status.Errorf(codes.FailedPrecondition, "We're running %v jobs, can't run no more", len(s.njobs))
	}

	s.CtxLog(ctx, "Running %v")

	s.njobs[req.GetJob().GetName()] = &pb.JobAssignment{Job: req.GetJob(), LastTransitionTime: time.Now().Unix(), Bits: int32(s.Bits)}
	go s.nmonitor(s.njobs[req.GetJob().GetName()])

	return &pb.RunResponse{}, nil
}

// KillJob - kills the job
func (s *Server) KillJob(ctx context.Context, req *pb.KillRequest) (*pb.KillResponse, error) {
	s.nMut.Lock()
	defer s.nMut.Unlock()

	if _, ok := s.njobs[req.GetJob().GetName()]; !ok {
		return nil, fmt.Errorf("Job was not running")
	}

	s.njobs[req.GetJob().GetName()].State = pb.State_KILLING
	return &pb.KillResponse{}, nil
}

//UpdateJob - updates the job
func (s *Server) UpdateJob(ctx context.Context, req *pb.UpdateRequest) (*pb.UpdateResponse, error) {
	s.nMut.Lock()
	defer s.nMut.Unlock()
	if _, ok := s.njobs[req.GetJob().GetName()]; !ok {
		return nil, fmt.Errorf("Job was not running")
	}

	s.njobs[req.GetJob().GetName()].State = pb.State_UPDATE_STARTING
	return &pb.UpdateResponse{}, nil
}

// ListJobs - lists the jobs
func (s *Server) ListJobs(ctx context.Context, req *pb.ListRequest) (*pb.ListResponse, error) {
	s.nMut.Lock()
	defer s.nMut.Unlock()
	resp := &pb.ListResponse{}
	for _, job := range s.njobs {
		resp.Jobs = append(resp.Jobs, job)
	}
	return resp, nil
}

func extractBitRate(output string) (string, string) {
	matcher := regexp.MustCompile("Rate=(.*?) ")
	matches := matcher.FindStringSubmatch(output)

	matcher2 := regexp.MustCompile("Access Point. ([A-F0-9:]*)")
	matches2 := matcher2.FindStringSubmatch(output)
	if len(matches) > 0 && len(matches2) > 0 {
		return strings.TrimRight(matches[1], " "), strings.TrimRight(matches2[1], " ")
	}
	return "", ""
}

// SlaveConfig gets the config for this slave
func (s *Server) SlaveConfig(ctx context.Context, req *pb.ConfigRequest) (*pb.ConfigResponse, error) {
	disks := s.disker.getDisks()
	requirements := make([]*pb.Requirement, 0)
	for _, disk := range disks {
		requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_DISK, Properties: disk})
	}
	requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_SERVER, Properties: s.Registry.Identifier})
	if s.Registry.Identifier == "argon" {
		requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_EXTERNAL, Properties: "external_ready"})
	}

	data, err := exec.Command("/usr/bin/lsusb").Output()
	if err != nil {
		return nil, fmt.Errorf("error listing usb components: %v", err)
	}
	s.CtxLog(ctx, fmt.Sprintf("USBRES: %v", string(data)))
	if strings.Contains(string(data), "TSP100II") {
		requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_RECEIPT_PRINTER})
	}

	out, _ := exec.Command("/sbin/iwconfig").Output()
	br, ap := extractBitRate(string(out))
	s.accessPoint = ap
	requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_NETWORK, Properties: br})
	requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_ACCESS_POINT, Properties: ap})

	requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_BITS, Properties: fmt.Sprintf("%v", s.Bits)})

	out, _ = exec.Command("cat", "/sys/firmware/devicetree/base/model").Output()
	requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_HOST_TYPE, Properties: string(out)})

	requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_ZONE, Properties: s.Registry.Zone})

	return &pb.ConfigResponse{Config: &pb.SlaveConfig{Requirements: requirements}}, nil
}

func (s *Server) FullShutdown(ctx context.Context, req *pb.ShutdownRequest) (*pb.ShutdownResponse, error) {
	defer func() {
		s.CtxLog(ctx, "Running shutdown")
		time.Sleep(time.Minute)

		err := exec.Command("sudo", "shutdown", "-h", "now").Run()
		s.CtxLog(ctx, fmt.Sprintf("Shutdown: %v", err))
	}()

	jobs, err := s.ListJobs(ctx, &pb.ListRequest{})
	if err != nil {
		return nil, err
	}

	s.CtxLog(ctx, fmt.Sprintf("Shutting down %v jobs", len(jobs.GetJobs())))

	wg := &sync.WaitGroup{}
	for _, job := range jobs.GetJobs() {
		if job.GetPort() != 0 {
			wg.Add(1)

			go func(job *pb.JobAssignment) {
				conn, err := utils.LFDial(fmt.Sprintf("%v:%v", job.GetHost(), job.GetPort()))
				if err != nil {
					return
				}
				s.CtxLog(ctx, fmt.Sprintf("Calling shutdown on %v", job))
				gsclient := pbgs.NewGoserverServiceClient(conn)
				_, err = gsclient.Shutdown(ctx, &pbgs.ShutdownRequest{})
				if err != nil {
					s.CtxLog(ctx, fmt.Sprintf("Failed shutdown: %v", err))
				}
			}(job)
		} else {
			s.CtxLog(ctx, fmt.Sprintf("Not shutting down %v", job))
		}
	}

	wg.Wait()

	return &pb.ShutdownResponse{}, nil
}
<commit_msg>Better logging, handle sync<commit_after>package main

import (
	"context"
	"fmt"
	"os/exec"
	"regexp"
	"strings"
	"sync"
	"time"

	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	pb "github.com/brotherlogic/gobuildslave/proto"
	pbgs "github.com/brotherlogic/goserver/proto"
	"github.com/brotherlogic/goserver/utils"
)

// RunJob - runs the job
func (s *Server) RunJob(ctx context.Context, req *pb.RunRequest) (*pb.RunResponse, error) {
	if !req.GetJob().GetBreakout() &&
		(s.Registry.Identifier == "clust6" ||
			s.Registry.Identifier == "clust3" ||
			s.Registry.Identifier == "clust7" ||
			s.Registry.Identifier == "clust8" ||
			s.Registry.Identifier == "clust4") {
		return &pb.RunResponse{}, status.Errorf(codes.FailedPrecondition, "we only run the basic set of jobs")
	}

	if req.GetBits() > 0 && s.Bits != int(req.GetBits()) {
		return &pb.RunResponse{}, status.Errorf(codes.FailedPrecondition, "Cannot run %v bits on this server", req.GetBits())
	}

	if !s.doesBuild && !req.Job.Breakout {
		return &pb.RunResponse{}, status.Errorf(codes.FailedPrecondition, "Refusing to build")
	}
	s.nMut.Lock()
	defer s.nMut.Unlock()
	if _, ok := s.njobs[req.GetJob().GetName()]; ok {
		return &pb.RunResponse{}, fmt.Errorf("Already running this job!")
	}

	if len(s.njobs) > s.maxJobs && !req.GetJob().GetBreakout() {
		return nil, status.Errorf(codes.FailedPrecondition, "We're running %v jobs, can't run no more", len(s.njobs))
	}

	s.CtxLog(ctx, "Running %v")

	s.njobs[req.GetJob().GetName()] = &pb.JobAssignment{Job: req.GetJob(), LastTransitionTime: time.Now().Unix(), Bits: int32(s.Bits)}
	go s.nmonitor(s.njobs[req.GetJob().GetName()])

	return &pb.RunResponse{}, nil
}

// KillJob - kills the job
func (s *Server) KillJob(ctx context.Context, req *pb.KillRequest) (*pb.KillResponse, error) {
	s.nMut.Lock()
	defer s.nMut.Unlock()

	if _, ok := s.njobs[req.GetJob().GetName()]; !ok {
		return nil, fmt.Errorf("Job was not running")
	}

	s.njobs[req.GetJob().GetName()].State = pb.State_KILLING
	return &pb.KillResponse{}, nil
}

//UpdateJob - updates the job
func (s *Server) UpdateJob(ctx context.Context, req *pb.UpdateRequest) (*pb.UpdateResponse, error) {
	s.nMut.Lock()
	defer s.nMut.Unlock()
	if _, ok := s.njobs[req.GetJob().GetName()]; !ok {
		return nil, fmt.Errorf("Job was not running")
	}

	s.njobs[req.GetJob().GetName()].State = pb.State_UPDATE_STARTING
	return &pb.UpdateResponse{}, nil
}

// ListJobs - lists the jobs
func (s *Server) ListJobs(ctx context.Context, req *pb.ListRequest) (*pb.ListResponse, error) {
	s.nMut.Lock()
	defer s.nMut.Unlock()
	resp := &pb.ListResponse{}
	for _, job := range s.njobs {
		resp.Jobs = append(resp.Jobs, job)
	}
	return resp, nil
}

func extractBitRate(output string) (string, string) {
	matcher := regexp.MustCompile("Rate=(.*?) ")
	matches := matcher.FindStringSubmatch(output)

	matcher2 := regexp.MustCompile("Access Point. ([A-F0-9:]*)")
	matches2 := matcher2.FindStringSubmatch(output)
	if len(matches) > 0 && len(matches2) > 0 {
		return strings.TrimRight(matches[1], " "), strings.TrimRight(matches2[1], " ")
	}
	return "", ""
}

// SlaveConfig gets the config for this slave
func (s *Server) SlaveConfig(ctx context.Context, req *pb.ConfigRequest) (*pb.ConfigResponse, error) {
	disks := s.disker.getDisks()
	requirements := make([]*pb.Requirement, 0)
	for _, disk := range disks {
		requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_DISK, Properties: disk})
	}
	requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_SERVER, Properties: s.Registry.Identifier})
	if s.Registry.Identifier == "argon" {
		requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_EXTERNAL, Properties: "external_ready"})
	}

	data, err := exec.Command("/usr/bin/lsusb").Output()
	if err != nil {
		return nil, fmt.Errorf("error listing usb components: %v", err)
	}
	s.CtxLog(ctx, fmt.Sprintf("USBRES: %v", string(data)))
	if strings.Contains(string(data), "TSP100II") {
		requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_RECEIPT_PRINTER})
	}

	out, _ := exec.Command("/sbin/iwconfig").Output()
	br, ap := extractBitRate(string(out))
	s.accessPoint = ap
	requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_NETWORK, Properties: br})
	requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_ACCESS_POINT, Properties: ap})

	requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_BITS, Properties: fmt.Sprintf("%v", s.Bits)})

	out, _ = exec.Command("cat", "/sys/firmware/devicetree/base/model").Output()
	requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_HOST_TYPE, Properties: string(out)})

	requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_ZONE, Properties: s.Registry.Zone})

	return &pb.ConfigResponse{Config: &pb.SlaveConfig{Requirements: requirements}}, nil
}

func (s *Server) FullShutdown(ctx context.Context, req *pb.ShutdownRequest) (*pb.ShutdownResponse, error) {
	defer func() {
		s.CtxLog(ctx, "Running shutdown")
		time.Sleep(time.Minute)

		err := exec.Command("sudo", "shutdown", "-h", "now").Run()
		s.CtxLog(ctx, fmt.Sprintf("Shutdown: %v", err))
	}()

	jobs, err := s.ListJobs(ctx, &pb.ListRequest{})
	if err != nil {
		return nil, err
	}

	s.CtxLog(ctx, fmt.Sprintf("Shutting down %v jobs", len(jobs.GetJobs())))

	wg := &sync.WaitGroup{}
	for _, job := range jobs.GetJobs() {
		if job.GetPort() != 0 {
			wg.Add(1)

			go func(job *pb.JobAssignment) {
				conn, err := utils.LFDial(fmt.Sprintf("%v:%v", job.GetHost(), job.GetPort()))
				if err != nil {
					return
				}
				s.CtxLog(ctx, fmt.Sprintf("Calling shutdown on %v", job))
				gsclient := pbgs.NewGoserverServiceClient(conn)
				_, err = gsclient.Shutdown(ctx, &pbgs.ShutdownRequest{})
				if err != nil {
					s.CtxLog(ctx, fmt.Sprintf("Failed shutdown: %v", err))
				}
				s.CtxLog(ctx, fmt.Sprintf("Done: %v -> %v", job, err))
				wg.Done()
			}(job)
		} else {
			s.CtxLog(ctx, fmt.Sprintf("Not shutting down %v", job))
		}
	}

	wg.Wait()

	return &pb.ShutdownResponse{}, nil
}
<|endoftext|>


<commit_before>package gode

import (
	"io/ioutil"
	"os"
	"testing"
	"time"

	"github.com/franela/goreq"
)

func TestSetup(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping in short mode")
	}
	err := os.MkdirAll("tmp", 0755)
	must(err)
	dir, err := ioutil.TempDir("tmp", "gode")
	must(err)
	defer os.RemoveAll(dir)
	SetRootPath(dir)
	must(Setup())
	if setup, _ := IsSetup(); !setup {
		t.Fatal("IsSetup() returned false")
	}
}

func TestSetupInvalidSha(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping in short mode")
	}
	err := os.MkdirAll("tmp", 0755)
	must(err)
	dir, err := ioutil.TempDir("tmp", "gode")
	must(err)
	defer os.RemoveAll(dir)
	SetRootPath(dir)
	target := targets[0]
	target.Sha = "INVALID"
	err = target.setup()
	if err != errInvalidSha {
		t.Fatal("Setup with invalid SHA")
	}
}

func TestWindowsSetup(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping in short mode")
	}
	err := os.MkdirAll("tmp", 0755)
	must(err)
	dir, err := ioutil.TempDir("tmp", "gode")
	must(err)
	defer os.RemoveAll(dir)
	SetRootPath(dir)
	target := getWindowsTarget()
	must(target.setup())
	if setup, _ := target.isSetup(); !setup {
		t.Fatal("IsSetup() returned false")
	}
}

func must(err error) {
	if err != nil {
		panic(err)
	}
}

func setup() {
	goreq.SetConnectTimeout(15 * time.Second)
	SetRootPath("tmp")
	must(Setup())
}

func getWindowsTarget() *Target {
	for _, t := range targets {
		if t.OS == "windows" {
			return &t
		}
	}
	return nil
}
<commit_msg>ensure error is shown<commit_after>package gode

import (
	"io/ioutil"
	"log"
	"os"
	"testing"
	"time"

	"github.com/franela/goreq"
)

func TestSetup(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping in short mode")
	}
	err := os.MkdirAll("tmp", 0755)
	must(err)
	dir, err := ioutil.TempDir("tmp", "gode")
	must(err)
	defer os.RemoveAll(dir)
	SetRootPath(dir)
	must(Setup())
	if setup, _ := IsSetup(); !setup {
		t.Fatal("IsSetup() returned false")
	}
}

func TestSetupInvalidSha(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping in short mode")
	}
	err := os.MkdirAll("tmp", 0755)
	must(err)
	dir, err := ioutil.TempDir("tmp", "gode")
	must(err)
	defer os.RemoveAll(dir)
	SetRootPath(dir)
	target := targets[0]
	target.Sha = "INVALID"
	err = target.setup()
	if err != errInvalidSha {
		t.Fatal("Setup with invalid SHA")
	}
}

func TestWindowsSetup(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping in short mode")
	}
	err := os.MkdirAll("tmp", 0755)
	must(err)
	dir, err := ioutil.TempDir("tmp", "gode")
	must(err)
	defer os.RemoveAll(dir)
	SetRootPath(dir)
	target := getWindowsTarget()
	must(target.setup())
	if setup, _ := target.isSetup(); !setup {
		t.Fatal("IsSetup() returned false")
	}
}

func must(err error) {
	if err != nil {
		log.Println(err)
		panic(err)
	}
}

func setup() {
	goreq.SetConnectTimeout(15 * time.Second)
	SetRootPath("tmp")
	must(Setup())
}

func getWindowsTarget() *Target {
	for _, t := range targets {
		if t.OS == "windows" {
			return &t
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/tls"
	"fmt"
	"net"
	"net/mail"
	"net/smtp"
	"os"
	"os/exec"
	"strings"
	"time"
)

func assemble(msg mail.Message) []byte {
	buf := new(bytes.Buffer)
	for h := range msg.Header {
		if strings.HasPrefix(h, "Yamn-") {
			Error.Printf("Ignoring internal mail header in assemble phase: %s", h)
		} else {
			buf.WriteString(h + ": " + msg.Header.Get(h) + "\n")
			//fmt.Printf("%s: %s\n", h, msg.Header.Get(h))
		}
	}
	buf.WriteString("\n")
	buf.ReadFrom(msg.Body)
	return buf.Bytes()
}

// headToAddy parses a header containing email addresses
func headToAddy(h mail.Header, header string) (addys []string) {
	_, exists := h[header]
	if !exists {
		return
	}
	addyList, err := h.AddressList(header)
	if err != nil {
		Warn.Printf("Failed to parse header: %s", header)
	}
	for _, addy := range addyList {
		addys = append(addys, addy.Address)
	}
	return
}

type emailAddress struct {
	name   string
	domain string
}

// splitAddress splits an email address into its component parts
func splitEmailAddress(addy string) (e emailAddress, err error) {
	// Email addresses must have '@' signs in them.
	if !strings.Contains(addy, "@") {
		err = fmt.Errorf("%s: Email address contains no '@'", addy)
		return
	}
	components := strings.Split(addy, "@")
	if len(components) != 2 {
		err = fmt.Errorf("%s: Malformed email address", addy)
		return
	}
	e.name = components[0]
	e.domain = components[1]
	return
}

// mxLookup returns the responsible MX for a given email address
func mxLookup(email string) (relay string, err error) {
	emailParts, err := splitEmailAddress(email)
	if err != nil {
		// Failed to ascertain domain name from email address
		return
	}
	mxRecords, err := net.LookupMX(emailParts.domain)
	if err != nil {
		relay = emailParts.domain
		Trace.Printf(
			"DNS MX lookup failed for %s.  Using hostname.",
			emailParts.domain,
		)
		err = nil
		return
	}
	for _, mx := range mxRecords {
		if !cfg.Mail.OnionRelay {
			// We don't want no onions!
			if strings.HasSuffix(mx.Host, ".onion.") {
				// Ignore the onion, find another.
				continue
			}
		}
		relay = mx.Host
		break
	}
	if relay == "" {
		// No suitable relays found, use the hostname.
		Info.Printf(
			"No valid MX records found for %s. Using hostname.",
			emailParts.domain,
		)
		relay = emailParts.domain
		return
	}
	Trace.Printf(
		"DNS lookup: Hostname=%s, MX=%s",
		emailParts.domain,
		relay,
	)
	return
}

// parseFrom takes a mail address of the format Name <name@foo> and validates
// it.  If custom From headers are not allowed, it will be tweaked to conform
// with the Remailer's configuration.
func parseFrom(h mail.Header) []string {
	from, err := h.AddressList("From")
	if err != nil {
		// The supplied address is invalid.  Use defaults instead.
		return []string{fmt.Sprintf(
			"%s <%s>",
			cfg.Mail.OutboundName,
			cfg.Mail.OutboundAddy,
		)}
	}
	if len(from) == 0 {
		// The address list is empty so return defaults
		return []string{fmt.Sprintf(
			"%s <%s>",
			cfg.Mail.OutboundName,
			cfg.Mail.OutboundAddy,
		)}
	}
	if cfg.Mail.CustomFrom {
		// Accept whatever was provided (it's already been validated by
		// AddressList).
		return []string{fmt.Sprintf(
			"%s <%s>",
			from[0].Name,
			from[0].Address,
		)}
	}
	if len(from[0].Name) == 0 {
		return []string{fmt.Sprintf(
			"%s <%s>",
			cfg.Mail.OutboundName,
			cfg.Mail.OutboundAddy,
		)}
	}
	return []string{fmt.Sprintf(
		"%s <%s>",
		from[0].Name,
		cfg.Mail.OutboundAddy,
	)}
}

// Read a file from the outbound pool and mail it
func mailPoolFile(filename string) (delFlag bool, err error) {
	// This flag implies that, by default, we don't delete pool messages
	delFlag = false

	f, err := os.Open(filename)
	if err != nil {
		Error.Printf("Failed to read file for mailing: %s", err)
		return
	}
	defer f.Close()

	msg, err := mail.ReadMessage(f)
	if err != nil {
		Error.Printf("Failed to process mail file: %s", err)
		// If we can't process it, it'll never get sent.  Mark for delete.
		delFlag = true
		return
	}

	// Test for a Pooled Date header in the message.
	pooledHeader := msg.Header.Get("Yamn-Pooled-Date")
	if pooledHeader == "" {
		// Legacy condition.  All current versions apply this header.
		Warn.Println("No Yamn-Pooled-Date header in message")
	} else {
		var pooledDate time.Time
		pooledDate, err = time.Parse(shortdate, pooledHeader)
		if err != nil {
			Error.Printf("%s: Failed to parse Yamn-Pooled-Date: %s", filename, err)
			return
		}
		age := daysAgo(pooledDate)
		if age > cfg.Pool.MaxAge {
			// The message has expired.  Give up trying to send it.
			Info.Printf(
				"%s: Refusing to mail pool file. Exceeds max age of %d days",
				filename,
				cfg.Pool.MaxAge,
			)
			// Set deletion flag.  We don't want to retain old
			// messages forever.
			delFlag = true
			return
		}
		if age > 0 {
			Trace.Printf("Mailing pooled file that's %d days old.", age)
		}
		// Delete the internal header we just tested.
		delete(msg.Header, "Yamn-Pooled-Date")
	}

	// Add some required headers to the message.
	msg.Header["Date"] = []string{time.Now().Format(rfc5322date)}
	msg.Header["Message-Id"] = []string{messageID()}
	msg.Header["From"] = parseFrom(msg.Header)
	sendTo := headToAddy(msg.Header, "To")
	sendTo = append(sendTo, headToAddy(msg.Header, "Cc")...)
	if len(sendTo) == 0 {
		err = fmt.Errorf("%s: No email recipients found", filename)
		// No point in repeatedly trying to resend a malformed file.
		delFlag = true
		return
	}
	// There is an assumption here that all errors from mailBytes should not
	// delete pool files (delFlag is false by default).
	err = mailBytes(assemble(*msg), sendTo)
	return
}

// Mail a byte payload to a given address
func mailBytes(payload []byte, sendTo []string) (err error) {
	// Test if the message is destined for the local remailer
	Trace.Printf("Message recipients are: %s", strings.Join(sendTo, ","))
	if cfg.Mail.Outfile {
		var f *os.File
		filename := randPoolFilename("outfile-")
		Trace.Printf("Writing output to %s", filename)
		f, err = os.Create(filename)
		if err != nil {
			Warn.Printf("Pool file creation failed: %s\n", err)
			return
		}
		defer f.Close()
		_, err = f.WriteString(string(payload))
		if err != nil {
			Warn.Printf("Outfile write failed: %s\n", err)
			return
		}
	} else if cfg.Mail.Pipe != "" {
		execSend(payload, cfg.Mail.Pipe)
	} else if cfg.Mail.Sendmail {
		err = sendmail(payload, sendTo)
		if err != nil {
			Warn.Println("Sendmail failed")
			return
		}
	} else {
		err = smtpRelay(payload, sendTo)
		if err != nil {
			Warn.Println("SMTP relay failed")
			return
		}
	}
	return
}

// Pipe mail to an external command (E.g. sendmail -t)
func execSend(payload []byte, execCmd string) {
	sendmail := new(exec.Cmd)
	sendmail.Args = strings.Fields(execCmd)
	sendmail.Path = sendmail.Args[0]

	stdin, err := sendmail.StdinPipe()
	if err != nil {
		panic(err)
	}
	defer stdin.Close()
	sendmail.Stdout = os.Stdout
	sendmail.Stderr = os.Stderr
	err = sendmail.Start()
	if err != nil {
		Error.Printf("%s: %s", execCmd, err)
		return
	}
	stdin.Write(payload)
	stdin.Close()
	err = sendmail.Wait()
	if err != nil {
		Error.Printf("%s: %s", execCmd, err)
		return
	}
}

func smtpRelay(payload []byte, sendTo []string) (err error) {
	conf := new(tls.Config)
	//conf.CipherSuites = []uint16{tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA}
	conf.InsecureSkipVerify = true
	//conf.MinVersion = tls.VersionSSL30
	//conf.MaxVersion = tls.VersionTLS10
	relay := cfg.Mail.SMTPRelay
	port := cfg.Mail.SMTPPort

	/*
		The following section tries to get the MX record for the
		recipient email address, when there is only a single recipient.
		If it succeeds, the email will be sent directly to the
		recipient MX.
	*/
	if cfg.Mail.MXRelay && len(sendTo) == 1 {
		Trace.Printf("DNS lookup of MX record for %s.", sendTo[0])
		mx, err := mxLookup(sendTo[0])
		if err == nil {
			Trace.Printf(
				"Doing direct relay for %s to %s:25.",
				sendTo[0],
				mx,
			)
			relay = mx
			port = 25
		}
	}
	serverAddr := fmt.Sprintf("%s:%d", relay, port)

	conn, err := net.Dial("tcp", serverAddr)
	if err != nil {
		Warn.Printf("Dial Error: Server=%s, Error=%s", serverAddr, err)
		return
	}

	client, err := smtp.NewClient(conn, relay)
	if err != nil {
		Warn.Printf(
			"SMTP Connection Error: Server=%s, Error=%s",
			serverAddr,
			err,
		)
		return
	}
	// Test if the remote MTA supports STARTTLS
	ok, _ := client.Extension("STARTTLS")
	if ok && cfg.Mail.UseTLS {
		if err = client.StartTLS(conf); err != nil {
			Warn.Printf(
				"Error performing STARTTLS: Server=%s, Error=%s",
				serverAddr,
				err,
			)
			return
		}
	}
	// If AUTH is supported and a UserID and Password are configured, try to
	// authenticate to the remote MTA.
	ok, _ = client.Extension("AUTH")
	if ok && cfg.Mail.Username != "" && cfg.Mail.Password != "" {
		auth := smtp.PlainAuth(
			"",
			cfg.Mail.Username,
			cfg.Mail.Password,
			cfg.Mail.SMTPRelay,
		)
		if err = client.Auth(auth); err != nil {
			Warn.Printf("Auth Error:  Server=%s, Error=%s", serverAddr, err)
			return
		}
	}
	// Remailer.Address is a legacy setting as clients may also need to
	// set the sender address if their ISPs MTA demands it's valid.
	// TODO remove cfg.Remailer.Address in a later version (27/04/2015)
	var sender string
	if cfg.Mail.Sender != "" {
		sender = cfg.Mail.Sender
	} else {
		sender = cfg.Remailer.Address
	}
	if err = client.Mail(sender); err != nil {
		Warn.Printf("SMTP Error: Server=%s, Error=%s", serverAddr, err)
		return
	}

	for _, addr := range sendTo {
		if err = client.Rcpt(addr); err != nil {
			Warn.Printf("Error: %s\n", err)
			return
		}
	}

	w, err := client.Data()
	if err != nil {
		Warn.Printf("Error: %s\n", err)
		return
	}

	_, err = w.Write(payload)
	if err != nil {
		Warn.Printf("Error: %s\n", err)
		return

	}

	err = w.Close()
	if err != nil {
		Warn.Printf("Error: %s\n", err)
		return

	}

	client.Quit()
	return
}

// sendmail invokes go's sendmail method
func sendmail(payload []byte, sendTo []string) (err error) {
	auth := smtp.PlainAuth(
		"",
		cfg.Mail.Username,
		cfg.Mail.Password,
		cfg.Mail.SMTPRelay)
	relay := fmt.Sprintf("%s:%d", cfg.Mail.SMTPRelay, cfg.Mail.SMTPPort)
	err = smtp.SendMail(relay, auth, cfg.Remailer.Address, sendTo, payload)
	if err != nil {
		Warn.Println(err)
		return
	}
	return
}
<commit_msg>Make ExecSend return an error variable<commit_after>package main

import (
	"bytes"
	"crypto/tls"
	"fmt"
	"net"
	"net/mail"
	"net/smtp"
	"os"
	"os/exec"
	"strings"
	"time"
)

func assemble(msg mail.Message) []byte {
	buf := new(bytes.Buffer)
	for h := range msg.Header {
		if strings.HasPrefix(h, "Yamn-") {
			Error.Printf("Ignoring internal mail header in assemble phase: %s", h)
		} else {
			buf.WriteString(h + ": " + msg.Header.Get(h) + "\n")
			//fmt.Printf("%s: %s\n", h, msg.Header.Get(h))
		}
	}
	buf.WriteString("\n")
	buf.ReadFrom(msg.Body)
	return buf.Bytes()
}

// headToAddy parses a header containing email addresses
func headToAddy(h mail.Header, header string) (addys []string) {
	_, exists := h[header]
	if !exists {
		return
	}
	addyList, err := h.AddressList(header)
	if err != nil {
		Warn.Printf("Failed to parse header: %s", header)
	}
	for _, addy := range addyList {
		addys = append(addys, addy.Address)
	}
	return
}

type emailAddress struct {
	name   string
	domain string
}

// splitAddress splits an email address into its component parts
func splitEmailAddress(addy string) (e emailAddress, err error) {
	// Email addresses must have '@' signs in them.
	if !strings.Contains(addy, "@") {
		err = fmt.Errorf("%s: Email address contains no '@'", addy)
		return
	}
	components := strings.Split(addy, "@")
	if len(components) != 2 {
		err = fmt.Errorf("%s: Malformed email address", addy)
		return
	}
	e.name = components[0]
	e.domain = components[1]
	return
}

// mxLookup returns the responsible MX for a given email address
func mxLookup(email string) (relay string, err error) {
	emailParts, err := splitEmailAddress(email)
	if err != nil {
		// Failed to ascertain domain name from email address
		return
	}
	mxRecords, err := net.LookupMX(emailParts.domain)
	if err != nil {
		relay = emailParts.domain
		Trace.Printf(
			"DNS MX lookup failed for %s.  Using hostname.",
			emailParts.domain,
		)
		err = nil
		return
	}
	for _, mx := range mxRecords {
		if !cfg.Mail.OnionRelay {
			// We don't want no onions!
			if strings.HasSuffix(mx.Host, ".onion.") {
				// Ignore the onion, find another.
				continue
			}
		}
		relay = mx.Host
		break
	}
	if relay == "" {
		// No suitable relays found, use the hostname.
		Info.Printf(
			"No valid MX records found for %s. Using hostname.",
			emailParts.domain,
		)
		relay = emailParts.domain
		return
	}
	Trace.Printf(
		"DNS lookup: Hostname=%s, MX=%s",
		emailParts.domain,
		relay,
	)
	return
}

// parseFrom takes a mail address of the format Name <name@foo> and validates
// it.  If custom From headers are not allowed, it will be tweaked to conform
// with the Remailer's configuration.
func parseFrom(h mail.Header) []string {
	from, err := h.AddressList("From")
	if err != nil {
		// The supplied address is invalid.  Use defaults instead.
		return []string{fmt.Sprintf(
			"%s <%s>",
			cfg.Mail.OutboundName,
			cfg.Mail.OutboundAddy,
		)}
	}
	if len(from) == 0 {
		// The address list is empty so return defaults
		return []string{fmt.Sprintf(
			"%s <%s>",
			cfg.Mail.OutboundName,
			cfg.Mail.OutboundAddy,
		)}
	}
	if cfg.Mail.CustomFrom {
		// Accept whatever was provided (it's already been validated by
		// AddressList).
		return []string{fmt.Sprintf(
			"%s <%s>",
			from[0].Name,
			from[0].Address,
		)}
	}
	if len(from[0].Name) == 0 {
		return []string{fmt.Sprintf(
			"%s <%s>",
			cfg.Mail.OutboundName,
			cfg.Mail.OutboundAddy,
		)}
	}
	return []string{fmt.Sprintf(
		"%s <%s>",
		from[0].Name,
		cfg.Mail.OutboundAddy,
	)}
}

// Read a file from the outbound pool and mail it
func mailPoolFile(filename string) (delFlag bool, err error) {
	// This flag implies that, by default, we don't delete pool messages
	delFlag = false

	f, err := os.Open(filename)
	if err != nil {
		Error.Printf("Failed to read file for mailing: %s", err)
		return
	}
	defer f.Close()

	msg, err := mail.ReadMessage(f)
	if err != nil {
		Error.Printf("Failed to process mail file: %s", err)
		// If we can't process it, it'll never get sent.  Mark for delete.
		delFlag = true
		return
	}

	// Test for a Pooled Date header in the message.
	pooledHeader := msg.Header.Get("Yamn-Pooled-Date")
	if pooledHeader == "" {
		// Legacy condition.  All current versions apply this header.
		Warn.Println("No Yamn-Pooled-Date header in message")
	} else {
		var pooledDate time.Time
		pooledDate, err = time.Parse(shortdate, pooledHeader)
		if err != nil {
			Error.Printf("%s: Failed to parse Yamn-Pooled-Date: %s", filename, err)
			return
		}
		age := daysAgo(pooledDate)
		if age > cfg.Pool.MaxAge {
			// The message has expired.  Give up trying to send it.
			Info.Printf(
				"%s: Refusing to mail pool file. Exceeds max age of %d days",
				filename,
				cfg.Pool.MaxAge,
			)
			// Set deletion flag.  We don't want to retain old
			// messages forever.
			delFlag = true
			return
		}
		if age > 0 {
			Trace.Printf("Mailing pooled file that's %d days old.", age)
		}
		// Delete the internal header we just tested.
		delete(msg.Header, "Yamn-Pooled-Date")
	}

	// Add some required headers to the message.
	msg.Header["Date"] = []string{time.Now().Format(rfc5322date)}
	msg.Header["Message-Id"] = []string{messageID()}
	msg.Header["From"] = parseFrom(msg.Header)
	sendTo := headToAddy(msg.Header, "To")
	sendTo = append(sendTo, headToAddy(msg.Header, "Cc")...)
	if len(sendTo) == 0 {
		err = fmt.Errorf("%s: No email recipients found", filename)
		// No point in repeatedly trying to resend a malformed file.
		delFlag = true
		return
	}
	// There is an assumption here that all errors from mailBytes should not
	// delete pool files (delFlag is false by default).
	err = mailBytes(assemble(*msg), sendTo)
	return
}

// Mail a byte payload to a given address
func mailBytes(payload []byte, sendTo []string) (err error) {
	// Test if the message is destined for the local remailer
	Trace.Printf("Message recipients are: %s", strings.Join(sendTo, ","))
	if cfg.Mail.Outfile {
		var f *os.File
		filename := randPoolFilename("outfile-")
		Trace.Printf("Writing output to %s", filename)
		f, err = os.Create(filename)
		if err != nil {
			Warn.Printf("Pool file creation failed: %s\n", err)
			return
		}
		defer f.Close()
		_, err = f.WriteString(string(payload))
		if err != nil {
			Warn.Printf("Outfile write failed: %s\n", err)
			return
		}
	} else if cfg.Mail.Pipe != "" {
		err = execSend(payload, cfg.Mail.Pipe)
		if err != nil {
			Warn.Println("Email pipe failed")
			return
		}
	} else if cfg.Mail.Sendmail {
		err = sendmail(payload, sendTo)
		if err != nil {
			Warn.Println("Sendmail failed")
			return
		}
	} else {
		err = smtpRelay(payload, sendTo)
		if err != nil {
			Warn.Println("SMTP relay failed")
			return
		}
	}
	return
}

// Pipe mail to an external command (E.g. sendmail -t)
func execSend(payload []byte, execCmd string) (err error) {
	sendmail := new(exec.Cmd)
	sendmail.Args = strings.Fields(execCmd)
	sendmail.Path = sendmail.Args[0]

	stdin, err := sendmail.StdinPipe()
	if err != nil {
		Error.Printf("%s: %s", execCmd, err)
		return
	}
	defer stdin.Close()
	sendmail.Stdout = os.Stdout
	sendmail.Stderr = os.Stderr
	err = sendmail.Start()
	if err != nil {
		Error.Printf("%s: %s", execCmd, err)
		return
	}
	stdin.Write(payload)
	stdin.Close()
	err = sendmail.Wait()
	if err != nil {
		Error.Printf("%s: %s", execCmd, err)
		return
	}
	return
}

func smtpRelay(payload []byte, sendTo []string) (err error) {
	conf := new(tls.Config)
	//conf.CipherSuites = []uint16{tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA}
	conf.InsecureSkipVerify = true
	//conf.MinVersion = tls.VersionSSL30
	//conf.MaxVersion = tls.VersionTLS10
	relay := cfg.Mail.SMTPRelay
	port := cfg.Mail.SMTPPort

	/*
		The following section tries to get the MX record for the
		recipient email address, when there is only a single recipient.
		If it succeeds, the email will be sent directly to the
		recipient MX.
	*/
	if cfg.Mail.MXRelay && len(sendTo) == 1 {
		Trace.Printf("DNS lookup of MX record for %s.", sendTo[0])
		mx, err := mxLookup(sendTo[0])
		if err == nil {
			Trace.Printf(
				"Doing direct relay for %s to %s:25.",
				sendTo[0],
				mx,
			)
			relay = mx
			port = 25
		}
	}
	serverAddr := fmt.Sprintf("%s:%d", relay, port)

	conn, err := net.Dial("tcp", serverAddr)
	if err != nil {
		Warn.Printf("Dial Error: Server=%s, Error=%s", serverAddr, err)
		return
	}

	client, err := smtp.NewClient(conn, relay)
	if err != nil {
		Warn.Printf(
			"SMTP Connection Error: Server=%s, Error=%s",
			serverAddr,
			err,
		)
		return
	}
	// Test if the remote MTA supports STARTTLS
	ok, _ := client.Extension("STARTTLS")
	if ok && cfg.Mail.UseTLS {
		if err = client.StartTLS(conf); err != nil {
			Warn.Printf(
				"Error performing STARTTLS: Server=%s, Error=%s",
				serverAddr,
				err,
			)
			return
		}
	}
	// If AUTH is supported and a UserID and Password are configured, try to
	// authenticate to the remote MTA.
	ok, _ = client.Extension("AUTH")
	if ok && cfg.Mail.Username != "" && cfg.Mail.Password != "" {
		auth := smtp.PlainAuth(
			"",
			cfg.Mail.Username,
			cfg.Mail.Password,
			cfg.Mail.SMTPRelay,
		)
		if err = client.Auth(auth); err != nil {
			Warn.Printf("Auth Error:  Server=%s, Error=%s", serverAddr, err)
			return
		}
	}
	// Remailer.Address is a legacy setting as clients may also need to
	// set the sender address if their ISPs MTA demands it's valid.
	// TODO remove cfg.Remailer.Address in a later version (27/04/2015)
	var sender string
	if cfg.Mail.Sender != "" {
		sender = cfg.Mail.Sender
	} else {
		sender = cfg.Remailer.Address
	}
	if err = client.Mail(sender); err != nil {
		Warn.Printf("SMTP Error: Server=%s, Error=%s", serverAddr, err)
		return
	}

	for _, addr := range sendTo {
		if err = client.Rcpt(addr); err != nil {
			Warn.Printf("Error: %s\n", err)
			return
		}
	}

	w, err := client.Data()
	if err != nil {
		Warn.Printf("Error: %s\n", err)
		return
	}

	_, err = w.Write(payload)
	if err != nil {
		Warn.Printf("Error: %s\n", err)
		return

	}

	err = w.Close()
	if err != nil {
		Warn.Printf("Error: %s\n", err)
		return

	}

	client.Quit()
	return
}

// sendmail invokes go's sendmail method
func sendmail(payload []byte, sendTo []string) (err error) {
	auth := smtp.PlainAuth(
		"",
		cfg.Mail.Username,
		cfg.Mail.Password,
		cfg.Mail.SMTPRelay)
	relay := fmt.Sprintf("%s:%d", cfg.Mail.SMTPRelay, cfg.Mail.SMTPPort)
	err = smtp.SendMail(relay, auth, cfg.Remailer.Address, sendTo, payload)
	if err != nil {
		Warn.Println(err)
		return
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"time"
)
import "github.com/kr/beanstalk"

func main() {
	fmt.Println("hello gobean")

	c, err := beanstalk.Dial("tcp", "127.0.0.1:11300")
	if err != nil {
		panic(err)
	}

	for {
		id, body, err := c.Reserve(24 * time.Hour)
		fmt.Println(id, string(body), err)
		c.Delete(id)
	}
}
<commit_msg>gobean --tubes=one,two,three<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"strings"
	"time"
)
import "github.com/kr/beanstalk"

func main() {
	tubes := tubesFlag()

	c, err := beanstalk.Dial("tcp", "127.0.0.1:11300")
	if err != nil {
		panic(err)
	}

	log.Println("watching", tubes)
	ts := beanstalk.NewTubeSet(c, tubes...)

	for {
		id, body, err := ts.Reserve(24 * time.Hour)
		fmt.Println(id, string(body), err)
		ts.Conn.Delete(id)
	}
}

func tubesFlag() []string {
	var raw string
	flag.StringVar(&raw, "tubes", "default", "Comma-separated list of tubes to watch")
	flag.Parse()
	return strings.Split(raw, ",")
}
<|endoftext|>


<commit_before>// Package goczmq is a golang binding for CZMQ 3. CZMQ is a high level binding
// for ZeroMQ. Along with ZeroMQ socket support, CZMQ provides "actor" based
// services for authentication, service discovery, and creating proxies.
// GoCZMQ provides direct bindings to CZMQ along with higher level go
// abstractions such as channels and io.ReadWriter interface support.
package goczmq

/*
#cgo !windows pkg-config: libczmq libzmq libsodium
#cgo windows LDFLAGS: -lws2_32 -liphlpapi -lrpcrt4 -lsodium -lzmq -lczmq
#cgo windows CFLAGS: -Wno-pedantic-ms-format -DLIBCZMQ_EXPORTS -DZMQ_DEFINED_STDINT -DLIBCZMQ_EXPORTS

#include "czmq.h"
#include <stdlib.h>
#include <string.h>
*/
import "C"

import (
	"errors"
)

const (
	Req    = int(C.ZMQ_REQ)
	Rep    = int(C.ZMQ_REP)
	Dealer = int(C.ZMQ_DEALER)
	Router = int(C.ZMQ_ROUTER)
	Pub    = int(C.ZMQ_PUB)
	Sub    = int(C.ZMQ_SUB)
	XPub   = int(C.ZMQ_XPUB)
	XSub   = int(C.ZMQ_XSUB)
	Push   = int(C.ZMQ_PUSH)
	Pull   = int(C.ZMQ_PULL)
	Pair   = int(C.ZMQ_PAIR)
	Stream = int(C.ZMQ_STREAM)

	Pollin  = int(C.ZMQ_POLLIN)
	Pollout = int(C.ZMQ_POLLOUT)

	FlagMore     = int(C.ZFRAME_MORE)
	FlagReuse    = int(C.ZFRAME_REUSE)
	FlagDontWait = int(C.ZFRAME_DONTWAIT)
	FlagNone     = 0

	CurveAllowAny = "*"
)

var (
	ErrActorCmd        = errors.New("error sending actor command")
	ErrSockAttach      = errors.New("error attaching zsock")
	ErrInvalidSockType = errors.New("invalid socket type")
)

func getStringType(k int) string {
	switch k {
	case Req:
		return "REQ"
	case Rep:
		return "REP"
	case Dealer:
		return "DEALER"
	case Router:
		return "ROUTER"
	case Pub:
		return "PUB"
	case Sub:
		return "SUB"
	case XPub:
		return "XPUB"
	case XSub:
		return "XSUB"
	case Push:
		return "PUSH"
	case Pull:
		return "PULL"
	case Pair:
		return "PAIR"
	case Stream:
		return "STREAM"
	default:
		return ""
	}
}
<commit_msg>add constants for zmq and czmq versions<commit_after>// Package goczmq is a golang binding for CZMQ 3. CZMQ is a high level binding
// for ZeroMQ. Along with ZeroMQ socket support, CZMQ provides "actor" based
// services for authentication, service discovery, and creating proxies.
// GoCZMQ provides direct bindings to CZMQ along with higher level go
// abstractions such as channels and io.ReadWriter interface support.
package goczmq

/*
#cgo !windows pkg-config: libczmq libzmq libsodium
#cgo windows LDFLAGS: -lws2_32 -liphlpapi -lrpcrt4 -lsodium -lzmq -lczmq
#cgo windows CFLAGS: -Wno-pedantic-ms-format -DLIBCZMQ_EXPORTS -DZMQ_DEFINED_STDINT -DLIBCZMQ_EXPORTS

#include "czmq.h"
#include <stdlib.h>
#include <string.h>
*/
import "C"

import (
	"errors"
)

const (
	Req    = int(C.ZMQ_REQ)
	Rep    = int(C.ZMQ_REP)
	Dealer = int(C.ZMQ_DEALER)
	Router = int(C.ZMQ_ROUTER)
	Pub    = int(C.ZMQ_PUB)
	Sub    = int(C.ZMQ_SUB)
	XPub   = int(C.ZMQ_XPUB)
	XSub   = int(C.ZMQ_XSUB)
	Push   = int(C.ZMQ_PUSH)
	Pull   = int(C.ZMQ_PULL)
	Pair   = int(C.ZMQ_PAIR)
	Stream = int(C.ZMQ_STREAM)

	Pollin  = int(C.ZMQ_POLLIN)
	Pollout = int(C.ZMQ_POLLOUT)

	FlagMore     = int(C.ZFRAME_MORE)
	FlagReuse    = int(C.ZFRAME_REUSE)
	FlagDontWait = int(C.ZFRAME_DONTWAIT)
	FlagNone     = 0

	CurveAllowAny = "*"

	ZMQVersionMajor  = int(C.ZMQ_VERSION_MAJOR)
	ZMQVersionMinor  = int(C.ZMQ_VERSION_MINOR)
	CZMQVersionMajor = int(C.CZMQ_VERSION_MAJOR)
	CZMQVersionMinor = int(C.CZMQ_VERSION_MINOR)
)

var (
	ErrActorCmd        = errors.New("error sending actor command")
	ErrSockAttach      = errors.New("error attaching zsock")
	ErrInvalidSockType = errors.New("invalid socket type")
)

func getStringType(k int) string {
	switch k {
	case Req:
		return "REQ"
	case Rep:
		return "REP"
	case Dealer:
		return "DEALER"
	case Router:
		return "ROUTER"
	case Pub:
		return "PUB"
	case Sub:
		return "SUB"
	case XPub:
		return "XPUB"
	case XSub:
		return "XSUB"
	case Push:
		return "PUSH"
	case Pull:
		return "PULL"
	case Pair:
		return "PAIR"
	case Stream:
		return "STREAM"
	default:
		return ""
	}
}
<|endoftext|>