<commit_before>// query_benchmarker speed tests Graphite using requests from stdin.
//
// It reads encoded Query objects from stdin, and makes concurrent requests
// to the provided HTTP endpoint. This program has no knowledge of the
// internals of the endpoint.
package main

import (
	"encoding/gob"
	"flag"
	"fmt"
	"github.com/influxdata/influxdb-comparisons/bulk_query"
	"github.com/influxdata/influxdb-comparisons/bulk_query/http"
	"github.com/influxdata/influxdb-comparisons/util/report"
	"io"
	"log"
	"sync"
	"time"
)

// Program option vars:
type GraphiteQueryBenchmarker struct {
	daemonUrl     string

	dialTimeout        time.Duration
	readTimeout        time.Duration
	writeTimeout       time.Duration
	httpClientType     string
	scanFinished       bool

	queryPool sync.Pool
	queryChan chan []*http.Query
}

var querier = &GraphiteQueryBenchmarker{}

// Parse args:
func init() {

	bulk_query.Benchmarker.Init()
	querier.Init()

	flag.Parse()

	bulk_query.Benchmarker.Validate()
	querier.Validate()

}

func (b *GraphiteQueryBenchmarker) Init() {
	flag.StringVar(&b.daemonUrl, "urls", "http://localhost:8080", "Graphite URL.")
	flag.DurationVar(&b.dialTimeout, "dial-timeout", time.Second*15, "TCP dial timeout.")
	flag.DurationVar(&b.readTimeout, "write-timeout", time.Second*300, "TCP write timeout.")
	flag.DurationVar(&b.writeTimeout, "read-timeout", time.Second*300, "TCP read timeout.")
	flag.StringVar(&b.httpClientType, "http-client-type", "fast", "HTTP client type {fast, default}")
}

func (b *GraphiteQueryBenchmarker) Validate() {
	fmt.Printf("Graphite URL: %v\n", b)

	if b.httpClientType == "fast" || b.httpClientType == "default" {
		fmt.Printf("Using HTTP client: %v\n", b.httpClientType)
		http.UseFastHttp = b.httpClientType == "fast"
	} else {
		log.Fatalf("Unsupported HTPP client type: %v", b.httpClientType)
	}
}

func (b *GraphiteQueryBenchmarker) Prepare() {
	// Make pools to minimize heap usage:
	b.queryPool = sync.Pool{
		New: func() interface{} {
			return &http.Query{
				HumanLabel:       make([]byte, 0, 1024),
				HumanDescription: make([]byte, 0, 1024),
				Method:           make([]byte, 0, 1024),
				Path:             make([]byte, 0, 1024),
				Body:             make([]byte, 0, 1024),
			}
		},
	}

	// Make data and control channels:
	b.queryChan = make(chan []*http.Query)
}

func (b *GraphiteQueryBenchmarker) GetProcessor() bulk_query.Processor {
	return b
}
func (b *GraphiteQueryBenchmarker) GetScanner() bulk_query.Scanner {
	return b
}

func (b *GraphiteQueryBenchmarker) PrepareProcess(i int) {
}

func (b *GraphiteQueryBenchmarker) RunProcess(i int, workersGroup *sync.WaitGroup, statPool sync.Pool, statChan chan *bulk_query.Stat) {
	w := http.NewHTTPClient(b.daemonUrl, bulk_query.Benchmarker.Debug(), b.dialTimeout, b.readTimeout, b.writeTimeout)
	b.processQueries(w, workersGroup, statPool, statChan)
}

func (b *GraphiteQueryBenchmarker) IsScanFinished() bool {
	return b.scanFinished
}

func (b *GraphiteQueryBenchmarker) CleanUp() {
	close(b.queryChan)
}

func (b GraphiteQueryBenchmarker) UpdateReport(params *report.QueryReportParams, reportTags [][2]string, extraVals []report.ExtraVal) (updatedTags [][2]string, updatedExtraVals []report.ExtraVal) {
	params.DBType = "Graphite"
	params.DestinationUrl = b.daemonUrl
	updatedTags = reportTags
	updatedExtraVals = extraVals
	return
}

func main() {
	bulk_query.Benchmarker.RunBenchmark(querier)
}

var qind int64

// scan reads encoded Queries and places them onto the workqueue.
func (b *GraphiteQueryBenchmarker) RunScan(r io.Reader, closeChan chan int) {
	dec := gob.NewDecoder(r)

	batch := make([]*http.Query, 0, bulk_query.Benchmarker.BatchSize())

	i := 0
loop:
	for {
		if bulk_query.Benchmarker.Limit() >= 0 && qind >= bulk_query.Benchmarker.Limit() {
			break
		}

		q := b.queryPool.Get().(*http.Query)
		err := dec.Decode(q)
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Fatal(err)
		}

		q.ID = qind
		batch = append(batch, q)
		i++
		if i == bulk_query.Benchmarker.BatchSize() {
			b.queryChan <- batch
			//batch = batch[:0]
			batch = nil
			batch = make([]*http.Query, 0, bulk_query.Benchmarker.BatchSize())
			i = 0
		}

		qind++
		select {
		case <-closeChan:
			log.Println("Received finish request")
			break loop
		default:
		}

	}
	b.scanFinished = true
}

// processQueries reads byte buffers from queryChan and writes them to the
// target server, while tracking latency.
func (b *GraphiteQueryBenchmarker) processQueries(w http.HTTPClient, workersGroup *sync.WaitGroup, statPool sync.Pool, statChan chan *bulk_query.Stat) error {
	opts := &http.HTTPClientDoOptions{
		Debug:                bulk_query.Benchmarker.Debug(),
		PrettyPrintResponses: bulk_query.Benchmarker.PrettyPrintResponses(),
	}
	var queriesSeen int64
	for queries := range b.queryChan {
		if len(queries) == 1 {
			if err := b.processSingleQuery(w, queries[0], opts, nil, nil, statPool, statChan); err != nil {
				log.Fatal(err)
			}
			queriesSeen++
		} else {
			var err error
			errors := 0
			done := 0
			errCh := make(chan error)
			doneCh := make(chan int, len(queries))
			for _, q := range queries {
				go b.processSingleQuery(w, q, opts, errCh, doneCh, statPool, statChan)
				queriesSeen++
			}

		loop:
			for {
				select {
				case err = <-errCh:
					errors++
				case <-doneCh:
					done++
					if done == len(queries) {
						break loop
					}
				}
			}
			close(errCh)
			close(doneCh)
			if err != nil {
				log.Fatal(err)
			}
		}
		if bulk_query.Benchmarker.WaitInterval().Seconds() > 0 {
			time.Sleep(bulk_query.Benchmarker.WaitInterval())
		}
	}
	workersGroup.Done()
	return nil
}

func (b *GraphiteQueryBenchmarker) processSingleQuery(w http.HTTPClient, q *http.Query, opts *http.HTTPClientDoOptions, errCh chan error, doneCh chan int, statPool sync.Pool, statChan chan *bulk_query.Stat) error {
	defer func() {
		if doneCh != nil {
			doneCh <- 1
		}
	}()
	lagMillis, err := w.Do(q, opts)
	stat := statPool.Get().(*bulk_query.Stat)
	stat.Init(q.HumanLabel, lagMillis)
	statChan <- stat
	b.queryPool.Put(q)
	if err != nil {
		qerr := fmt.Errorf("Error during request of query %s: %s\n", q.String(), err.Error())
		if errCh != nil {
			errCh <- qerr
			return nil
		} else {
			return qerr
		}
	}

	return nil
}
<commit_msg>fixing log message<commit_after>// query_benchmarker speed tests Graphite using requests from stdin.
//
// It reads encoded Query objects from stdin, and makes concurrent requests
// to the provided HTTP endpoint. This program has no knowledge of the
// internals of the endpoint.
package main

import (
	"encoding/gob"
	"flag"
	"fmt"
	"github.com/influxdata/influxdb-comparisons/bulk_query"
	"github.com/influxdata/influxdb-comparisons/bulk_query/http"
	"github.com/influxdata/influxdb-comparisons/util/report"
	"io"
	"log"
	"sync"
	"time"
)

// Program option vars:
type GraphiteQueryBenchmarker struct {
	daemonUrl     string

	dialTimeout        time.Duration
	readTimeout        time.Duration
	writeTimeout       time.Duration
	httpClientType     string
	scanFinished       bool

	queryPool sync.Pool
	queryChan chan []*http.Query
}

var querier = &GraphiteQueryBenchmarker{}

// Parse args:
func init() {

	bulk_query.Benchmarker.Init()
	querier.Init()

	flag.Parse()

	bulk_query.Benchmarker.Validate()
	querier.Validate()

}

func (b *GraphiteQueryBenchmarker) Init() {
	flag.StringVar(&b.daemonUrl, "urls", "http://localhost:8080", "Graphite URL.")
	flag.DurationVar(&b.dialTimeout, "dial-timeout", time.Second*15, "TCP dial timeout.")
	flag.DurationVar(&b.readTimeout, "write-timeout", time.Second*300, "TCP write timeout.")
	flag.DurationVar(&b.writeTimeout, "read-timeout", time.Second*300, "TCP read timeout.")
	flag.StringVar(&b.httpClientType, "http-client-type", "fast", "HTTP client type {fast, default}")
}

func (b *GraphiteQueryBenchmarker) Validate() {
	fmt.Printf("Graphite URL: %v\n", b.daemonUrl)

	if b.httpClientType == "fast" || b.httpClientType == "default" {
		fmt.Printf("Using HTTP client: %v\n", b.httpClientType)
		http.UseFastHttp = b.httpClientType == "fast"
	} else {
		log.Fatalf("Unsupported HTPP client type: %v", b.httpClientType)
	}
}

func (b *GraphiteQueryBenchmarker) Prepare() {
	// Make pools to minimize heap usage:
	b.queryPool = sync.Pool{
		New: func() interface{} {
			return &http.Query{
				HumanLabel:       make([]byte, 0, 1024),
				HumanDescription: make([]byte, 0, 1024),
				Method:           make([]byte, 0, 1024),
				Path:             make([]byte, 0, 1024),
				Body:             make([]byte, 0, 1024),
			}
		},
	}

	// Make data and control channels:
	b.queryChan = make(chan []*http.Query)
}

func (b *GraphiteQueryBenchmarker) GetProcessor() bulk_query.Processor {
	return b
}
func (b *GraphiteQueryBenchmarker) GetScanner() bulk_query.Scanner {
	return b
}

func (b *GraphiteQueryBenchmarker) PrepareProcess(i int) {
}

func (b *GraphiteQueryBenchmarker) RunProcess(i int, workersGroup *sync.WaitGroup, statPool sync.Pool, statChan chan *bulk_query.Stat) {
	w := http.NewHTTPClient(b.daemonUrl, bulk_query.Benchmarker.Debug(), b.dialTimeout, b.readTimeout, b.writeTimeout)
	b.processQueries(w, workersGroup, statPool, statChan)
}

func (b *GraphiteQueryBenchmarker) IsScanFinished() bool {
	return b.scanFinished
}

func (b *GraphiteQueryBenchmarker) CleanUp() {
	close(b.queryChan)
}

func (b GraphiteQueryBenchmarker) UpdateReport(params *report.QueryReportParams, reportTags [][2]string, extraVals []report.ExtraVal) (updatedTags [][2]string, updatedExtraVals []report.ExtraVal) {
	params.DBType = "Graphite"
	params.DestinationUrl = b.daemonUrl
	updatedTags = reportTags
	updatedExtraVals = extraVals
	return
}

func main() {
	bulk_query.Benchmarker.RunBenchmark(querier)
}

var qind int64

// scan reads encoded Queries and places them onto the workqueue.
func (b *GraphiteQueryBenchmarker) RunScan(r io.Reader, closeChan chan int) {
	dec := gob.NewDecoder(r)

	batch := make([]*http.Query, 0, bulk_query.Benchmarker.BatchSize())

	i := 0
loop:
	for {
		if bulk_query.Benchmarker.Limit() >= 0 && qind >= bulk_query.Benchmarker.Limit() {
			break
		}

		q := b.queryPool.Get().(*http.Query)
		err := dec.Decode(q)
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Fatal(err)
		}

		q.ID = qind
		batch = append(batch, q)
		i++
		if i == bulk_query.Benchmarker.BatchSize() {
			b.queryChan <- batch
			//batch = batch[:0]
			batch = nil
			batch = make([]*http.Query, 0, bulk_query.Benchmarker.BatchSize())
			i = 0
		}

		qind++
		select {
		case <-closeChan:
			log.Println("Received finish request")
			break loop
		default:
		}

	}
	b.scanFinished = true
}

// processQueries reads byte buffers from queryChan and writes them to the
// target server, while tracking latency.
func (b *GraphiteQueryBenchmarker) processQueries(w http.HTTPClient, workersGroup *sync.WaitGroup, statPool sync.Pool, statChan chan *bulk_query.Stat) error {
	opts := &http.HTTPClientDoOptions{
		Debug:                bulk_query.Benchmarker.Debug(),
		PrettyPrintResponses: bulk_query.Benchmarker.PrettyPrintResponses(),
	}
	var queriesSeen int64
	for queries := range b.queryChan {
		if len(queries) == 1 {
			if err := b.processSingleQuery(w, queries[0], opts, nil, nil, statPool, statChan); err != nil {
				log.Fatal(err)
			}
			queriesSeen++
		} else {
			var err error
			errors := 0
			done := 0
			errCh := make(chan error)
			doneCh := make(chan int, len(queries))
			for _, q := range queries {
				go b.processSingleQuery(w, q, opts, errCh, doneCh, statPool, statChan)
				queriesSeen++
			}

		loop:
			for {
				select {
				case err = <-errCh:
					errors++
				case <-doneCh:
					done++
					if done == len(queries) {
						break loop
					}
				}
			}
			close(errCh)
			close(doneCh)
			if err != nil {
				log.Fatal(err)
			}
		}
		if bulk_query.Benchmarker.WaitInterval().Seconds() > 0 {
			time.Sleep(bulk_query.Benchmarker.WaitInterval())
		}
	}
	workersGroup.Done()
	return nil
}

func (b *GraphiteQueryBenchmarker) processSingleQuery(w http.HTTPClient, q *http.Query, opts *http.HTTPClientDoOptions, errCh chan error, doneCh chan int, statPool sync.Pool, statChan chan *bulk_query.Stat) error {
	defer func() {
		if doneCh != nil {
			doneCh <- 1
		}
	}()
	lagMillis, err := w.Do(q, opts)
	stat := statPool.Get().(*bulk_query.Stat)
	stat.Init(q.HumanLabel, lagMillis)
	statChan <- stat
	b.queryPool.Put(q)
	if err != nil {
		qerr := fmt.Errorf("Error during request of query %s: %s\n", q.String(), err.Error())
		if errCh != nil {
			errCh <- qerr
			return nil
		} else {
			return qerr
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2014 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package unit

import (
	"reflect"
	"testing"
)

func TestUnitHash(t *testing.T) {
	u, err := NewUnitFile("[Service]\nExecStart=/bin/sleep 100\n")
	if err != nil {
		t.Fatalf("Unexpected error encountered creating unit: %v", err)
	}

	gotHash := u.Hash()
	gotHashString := gotHash.String()
	expectHashString := "1c6fb6f3684bafb0c173d8b8b957ceff031180c1"
	if gotHashString != expectHashString {
		t.Fatalf("Unit Hash (%s) does not match expected (%s)", gotHashString, expectHashString)
	}

	expectHashShort := "1c6fb6f"
	gotHashShort := gotHash.Short()
	if gotHashShort != expectHashShort {
		t.Fatalf("Unit Hash short (%s) does not match expected (%s)", gotHashShort, expectHashShort)
	}

	eh := &Hash{}
	if !eh.Empty() {
		t.Fatalf("Empty hash check failed: %v", eh.Empty())
	}
}

func TestHashFromHexString(t *testing.T) {
	u, err := NewUnitFile("[Service]\nExecStart=/bin/sleep 100\n")
	if err != nil {
		t.Fatalf("Unexpected error encountered creating unit: %v", err)
	}
	gotHash := u.Hash()

	expectHashString := "1c6fb6f3684bafb0c173d8b8b957ceff031180c1"
	rehashed, err := HashFromHexString(expectHashString)
	if err != nil {
		t.Fatalf("HashFromHexString failed with: %v", err)
	}
	if rehashed != gotHash {
		t.Fatalf("HashFromHexString not equal to original hash")
	}
}

func TestRecognizedUnitTypes(t *testing.T) {
	tts := []struct {
		name string
		ok   bool
	}{
		{"foo.service", true},
		{"foo.socket", true},
		{"foo.path", true},
		{"foo.timer", true},
		{"foo.mount", true},
		{"foo.automount", true},
		{"foo.device", true},
		{"foo.swap", false},
		{"foo.target", false},
		{"foo.snapshot", false},
		{"foo.network", false},
		{"foo.netdev", false},
		{"foo.link", false},
		{"foo.unknown", false},
	}

	for _, tt := range tts {
		ok := RecognizedUnitType(tt.name)
		if ok != tt.ok {
			t.Errorf("Case failed: name=%s expect=%t result=%t", tt.name, tt.ok, ok)
		}
	}
}

func TestDefaultUnitType(t *testing.T) {
	tts := []struct {
		name string
		out  string
	}{
		{"foo", "foo.service"},
		{"foo.service", "foo.service.service"},
		{"foo.link", "foo.link.service"},
	}

	for _, tt := range tts {
		out := DefaultUnitType(tt.name)
		if out != tt.out {
			t.Errorf("Case failed: name=%s expect=%s result=%s", tt.name, tt.out, out)
		}
	}
}

func TestNewUnitState(t *testing.T) {
	want := &UnitState{
		LoadState:   "ls",
		ActiveState: "as",
		SubState:    "ss",
		MachineID:   "id",
	}

	got := NewUnitState("ls", "as", "ss", "id")
	if !reflect.DeepEqual(got, want) {
		t.Fatalf("NewUnitState did not create a correct UnitState: got %s, want %s", got, want)
	}

}

func TestNamedUnit(t *testing.T) {
	tts := []struct {
		fname  string
		name   string
		pref   string
		tmpl   string
		inst   string
		isinst bool
		istmpl bool
	}{
		{"foo.service", "foo", "foo", "", "", false, false},
		{"foo@.service", "foo@", "foo", "foo@.service", "", false, true},
		{"foo@bar.service", "foo@bar", "foo", "foo@.service", "bar", true, false},
		{"foo@bar@baz.service", "foo@bar@baz", "foo", "foo@.service", "bar@baz", true, false},
		{"foo.1@.service", "foo.1@", "foo.1", "foo.1@.service", "", false, true},
		{"foo.1@2.service", "foo.1@2", "foo.1", "foo.1@.service", "2", true, false},
		{"ssh@.socket", "ssh@", "ssh", "ssh@.socket", "", false, true},
		{"ssh@1.socket", "ssh@1", "ssh", "ssh@.socket", "1", true, false},
	}
	for _, tt := range tts {
		u := NewUnitNameInfo(tt.fname)
		if u == nil {
			t.Errorf("NewUnitNameInfo(%s) returned nil InstanceUnit!", tt.name)
			continue
		}
		if u.FullName != tt.fname {
			t.Errorf("NewUnitNameInfo(%s) returned bad fullname: got %s, want %s", tt.name, u.FullName, tt.fname)
		}
		if u.Name != tt.name {
			t.Errorf("NewUnitNameInfo(%s) returned bad name: got %s, want %s", tt.name, u.Name, tt.name)
		}
		if u.Template != tt.tmpl {
			t.Errorf("NewUnitNameInfo(%s) returned bad template name: got %s, want %s", tt.name, u.Template, tt.tmpl)
		}
		if u.Prefix != tt.pref {
			t.Errorf("NewUnitNameInfo(%s) returned bad prefix name: got %s, want %s", tt.name, u.Prefix, tt.pref)
		}
		if u.Instance != tt.inst {
			t.Errorf("NewUnitNameInfo(%s) returned bad instance name: got %s, want %s", tt.name, u.Instance, tt.inst)
		}
		i := u.IsInstance()
		if i != tt.isinst {
			t.Errorf("NewUnitNameInfo(%s).IsInstance returned %t, want %t", tt.name, i, tt.isinst)
		}
		i = u.IsTemplate()
		if i != tt.istmpl {
			t.Errorf("NewUnitNameInfo(%s).IsTemplate returned %t, want %t", tt.name, i, tt.istmpl)
		}
	}

	bad := []string{"foo", "bar@baz"}
	for _, tt := range bad {
		if NewUnitNameInfo(tt) != nil {
			t.Errorf("NewUnitNameInfo returned non-nil InstanceUnit unexpectedly")
		}
	}

}

func TestDeserialize(t *testing.T) {
	contents := `
This=Ignored
[Unit]
;ignore this guy
Description = Foo

[Service]
ExecStart=echo "ping";
ExecStop=echo "pong"
# ignore me, too
ExecStop=echo post

[X-Fleet]
MachineMetadata=foo=bar
MachineMetadata=baz=qux
`

	expected := map[string]map[string][]string{
		"Unit": {
			"Description": {"Foo"},
		},
		"Service": {
			"ExecStart": {`echo "ping";`},
			"ExecStop":  {`echo "pong"`, "echo post"},
		},
		"X-Fleet": {
			"MachineMetadata": {"foo=bar", "baz=qux"},
		},
	}

	unitFile, err := NewUnitFile(contents)
	if err != nil {
		t.Fatalf("Unexpected error parsing unit %q: %v", contents, err)
	}

	if !reflect.DeepEqual(expected, unitFile.Contents) {
		t.Fatalf("Map func did not produce expected output.\nActual=%v\nExpected=%v", unitFile.Contents, expected)
	}
}

func TestDeserializedUnitGarbage(t *testing.T) {
	contents := `
>>>>>>>>>>>>>
[Service]
ExecStart=jim
# As long as a line has an equals sign, systemd is happy, so we should pass it through
<<<<<<<<<<<=bar
`
	expected := map[string]map[string][]string{
		"Service": {
			"ExecStart":   {"jim"},
			"<<<<<<<<<<<": {"bar"},
		},
	}
	unitFile, err := NewUnitFile(contents)
	if err != nil {
		t.Fatalf("Unexpected error parsing unit %q: %v", contents, err)
	}

	if !reflect.DeepEqual(expected, unitFile.Contents) {
		t.Fatalf("Map func did not produce expected output.\nActual=%v\nExpected=%v", unitFile.Contents, expected)
	}
}

func TestDeserializeEscapedMultilines(t *testing.T) {
	contents := `
[Service]
ExecStart=echo \
  "pi\
  ng"
ExecStop=\
echo "po\
ng"
# comments within continuation should not be ignored
ExecStopPre=echo\
#pang
ExecStopPost=echo\
#peng\
pung
`
	expected := map[string]map[string][]string{
		"Service": {
			"ExecStart": {`echo \
  "pi\
  ng"`},
			"ExecStop": {`\
echo "po\
ng"`},
			"ExecStopPre": {`echo\
#pang`},
			"ExecStopPost": {`echo\
#peng\
pung`},
		},
	}
	unitFile, err := NewUnitFile(contents)
	if err != nil {
		t.Fatalf("Unexpected error parsing unit %q: %v", contents, err)
	}

	if !reflect.DeepEqual(expected, unitFile.Contents) {
		t.Fatalf("Map func did not produce expected output.\nActual=%v\nExpected=%v", unitFile.Contents, expected)
	}
}

func TestSerializeDeserialize(t *testing.T) {
	contents := `
[Unit]
Description = Foo
`
	deserialized, err := NewUnitFile(contents)
	if err != nil {
		t.Fatalf("Unexpected error parsing unit %q: %v", contents, err)
	}
	section := deserialized.Contents["Unit"]
	if val, ok := section["Description"]; !ok || val[0] != "Foo" {
		t.Errorf("Failed to persist data through serialize/deserialize: %v", val)
	}

	serialized := deserialized.String()
	deserialized, err = NewUnitFile(serialized)
	if err != nil {
		t.Fatalf("Unexpected error parsing unit %q: %v", serialized, err)
	}

	section = deserialized.Contents["Unit"]
	if val, ok := section["Description"]; !ok || val[0] != "Foo" {
		t.Errorf("Failed to persist data through serialize/deserialize: %v", val)
	}
}

func TestDescription(t *testing.T) {
	contents := `
[Unit]
Description = Foo

[Service]
ExecStart=echo "ping";
ExecStop=echo "pong";
`

	unitFile, err := NewUnitFile(contents)
	if err != nil {
		t.Fatalf("Unexpected error parsing unit %q: %v", contents, err)
	}
	if unitFile.Description() != "Foo" {
		t.Fatalf("Unit.Description is incorrect")
	}
}

func TestDescriptionNotDefined(t *testing.T) {
	contents := `
[Unit]

[Service]
ExecStart=echo "ping";
ExecStop=echo "pong";
`

	unitFile, err := NewUnitFile(contents)
	if err != nil {
		t.Fatalf("Unexpected error parsing unit %q: %v", contents, err)
	}
	if unitFile.Description() != "" {
		t.Fatalf("Unit.Description is incorrect")
	}
}

func TestBadUnitsFail(t *testing.T) {
	bad := []string{
		`
[Unit]

[Service]
<<<<<<<<<<<<<<<<
`,
		`
[Unit]
nonsense upon stilts
`,
	}
	for _, tt := range bad {
		if _, err := NewUnitFile(tt); err == nil {
			t.Fatalf("Did not get expected error creating Unit from %q", tt)
		}
	}
}

func TestParseMultivalueLine(t *testing.T) {
	tests := []struct {
		in  string
		out []string
	}{
		{`"bar" "ping" "pong"`, []string{`bar`, `ping`, `pong`}},
		{`"bar"`, []string{`bar`}},
		{``, []string{""}},
		{`""`, []string{``}},
		{`"bar`, []string{`"bar`}},
		{`bar"`, []string{`bar"`}},
		{`foo\"bar`, []string{`foo\"bar`}},

		{
			`"bar" "`,
			[]string{`bar`, ``},
			//TODO(bcwaldon): should be something like this:
			// []string{`bar`},
		},

		{
			`"foo\"bar"`,
			[]string{`foo\bar`},
			//TODO(bcwaldon): should be something like this:
			// []string{`foo\"bar`},
		},
	}
	for i, tt := range tests {
		out := parseMultivalueLine(tt.in)
		if !reflect.DeepEqual(tt.out, out) {
			t.Errorf("case %d:, epected %v, got %v", i, tt.out, out)
		}
	}
}
<commit_msg>Revert "unit: split out TestHashFromHexString"<commit_after>// Copyright 2014 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package unit

import (
	"reflect"
	"testing"
)

func TestUnitHash(t *testing.T) {
	u, err := NewUnitFile("[Service]\nExecStart=/bin/sleep 100\n")
	if err != nil {
		t.Fatalf("Unexpected error encountered creating unit: %v", err)
	}

	gotHash := u.Hash()
	gotHashString := gotHash.String()
	expectHashString := "1c6fb6f3684bafb0c173d8b8b957ceff031180c1"
	if gotHashString != expectHashString {
		t.Fatalf("Unit Hash (%s) does not match expected (%s)", gotHashString, expectHashString)
	}

	expectHashShort := "1c6fb6f"
	gotHashShort := gotHash.Short()
	if gotHashShort != expectHashShort {
		t.Fatalf("Unit Hash short (%s) does not match expected (%s)", gotHashShort, expectHashShort)
	}

	eh := &Hash{}
	if !eh.Empty() {
		t.Fatalf("Empty hash check failed: %v", eh.Empty())
	}

	rehashed, err := HashFromHexString(expectHashString)
	if err != nil {
		t.Fatalf("HashFromHexString failed with: %v", err)
	}
	if rehashed != gotHash {
		t.Fatalf("HashFromHexString not equal to original hash")
	}
}

func TestRecognizedUnitTypes(t *testing.T) {
	tts := []struct {
		name string
		ok   bool
	}{
		{"foo.service", true},
		{"foo.socket", true},
		{"foo.path", true},
		{"foo.timer", true},
		{"foo.mount", true},
		{"foo.automount", true},
		{"foo.device", true},
		{"foo.swap", false},
		{"foo.target", false},
		{"foo.snapshot", false},
		{"foo.network", false},
		{"foo.netdev", false},
		{"foo.link", false},
		{"foo.unknown", false},
	}

	for _, tt := range tts {
		ok := RecognizedUnitType(tt.name)
		if ok != tt.ok {
			t.Errorf("Case failed: name=%s expect=%t result=%t", tt.name, tt.ok, ok)
		}
	}
}

func TestDefaultUnitType(t *testing.T) {
	tts := []struct {
		name string
		out  string
	}{
		{"foo", "foo.service"},
		{"foo.service", "foo.service.service"},
		{"foo.link", "foo.link.service"},
	}

	for _, tt := range tts {
		out := DefaultUnitType(tt.name)
		if out != tt.out {
			t.Errorf("Case failed: name=%s expect=%s result=%s", tt.name, tt.out, out)
		}
	}
}

func TestNewUnitState(t *testing.T) {
	want := &UnitState{
		LoadState:   "ls",
		ActiveState: "as",
		SubState:    "ss",
		MachineID:   "id",
	}

	got := NewUnitState("ls", "as", "ss", "id")
	if !reflect.DeepEqual(got, want) {
		t.Fatalf("NewUnitState did not create a correct UnitState: got %s, want %s", got, want)
	}

}

func TestNamedUnit(t *testing.T) {
	tts := []struct {
		fname  string
		name   string
		pref   string
		tmpl   string
		inst   string
		isinst bool
		istmpl bool
	}{
		{"foo.service", "foo", "foo", "", "", false, false},
		{"foo@.service", "foo@", "foo", "foo@.service", "", false, true},
		{"foo@bar.service", "foo@bar", "foo", "foo@.service", "bar", true, false},
		{"foo@bar@baz.service", "foo@bar@baz", "foo", "foo@.service", "bar@baz", true, false},
		{"foo.1@.service", "foo.1@", "foo.1", "foo.1@.service", "", false, true},
		{"foo.1@2.service", "foo.1@2", "foo.1", "foo.1@.service", "2", true, false},
		{"ssh@.socket", "ssh@", "ssh", "ssh@.socket", "", false, true},
		{"ssh@1.socket", "ssh@1", "ssh", "ssh@.socket", "1", true, false},
	}
	for _, tt := range tts {
		u := NewUnitNameInfo(tt.fname)
		if u == nil {
			t.Errorf("NewUnitNameInfo(%s) returned nil InstanceUnit!", tt.name)
			continue
		}
		if u.FullName != tt.fname {
			t.Errorf("NewUnitNameInfo(%s) returned bad fullname: got %s, want %s", tt.name, u.FullName, tt.fname)
		}
		if u.Name != tt.name {
			t.Errorf("NewUnitNameInfo(%s) returned bad name: got %s, want %s", tt.name, u.Name, tt.name)
		}
		if u.Template != tt.tmpl {
			t.Errorf("NewUnitNameInfo(%s) returned bad template name: got %s, want %s", tt.name, u.Template, tt.tmpl)
		}
		if u.Prefix != tt.pref {
			t.Errorf("NewUnitNameInfo(%s) returned bad prefix name: got %s, want %s", tt.name, u.Prefix, tt.pref)
		}
		if u.Instance != tt.inst {
			t.Errorf("NewUnitNameInfo(%s) returned bad instance name: got %s, want %s", tt.name, u.Instance, tt.inst)
		}
		i := u.IsInstance()
		if i != tt.isinst {
			t.Errorf("NewUnitNameInfo(%s).IsInstance returned %t, want %t", tt.name, i, tt.isinst)
		}
		i = u.IsTemplate()
		if i != tt.istmpl {
			t.Errorf("NewUnitNameInfo(%s).IsTemplate returned %t, want %t", tt.name, i, tt.istmpl)
		}
	}

	bad := []string{"foo", "bar@baz"}
	for _, tt := range bad {
		if NewUnitNameInfo(tt) != nil {
			t.Errorf("NewUnitNameInfo returned non-nil InstanceUnit unexpectedly")
		}
	}

}

func TestDeserialize(t *testing.T) {
	contents := `
This=Ignored
[Unit]
;ignore this guy
Description = Foo

[Service]
ExecStart=echo "ping";
ExecStop=echo "pong"
# ignore me, too
ExecStop=echo post

[X-Fleet]
MachineMetadata=foo=bar
MachineMetadata=baz=qux
`

	expected := map[string]map[string][]string{
		"Unit": {
			"Description": {"Foo"},
		},
		"Service": {
			"ExecStart": {`echo "ping";`},
			"ExecStop":  {`echo "pong"`, "echo post"},
		},
		"X-Fleet": {
			"MachineMetadata": {"foo=bar", "baz=qux"},
		},
	}

	unitFile, err := NewUnitFile(contents)
	if err != nil {
		t.Fatalf("Unexpected error parsing unit %q: %v", contents, err)
	}

	if !reflect.DeepEqual(expected, unitFile.Contents) {
		t.Fatalf("Map func did not produce expected output.\nActual=%v\nExpected=%v", unitFile.Contents, expected)
	}
}

func TestDeserializedUnitGarbage(t *testing.T) {
	contents := `
>>>>>>>>>>>>>
[Service]
ExecStart=jim
# As long as a line has an equals sign, systemd is happy, so we should pass it through
<<<<<<<<<<<=bar
`
	expected := map[string]map[string][]string{
		"Service": {
			"ExecStart":   {"jim"},
			"<<<<<<<<<<<": {"bar"},
		},
	}
	unitFile, err := NewUnitFile(contents)
	if err != nil {
		t.Fatalf("Unexpected error parsing unit %q: %v", contents, err)
	}

	if !reflect.DeepEqual(expected, unitFile.Contents) {
		t.Fatalf("Map func did not produce expected output.\nActual=%v\nExpected=%v", unitFile.Contents, expected)
	}
}

func TestDeserializeEscapedMultilines(t *testing.T) {
	contents := `
[Service]
ExecStart=echo \
  "pi\
  ng"
ExecStop=\
echo "po\
ng"
# comments within continuation should not be ignored
ExecStopPre=echo\
#pang
ExecStopPost=echo\
#peng\
pung
`
	expected := map[string]map[string][]string{
		"Service": {
			"ExecStart": {`echo \
  "pi\
  ng"`},
			"ExecStop": {`\
echo "po\
ng"`},
			"ExecStopPre": {`echo\
#pang`},
			"ExecStopPost": {`echo\
#peng\
pung`},
		},
	}
	unitFile, err := NewUnitFile(contents)
	if err != nil {
		t.Fatalf("Unexpected error parsing unit %q: %v", contents, err)
	}

	if !reflect.DeepEqual(expected, unitFile.Contents) {
		t.Fatalf("Map func did not produce expected output.\nActual=%v\nExpected=%v", unitFile.Contents, expected)
	}
}

func TestSerializeDeserialize(t *testing.T) {
	contents := `
[Unit]
Description = Foo
`
	deserialized, err := NewUnitFile(contents)
	if err != nil {
		t.Fatalf("Unexpected error parsing unit %q: %v", contents, err)
	}
	section := deserialized.Contents["Unit"]
	if val, ok := section["Description"]; !ok || val[0] != "Foo" {
		t.Errorf("Failed to persist data through serialize/deserialize: %v", val)
	}

	serialized := deserialized.String()
	deserialized, err = NewUnitFile(serialized)
	if err != nil {
		t.Fatalf("Unexpected error parsing unit %q: %v", serialized, err)
	}

	section = deserialized.Contents["Unit"]
	if val, ok := section["Description"]; !ok || val[0] != "Foo" {
		t.Errorf("Failed to persist data through serialize/deserialize: %v", val)
	}
}

func TestDescription(t *testing.T) {
	contents := `
[Unit]
Description = Foo

[Service]
ExecStart=echo "ping";
ExecStop=echo "pong";
`

	unitFile, err := NewUnitFile(contents)
	if err != nil {
		t.Fatalf("Unexpected error parsing unit %q: %v", contents, err)
	}
	if unitFile.Description() != "Foo" {
		t.Fatalf("Unit.Description is incorrect")
	}
}

func TestDescriptionNotDefined(t *testing.T) {
	contents := `
[Unit]

[Service]
ExecStart=echo "ping";
ExecStop=echo "pong";
`

	unitFile, err := NewUnitFile(contents)
	if err != nil {
		t.Fatalf("Unexpected error parsing unit %q: %v", contents, err)
	}
	if unitFile.Description() != "" {
		t.Fatalf("Unit.Description is incorrect")
	}
}

func TestBadUnitsFail(t *testing.T) {
	bad := []string{
		`
[Unit]

[Service]
<<<<<<<<<<<<<<<<
`,
		`
[Unit]
nonsense upon stilts
`,
	}
	for _, tt := range bad {
		if _, err := NewUnitFile(tt); err == nil {
			t.Fatalf("Did not get expected error creating Unit from %q", tt)
		}
	}
}

func TestParseMultivalueLine(t *testing.T) {
	tests := []struct {
		in  string
		out []string
	}{
		{`"bar" "ping" "pong"`, []string{`bar`, `ping`, `pong`}},
		{`"bar"`, []string{`bar`}},
		{``, []string{""}},
		{`""`, []string{``}},
		{`"bar`, []string{`"bar`}},
		{`bar"`, []string{`bar"`}},
		{`foo\"bar`, []string{`foo\"bar`}},

		{
			`"bar" "`,
			[]string{`bar`, ``},
			//TODO(bcwaldon): should be something like this:
			// []string{`bar`},
		},

		{
			`"foo\"bar"`,
			[]string{`foo\bar`},
			//TODO(bcwaldon): should be something like this:
			// []string{`foo\"bar`},
		},
	}
	for i, tt := range tests {
		out := parseMultivalueLine(tt.in)
		if !reflect.DeepEqual(tt.out, out) {
			t.Errorf("case %d:, epected %v, got %v", i, tt.out, out)
		}
	}
}
<|endoftext|>


<commit_before>package postCmd

import (
	// Stdlib
	"bytes"
	"errors"
	"fmt"

	// Internal
	"github.com/salsaflow/salsaflow/errs"
	"github.com/salsaflow/salsaflow/git"
	"github.com/salsaflow/salsaflow/modules"
)

func postRevisions(revisions ...string) (err error) {
	// Get the commit list for the given revision ranges.
	task := "Get the commits to be posted for code review"
	commits, err := git.ShowCommits(revisions...)
	if err != nil {
		return errs.NewError(task, err)
	}

	// Make sure there are no merge commits.
	if err := ensureNoMergeCommits(commits); err != nil {
		return err
	}

	// Make sure the Story-Id tag is not missing.
	if err := ensureStoryIdTag(commits); err != nil {
		return err
	}

	// Prompt the user to confirm.
	if err := promptUserToConfirmCommits(commits); err != nil {
		return err
	}

	// Post the review requests, in this case it will be only one.
	if err := postCommitsForReview(commits); err != nil {
		return err
	}

	// In case there is no error, tell the user they can do next.
	return printFollowup()
}

func ensureStoryIdTag(commits []*git.Commit) error {
	task := "Make sure all commits contain the Story-Id tag"

	storyIdTag := flagStoryIdTag
	if storyIdTag != "" && storyIdTag != git.StoryIdUnassignedTagValue {
		if err := checkStoryIdTag(storyIdTag); err != nil {
			return errs.NewError(task, err)
		}
	}

	var (
		hint    = bytes.NewBufferString("\n")
		missing bool
	)
	for _, commit := range commits {
		if commit.StoryIdTag == "" {
			if storyIdTag != "" {
				commit.StoryIdTag = storyIdTag
				continue
			}

			fmt.Fprintf(hint, "Commit %v is missing the Story-Id tag\n", commit.SHA)
			missing = true
		}
	}
	fmt.Fprintf(hint, "\n")

	if missing {
		return errs.NewErrorWithHint(
			task, errors.New("Story-Id tag missing"), hint.String())
	}
	return nil
}

func checkStoryIdTag(tag string) error {
	tracker, err := modules.GetIssueTracker()
	if err != nil {
		return err
	}

	_, err = tracker.StoryTagToReadableStoryId(tag)
	return err
}
<commit_msg>review post: Make sure commits are pushed<commit_after>package postCmd

import (
	// Stdlib
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"strings"

	// Internal
	"github.com/salsaflow/salsaflow/errs"
	"github.com/salsaflow/salsaflow/git"
	"github.com/salsaflow/salsaflow/modules"
)

func postRevisions(revisions ...string) (err error) {
	// Get the commit list for the given revision ranges.
	task := "Get the commits to be posted for code review"
	commits, err := git.ShowCommits(revisions...)
	if err != nil {
		return errs.NewError(task, err)
	}

	// Make sure there are no merge commits.
	if err := ensureNoMergeCommits(commits); err != nil {
		return err
	}

	// Make sure the Story-Id tag is not missing.
	if err := ensureStoryIdTag(commits); err != nil {
		return err
	}

	// Make sure the commits exist in the upstream repository.
	if err := ensureCommitsPushed(commits); err != nil {
		return err
	}

	// Prompt the user to confirm.
	if err := promptUserToConfirmCommits(commits); err != nil {
		return err
	}

	// Post the review requests, in this case it will be only one.
	if err := postCommitsForReview(commits); err != nil {
		return err
	}

	// In case there is no error, tell the user they can do next.
	return printFollowup()
}

func ensureStoryIdTag(commits []*git.Commit) error {
	task := "Make sure all commits contain the Story-Id tag"

	storyIdTag := flagStoryIdTag
	if storyIdTag != "" && storyIdTag != git.StoryIdUnassignedTagValue {
		if err := checkStoryIdTag(storyIdTag); err != nil {
			return errs.NewError(task, err)
		}
	}

	var (
		hint    = bytes.NewBufferString("\n")
		missing bool
	)
	for _, commit := range commits {
		if commit.StoryIdTag == "" {
			if storyIdTag != "" {
				commit.StoryIdTag = storyIdTag
				continue
			}

			fmt.Fprintf(hint, "Commit %v is missing the Story-Id tag\n", commit.SHA)
			missing = true
		}
	}
	fmt.Fprintf(hint, "\n")

	if missing {
		return errs.NewErrorWithHint(
			task, errors.New("Story-Id tag missing"), hint.String())
	}
	return nil
}

func checkStoryIdTag(tag string) error {
	tracker, err := modules.GetIssueTracker()
	if err != nil {
		return err
	}

	_, err = tracker.StoryTagToReadableStoryId(tag)
	return err
}

func ensureCommitsPushed(commits []*git.Commit) error {
	task := "Make sure that all commits exist in the upstream repository"

	// Load git-related config.
	gitConfig, err := git.LoadConfig()
	if err != nil {
		return errs.NewError(task, err)
	}
	remoteName := gitConfig.RemoteName
	remotePrefix := remoteName + "/"

	// Check each commit one by one.
	//
	// We run `git branch -r --contains HASH` for each commit,
	// then we check the output. In case there is a branch prefixed
	// with the right upstream name, the commit is treated as pushed.
	var (
		hint    = bytes.NewBufferString("\n")
		missing bool
	)
CommitLoop:
	for _, commit := range commits {
		// Get `git branch -r --contains HASH` output.
		stdout, err := git.Run("branch", "-r", "--contains", commit.SHA)
		if err != nil {
			return errs.NewError(task, err)
		}

		// Parse `git branch` output line by line.
		scanner := bufio.NewScanner(stdout)
		for scanner.Scan() {
			line := scanner.Text()
			if strings.HasPrefix(line, remotePrefix) {
				// The commit is contained in a remote branch, continue.
				continue CommitLoop
			}
		}
		if err := scanner.Err(); err != nil {
			return errs.NewError(task, err)
		}

		// The commit is not contained in any remote branch, bummer.
		fmt.Fprintf(hint,
			"Commit %v has not been pushed into remote '%v' yet.\n", commit.SHA, remoteName)
		missing = true
	}
	fmt.Fprintf(hint, "\n")
	fmt.Fprintf(hint, "All selected commits need to be pushed into the upstream pository.\n")
	fmt.Fprintf(hint, "Please make sure that is the case before trying again.\n")
	fmt.Fprintf(hint, "\n")

	// Return an error in case there is any commit that is not pushed.
	if missing {
		return errs.NewErrorWithHint(
			task, fmt.Errorf("some commits not found in upstream '%v'", remoteName), hint.String())
	}
	return nil
}
<|endoftext|>


<commit_before>package tnt

import "testing"

func BenchmarkSelectPack(b *testing.B) {
	// run the Fib function b.N times
	for n := 0; n < b.N; n++ {
		query := &Select{
			Values: Tuple{PackL(11), PackL(12)},
			Space:  10,
			Offset: 13,
			Limit:  14,
			Index:  15,
		}
		query.Pack()
	}
}
<commit_msg>PackL benchmark<commit_after>package tnt

import (
	"bytes"
	"encoding/binary"
	"testing"
)

func BenchmarkSelectPack(b *testing.B) {
	// run the Fib function b.N times
	for n := 0; n < b.N; n++ {
		query := &Select{
			Values: Tuple{PackL(11), PackL(12)},
			Space:  10,
			Offset: 13,
			Limit:  14,
			Index:  15,
		}
		query.Pack()
	}
}

func BenchmarkPackL(b *testing.B) {
	value := uint32(4294866796)
	for n := 0; n < b.N; n++ {
		PackL(value)
	}
}

func BenchmarkPackL1(b *testing.B) {
	value := uint32(4294866796)
	for n := 0; n < b.N; n++ {
		body := new(bytes.Buffer)
		binary.Write(body, binary.LittleEndian, value)
		body.Bytes()
	}
}
<|endoftext|>


<commit_before>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present  Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

package model

import (
	"net/url"
	"strings"

	"github.com/admpub/nging/application/dbschema"
	"github.com/admpub/nging/application/model/base"
	"github.com/webx-top/db"
	"github.com/webx-top/echo"
)

type DbSyncWithAccount struct {
	*dbschema.DbSync
	SrcAccount *dbschema.DbAccount `db:"-,relation=id:source_account_id"`
	DstAccount *dbschema.DbAccount `db:"-,relation=id:destination_account_id"`
}

func NewDbSync(ctx echo.Context) *DbSync {
	return &DbSync{
		DbSync: &dbschema.DbSync{},
		Base:   base.New(ctx),
	}
}

type DbSync struct {
	*dbschema.DbSync
	*base.Base
}

func (a *DbSync) ToDSNFromAccount(acc *dbschema.DbAccount) string {
	return a.ToDSN(acc.User, acc.Password, acc.Host, acc.Name)
}

func (a *DbSync) ToDSN(user, pwd, host, db string) string {
	//test:test@(127.0.0.1:3306)/test_0
	if len(user) == 0 {
		user = `root`
	}
	if len(pwd) == 0 {
		pwd = `root`
	}
	if len(host) == 0 {
		host = `127.0.0.1:3306`
	}
	return url.QueryEscape(user) + `:` + url.QueryEscape(pwd) + `@(` + host + `)/` + db
}

func (a *DbSync) ParseDSN(dsn string) (user string, pwd string, host string, dbName string) {
	idx := strings.Index(dsn, `:`)
	user, _ = url.QueryUnescape(dsn[0:idx])

	dsn = dsn[idx+1:]
	idx = strings.Index(dsn, `@`)
	pwd, _ = url.QueryUnescape(dsn[0:idx])

	dsn = dsn[idx+1:]
	idx = strings.Index(dsn, `/`)
	host = dsn[0:idx]

	host = strings.TrimPrefix(host, `(`)
	host = strings.TrimSuffix(host, `)`)

	dbName = dsn[idx+1:]
	return
}

func (a *DbSync) HidePassword(dsn string) string {
	idx := strings.Index(dsn, `:`)
	user, _ := url.QueryUnescape(dsn[0:idx])

	dsn = dsn[idx+1:]
	idx = strings.Index(dsn, `@`)
	return user + `:***@` + dsn[idx+1:]
}

func (a *DbSync) Add() (interface{}, error) {
	return a.DbSync.Add()
}

func (a *DbSync) Edit(mw func(db.Result) db.Result, args ...interface{}) error {
	return a.DbSync.Edit(mw, args...)
}
<commit_msg>update<commit_after>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present  Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

package model

import (
	"net/url"
	"strings"

	"github.com/admpub/nging/application/dbschema"
	"github.com/admpub/nging/application/model/base"
	"github.com/webx-top/db"
	"github.com/webx-top/echo"
)

type DbSyncWithAccount struct {
	*dbschema.DbSync
	SrcAccount *dbschema.DbAccount `db:"-,relation=id:source_account_id"`
	DstAccount *dbschema.DbAccount `db:"-,relation=id:destination_account_id"`
}

func NewDbSync(ctx echo.Context) *DbSync {
	return &DbSync{
		DbSync: &dbschema.DbSync{},
		Base:   base.New(ctx),
	}
}

type DbSync struct {
	*dbschema.DbSync
	*base.Base
}

func (a *DbSync) ToDSNFromAccount(acc *dbschema.DbAccount) string {
	return a.ToDSN(acc.User, acc.Password, acc.Host, acc.Name)
}

func (a *DbSync) ToDSN(user, pwd, host, db string) string {
	//test:test@(127.0.0.1:3306)/test_0
	if len(user) == 0 {
		user = `root`
	}
	if len(pwd) == 0 {
		pwd = `root`
	}
	if len(host) == 0 {
		host = `127.0.0.1:3306`
	}
	return url.QueryEscape(user) + `:` + url.QueryEscape(pwd) + `@(` + host + `)/` + db
}

func (a *DbSync) ParseDSN(dsn string) (user string, pwd string, host string, dbName string) {
	if len(dsn) == 0 {
		return
	}
	idx := strings.Index(dsn, `:`)
	if idx > 0 {
		user, _ = url.QueryUnescape(dsn[0:idx])
	}

	dsn = dsn[idx+1:]
	idx = strings.Index(dsn, `@`)
	if idx > 0 {
		pwd, _ = url.QueryUnescape(dsn[0:idx])
	}

	dsn = dsn[idx+1:]
	idx = strings.Index(dsn, `/`)
	if idx > 0 {
		host = dsn[0:idx]
	}

	host = strings.TrimPrefix(host, `(`)
	host = strings.TrimSuffix(host, `)`)
	if len(dsn) > idx+1 {
		dbName = dsn[idx+1:]
	}
	return
}

func (a *DbSync) HidePassword(dsn string) string {
	if len(dsn) == 0 {
		return dsn
	}
	idx := strings.Index(dsn, `:`)
	var user string
	if idx > 0 {
		user, _ = url.QueryUnescape(dsn[0:idx])
	}

	dsn = dsn[idx+1:]
	idx = strings.Index(dsn, `@`)
	return user + `:***@` + dsn[idx+1:]
}

func (a *DbSync) Add() (interface{}, error) {
	return a.DbSync.Add()
}

func (a *DbSync) Edit(mw func(db.Result) db.Result, args ...interface{}) error {
	return a.DbSync.Edit(mw, args...)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"net"
	"os"

	"github.com/brooksbp/go.netflow/pkg/nfv9"
)

var (
	flagListen = flag.String("listen", ":9999", "host:port to listen on.")
)

func main() {
	flag.Parse()

	listenStr := *flagListen

	addr, err := net.ResolveUDPAddr("udp", listenStr)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	conn, err := net.ListenUDP("udp", addr)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	template_cache := nfv9.NewTemplateCache()

	var buf [4096]byte
	for {
		n, _, err := conn.ReadFromUDP(buf[0:])
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}

		framer := nfv9.NewFramer(bytes.NewBuffer(buf[:n]), template_cache)
		frame, err := framer.ReadFrame()
		if err != nil {
			fmt.Println("Error: ", err, frame)
		}
		fmt.Println(frame.Header.String())
		for _, fs := range frame.FlowSets {
			switch flowset := fs.(type) {
			case nfv9.TemplateFlowSet:
				break
			case nfv9.DataFlowSet:
				// Print the data.
				if template, ok := template_cache.Get(flowset.FlowSetID); ok {
					for _, record := range flowset.Records {
						var i int
						for _, field := range template.Fields {
							ty := int(field.Type)
							len := int(field.Length)

							entry := nfv9.FieldMap[ty]

							fmt.Print(entry.Name, ": ", entry.String(record.Fields[i:i+len]), " ")

							i += len
						}
						fmt.Print("\n")
					}
				}
				break
			default:
				fmt.Println("Unknown flowset")
			}
		}
	}
}
<commit_msg>Lift DataFlowSet printing into own func.<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"net"
	"os"

	"github.com/brooksbp/go.netflow/pkg/nfv9"
)

var (
	flagListen = flag.String("listen", ":9999", "host:port to listen on.")
)

func PrintDataFlowSet(dfs *nfv9.DataFlowSet, tc *nfv9.TemplateCache) {
	template, ok := tc.Get(dfs.FlowSetID)
	if !ok {
		return
	}
	for _, record := range dfs.Records {
		var i int
		for _, field := range template.Fields {
			ty := int(field.Type)
			len := int(field.Length)

			entry := nfv9.FieldMap[ty]

			fmt.Print(entry.Name, ": ", entry.String(record.Fields[i:i+len]), " ")

			i += len
		}
		fmt.Println()
	}
}

func main() {
	flag.Parse()

	listenStr := *flagListen

	addr, err := net.ResolveUDPAddr("udp", listenStr)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	conn, err := net.ListenUDP("udp", addr)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	template_cache := nfv9.NewTemplateCache()

	var buf [4096]byte
	for {
		n, _, err := conn.ReadFromUDP(buf[0:])
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}

		framer := nfv9.NewFramer(bytes.NewBuffer(buf[:n]), template_cache)
		frame, err := framer.ReadFrame()
		if err != nil {
			fmt.Println("Error: ", err, frame)
		}
		fmt.Println(frame.Header.String())
		for _, fs := range frame.FlowSets {
			switch flowset := fs.(type) {
			case nfv9.TemplateFlowSet:
				break
			case nfv9.DataFlowSet:
				PrintDataFlowSet(&flowset, template_cache)
				break
			default:
				fmt.Println("Unknown flowset")
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net/http"
	"text/template"
)

type CommandCenter struct {
	server
	tld       string
	apps      []*App
	servers   []Server
	autoStart bool
}

var tmpl *template.Template

func init() {
	t, err := template.New("-").Parse(baseHTML)
	fail(err)
	tmpl = t
}

func NewCommandCenter(c *Config) *CommandCenter {
	cc := &CommandCenter{tld: c.Tld}
	cc.name = "bam"
	cc.servers = createAliasedServers(c.Aliases)
	cc.apps = LoadApps(c.AppsDir)
	cc.autoStart = c.AutoStart
	return cc
}

func (cc *CommandCenter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Content-Type", "text/html")
	data := map[string]interface{}{
		"Tld":            cc.tld,
		"aliasedServers": cc.servers,
		"apps":           cc.apps,
	}
	tmpl.Execute(w, data)
}

func (cc *CommandCenter) Start() error {
	port, _ := FreePort()
	cc.port = port
	if cc.autoStart {
		go func() {
			cc.startApps()
		}()
	}
	return http.ListenAndServe(fmt.Sprintf(":%d", cc.port), cc)
}

func (cc *CommandCenter) List() []Server {
	s := cc.servers
	s = append(s, cc)
	for _, app := range cc.apps {
		s = append(s, app)
	}
	return s
}

func (cc *CommandCenter) startApps() {
	for _, app := range cc.apps {
		log.Printf("Starting app %s\n", app.Name())
		go app.Start()
	}
}

func createAliasedServers(aliases map[string]int) []Server {
	servers := []Server{}
	for name, port := range aliases {
		servers = append(servers, NewServer(name, port))
	}
	return servers
}

const baseHTML = `

    <title>Bam's Command Center</title>

		{{$tld := .Tld}}

BAM!!!

Aliased servers

				{{range .aliasedServers}}

{{.Name}}

				{{end}}

Applications

				{{range .apps}}
					{{ if .Started}}

					{{ else }}

					{{ end }}
						{{.Name}}

				{{end}}

`
<commit_msg>add start/stop actions to ui<commit_after>package main

import (
	"fmt"
	"log"
	"net/http"
	"text/template"
)

type CommandCenter struct {
	server
	tld       string
	apps      []*App
	servers   []Server
	autoStart bool
}

var tmpl *template.Template

func init() {
	t, err := template.New("-").Parse(baseHTML)
	fail(err)
	tmpl = t
}

func NewCommandCenter(c *Config) *CommandCenter {
	cc := &CommandCenter{tld: c.Tld}
	cc.name = "bam"
	cc.servers = createAliasedServers(c.Aliases)
	cc.apps = LoadApps(c.AppsDir)
	cc.autoStart = c.AutoStart
	return cc
}

func (cc *CommandCenter) List() []Server {
	s := cc.servers
	s = append(s, cc)
	for _, app := range cc.apps {
		s = append(s, app)
	}
	return s
}

func (cc *CommandCenter) Start() error {
	port, _ := FreePort()
	cc.port = port
	if cc.autoStart {
		go func() {
			cc.startApps()
		}()
	}
	return http.ListenAndServe(fmt.Sprintf(":%d", cc.port), cc.createHandler())
}

func (cc *CommandCenter) startApps() {
	for _, app := range cc.apps {
		log.Printf("Starting app %s\n", app.Name())
		go app.Start()
	}
}

func (cc *CommandCenter) createHandler() http.Handler {
	mux := http.NewServeMux()
	mux.HandleFunc("/", cc.index)
	mux.HandleFunc("/start", cc.start)
	mux.HandleFunc("/stop", cc.stop)
	return mux
}

func (cc *CommandCenter) index(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Content-Type", "text/html")
	data := map[string]interface{}{
		"Tld":            cc.tld,
		"aliasedServers": cc.servers,
		"apps":           cc.apps,
	}
	tmpl.Execute(w, data)
}

func (cc *CommandCenter) start(w http.ResponseWriter, r *http.Request) {
	cc.action(w, r, func(a *App) error {
		log.Printf("Starting app %s\n", a.Name())
		return a.Start()
	})
}

func (cc *CommandCenter) stop(w http.ResponseWriter, r *http.Request) {
	cc.action(w, r, func(a *App) error {
		log.Printf("Stopping app %s\n", a.Name())
		return a.Stop()
	})
}

func (cc *CommandCenter) action(w http.ResponseWriter, r *http.Request, action func(a *App) error) {
	name := r.URL.Query().Get("app")
	for _, app := range cc.apps {
		if app.Name() == name {
			action(app)
			break
		}
	}
	http.Redirect(w, r, "/", 302)
}

func createAliasedServers(aliases map[string]int) []Server {
	servers := []Server{}
	for name, port := range aliases {
		servers = append(servers, NewServer(name, port))
	}
	return servers
}

const baseHTML = `

    <title>Bam's Command Center</title>

		{{$tld := .Tld}}

BAM!!!

Aliased servers

				{{range .aliasedServers}}

{{.Name}}

				{{end}}

Applications

				{{range .apps}}
					{{ if .Started}}

					{{ else }}

					{{ end }}

						{{.Name}}

						{{ if .Started}}
							Stop
						{{ else }}
							Start
						{{ end }}

				{{end}}

`
<|endoftext|>


<commit_before>package commands

import (
	"log"
	"os"
	"os/signal"
	"syscall"

	"github.com/reviewboard/rb-gateway/api"
	"github.com/reviewboard/rb-gateway/config"
)

func Serve(configPath string) {
	var cfg *config.Config
	configWatcher := config.Watch(configPath)

	select {
	case cfg = <-configWatcher.NewConfig:
		break

	case err := <-configWatcher.Errors:
		log.Fatal("Could not watch configuration file: ", err)
	}

	if cfg.TokenStorePath == ":memory:" {
		log.Fatal("Cannot use memory store outside of tests.")
	}

	api, err := api.New(*cfg)
	if err != nil {
		log.Fatalf("Could not create API: %s", err.Error())
	}

	hup := make(chan os.Signal, 1)
	signal.Notify(hup, syscall.SIGHUP)

	interrupt := make(chan os.Signal, 1)
	signal.Notify(interrupt, os.Interrupt)

	for {
		var newCfg *config.Config = nil
		shouldExit := false
		log.Println("Starting rb-gateway server on port", cfg.Port)
		log.Println("Quit the server with CONTROL-C.")

		server := api.Serve()

		select {
		case newCfg = <-configWatcher.NewConfig:
			log.Println("Detected configuration change, reloading...")

		case err := <-configWatcher.Errors:
			log.Fatal("Unexpected error: ", err)

		case <-hup:
			log.Println("Received SIGHUP, reloading configuration...")

			var err error
			if newCfg, err = configWatcher.ForceReload(); err != nil {
				log.Fatal("Unexpected error: ", err)
			}

		case <-interrupt:
			shouldExit = true
			signal.Reset(os.Interrupt)
			log.Println("Received SIGINT, shutting down...")
			log.Println("CONTROL-C again to force quit.")
		}

		err = api.Shutdown(server)
		if err != nil {
			log.Fatalf("An error occurred while shutting down the server: %s", err.Error())
		}

		log.Println("Server shut down.")

		if shouldExit {
			break
		}

		if newCfg != nil {
			if newCfg.TokenStorePath == ":memory:" {
				log.Println("Failed to reload configuration: cannot use memory store outside of tests.")
				log.Println("Configuration was not reloaded.")
			} else if err = api.SetConfig(*newCfg); err != nil {
				log.Printf("Failed to reload configuration: %s\n", err.Error())
			} else {
				log.Println("Configuration reloaded.")
			}
		}
	}
}
<commit_msg>Terminate upon receipt of SIGTERM<commit_after>package commands

import (
	"log"
	"os"
	"os/signal"
	"syscall"

	"github.com/reviewboard/rb-gateway/api"
	"github.com/reviewboard/rb-gateway/config"
)

func Serve(configPath string) {
	var cfg *config.Config
	configWatcher := config.Watch(configPath)

	select {
	case cfg = <-configWatcher.NewConfig:
		break

	case err := <-configWatcher.Errors:
		log.Fatal("Could not watch configuration file: ", err)
	}

	if cfg.TokenStorePath == ":memory:" {
		log.Fatal("Cannot use memory store outside of tests.")
	}

	api, err := api.New(*cfg)
	if err != nil {
		log.Fatalf("Could not create API: %s", err.Error())
	}

	hup := make(chan os.Signal, 1)
	signal.Notify(hup, syscall.SIGHUP)

	interrupt := make(chan os.Signal, 1)
	signal.Notify(interrupt, os.Interrupt)

	terminate := make(chan os.Signal, 1)
	signal.Notify(terminate, syscall.SIGTERM)

	for {
		var newCfg *config.Config = nil
		shouldExit := false
		log.Println("Starting rb-gateway server on port", cfg.Port)
		log.Println("Quit the server with CONTROL-C.")

		server := api.Serve()

		select {
		case newCfg = <-configWatcher.NewConfig:
			log.Println("Detected configuration change, reloading...")

		case err := <-configWatcher.Errors:
			log.Fatal("Unexpected error: ", err)

		case <-hup:
			log.Println("Received SIGHUP, reloading configuration...")

			var err error
			if newCfg, err = configWatcher.ForceReload(); err != nil {
				log.Fatal("Unexpected error: ", err)
			}

		case <-interrupt:
			shouldExit = true
			signal.Reset(os.Interrupt)
			log.Println("Received SIGINT, shutting down...")
			log.Println("CONTROL-C again to force quit.")

		case <-terminate:
			shouldExit = true
			log.Println("Received SIGTERM, shutting down...")
		}

		err = api.Shutdown(server)
		if err != nil {
			log.Fatalf("An error occurred while shutting down the server: %s", err.Error())
		}

		log.Println("Server shut down.")

		if shouldExit {
			break
		}

		if newCfg != nil {
			if newCfg.TokenStorePath == ":memory:" {
				log.Println("Failed to reload configuration: cannot use memory store outside of tests.")
				log.Println("Configuration was not reloaded.")
			} else if err = api.SetConfig(*newCfg); err != nil {
				log.Printf("Failed to reload configuration: %s\n", err.Error())
			} else {
				log.Println("Configuration reloaded.")
			}
		}
	}
}
<|endoftext|>


<commit_before>package commands

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/zetamatta/nyagos/alias"
	"github.com/zetamatta/nyagos/dos"
	"github.com/zetamatta/nyagos/shell"
)

const (
	WHICH_NOT_FOUND = 1
)

func envToList(first1 string, envs ...string) []string {
	result := make([]string, 1, 20)
	result[0] = first1
	for _, env := range envs {
		list1 := filepath.SplitList(os.Getenv(env))
		result = append(result, list1...)
	}
	return result
}

func cmdWhich(ctx context.Context, cmd Param) (int, error) {
	all := false
	var pathList []string
	var extList []string
	for _, name := range cmd.Args()[1:] {
		if name == "-a" {
			all = true
			pathList = envToList(".", "PATH", "NYAGOSPATH")
			extList = envToList("", "PATHEXT")
			continue
		}
		if a, ok := alias.Table[strings.ToLower(name)]; ok {
			fmt.Fprintf(cmd.Out(), "%s: aliased to %s\n", name, a.String())
			if !all {
				continue
			}
		}
		if _, ok := buildInCommand[name]; ok {
			fmt.Fprintf(cmd.Out(), "%s: built-in command\n", name)
			if !all {
				continue
			}
		}
		if all {
			for _, dir1 := range pathList {
				for _, ext1 := range extList {
					fullpath1 := filepath.Join(dir1, name)
					fullpath1 = fullpath1 + ext1
					if _, err1 := os.Stat(fullpath1); err1 == nil {
						fmt.Fprintln(cmd.Out(), fullpath1)
					}
				}
			}

		} else {
			path := dos.LookPath(shell.LookCurdirOrder, name, "NYAGOSPATH")
			if path == "" {
				return WHICH_NOT_FOUND, fmt.Errorf("which %s: not found", name)
			}
			fmt.Fprintln(cmd.Out(), filepath.Clean(path))
		}
	}
	return 0, nil
}
<commit_msg>Fix commands/which.go for golint<commit_after>package commands

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/zetamatta/nyagos/alias"
	"github.com/zetamatta/nyagos/dos"
	"github.com/zetamatta/nyagos/shell"
)

const (
	errnoWhichNotFound = 1
)

func envToList(first1 string, envs ...string) []string {
	result := make([]string, 1, 20)
	result[0] = first1
	for _, env := range envs {
		list1 := filepath.SplitList(os.Getenv(env))
		result = append(result, list1...)
	}
	return result
}

func cmdWhich(ctx context.Context, cmd Param) (int, error) {
	all := false
	var pathList []string
	var extList []string
	for _, name := range cmd.Args()[1:] {
		if name == "-a" {
			all = true
			pathList = envToList(".", "PATH", "NYAGOSPATH")
			extList = envToList("", "PATHEXT")
			continue
		}
		if a, ok := alias.Table[strings.ToLower(name)]; ok {
			fmt.Fprintf(cmd.Out(), "%s: aliased to %s\n", name, a.String())
			if !all {
				continue
			}
		}
		if _, ok := buildInCommand[name]; ok {
			fmt.Fprintf(cmd.Out(), "%s: built-in command\n", name)
			if !all {
				continue
			}
		}
		if all {
			for _, dir1 := range pathList {
				for _, ext1 := range extList {
					fullpath1 := filepath.Join(dir1, name)
					fullpath1 = fullpath1 + ext1
					if _, err1 := os.Stat(fullpath1); err1 == nil {
						fmt.Fprintln(cmd.Out(), fullpath1)
					}
				}
			}

		} else {
			path := dos.LookPath(shell.LookCurdirOrder, name, "NYAGOSPATH")
			if path == "" {
				return errnoWhichNotFound, fmt.Errorf("which %s: not found", name)
			}
			fmt.Fprintln(cmd.Out(), filepath.Clean(path))
		}
	}
	return 0, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package aggregator

import (
	"fmt"
	"net/http"
	"strings"
	"sync"
	"time"

	restful "github.com/emicklei/go-restful"
	"github.com/go-openapi/spec"

	"k8s.io/apiserver/pkg/server"
	"k8s.io/kube-aggregator/pkg/apis/apiregistration"
	"k8s.io/kube-openapi/pkg/aggregator"
	"k8s.io/kube-openapi/pkg/builder"
	"k8s.io/kube-openapi/pkg/common"
	"k8s.io/kube-openapi/pkg/handler"
)

// SpecAggregator calls out to http handlers of APIServices and merges specs. It keeps state of the last
// known specs including the http etag.
type SpecAggregator interface {
	AddUpdateAPIService(handler http.Handler, apiService *apiregistration.APIService) error
	UpdateAPIServiceSpec(apiServiceName string, spec *spec.Swagger, etag string) error
	RemoveAPIServiceSpec(apiServiceName string) error
	GetAPIServiceInfo(apiServiceName string) (handler http.Handler, etag string, exists bool)
	GetAPIServiceNames() []string
}

const (
	aggregatorUser                = "system:aggregator"
	specDownloadTimeout           = 60 * time.Second
	localDelegateChainNamePrefix  = "k8s_internal_local_delegation_chain_"
	localDelegateChainNamePattern = localDelegateChainNamePrefix + "%010d"

	// A randomly generated UUID to differentiate local and remote eTags.
	locallyGeneratedEtagPrefix = "\"6E8F849B434D4B98A569B9D7718876E9-"
)

// IsLocalAPIService returns true for local specs from delegates.
func IsLocalAPIService(apiServiceName string) bool {
	return strings.HasPrefix(apiServiceName, localDelegateChainNamePrefix)
}

// GetAPIServicesName returns the names of APIServices recorded in specAggregator.openAPISpecs.
// We use this function to pass the names of local APIServices to the controller in this package,
// so that the controller can periodically sync the OpenAPI spec from delegation API servers.
func (s *specAggregator) GetAPIServiceNames() []string {
	names := make([]string, len(s.openAPISpecs))
	for key := range s.openAPISpecs {
		names = append(names, key)
	}
	return names
}

// BuildAndRegisterAggregator registered OpenAPI aggregator handler. This function is not thread safe as it only being called on startup.
func BuildAndRegisterAggregator(downloader *Downloader, delegationTarget server.DelegationTarget, webServices []*restful.WebService,
	config *common.Config, pathHandler common.PathHandler) (SpecAggregator, error) {
	s := &specAggregator{
		openAPISpecs: map[string]*openAPISpecInfo{},
	}

	i := 0
	// Build Aggregator's spec
	aggregatorOpenAPISpec, err := builder.BuildOpenAPISpec(webServices, config)
	if err != nil {
		return nil, err
	}

	// Reserving non-name spec for aggregator's Spec.
	s.addLocalSpec(aggregatorOpenAPISpec, nil, fmt.Sprintf(localDelegateChainNamePattern, i), "")
	i++
	for delegate := delegationTarget; delegate != nil; delegate = delegate.NextDelegate() {
		handler := delegate.UnprotectedHandler()
		if handler == nil {
			continue
		}
		delegateSpec, etag, _, err := downloader.Download(handler, "")
		if err != nil {
			return nil, err
		}
		if delegateSpec == nil {
			continue
		}
		s.addLocalSpec(delegateSpec, handler, fmt.Sprintf(localDelegateChainNamePattern, i), etag)
		i++
	}

	// Build initial spec to serve.
	specToServe, err := s.buildOpenAPISpec()
	if err != nil {
		return nil, err
	}

	// Install handler
	s.openAPIVersionedService, err = handler.RegisterOpenAPIVersionedService(
		specToServe, "/openapi/v2", pathHandler)
	if err != nil {
		return nil, err
	}

	return s, nil
}

type specAggregator struct {
	// mutex protects all members of this struct.
	rwMutex sync.RWMutex

	// Map of API Services' OpenAPI specs by their name
	openAPISpecs map[string]*openAPISpecInfo

	// provided for dynamic OpenAPI spec
	openAPIVersionedService *handler.OpenAPIService
}

var _ SpecAggregator = &specAggregator{}

// This function is not thread safe as it only being called on startup.
func (s *specAggregator) addLocalSpec(spec *spec.Swagger, localHandler http.Handler, name, etag string) {
	localAPIService := apiregistration.APIService{}
	localAPIService.Name = name
	s.openAPISpecs[name] = &openAPISpecInfo{
		etag:       etag,
		apiService: localAPIService,
		handler:    localHandler,
		spec:       spec,
	}
}

// openAPISpecInfo is used to store OpenAPI spec with its priority.
// It can be used to sort specs with their priorities.
type openAPISpecInfo struct {
	apiService apiregistration.APIService

	// Specification of this API Service. If null then the spec is not loaded yet.
	spec    *spec.Swagger
	handler http.Handler
	etag    string
}

// buildOpenAPISpec aggregates all OpenAPI specs.  It is not thread-safe. The caller is responsible to hold proper locks.
func (s *specAggregator) buildOpenAPISpec() (specToReturn *spec.Swagger, err error) {
	specs := []openAPISpecInfo{}
	for _, specInfo := range s.openAPISpecs {
		if specInfo.spec == nil {
			continue
		}
		specs = append(specs, *specInfo)
	}
	if len(specs) == 0 {
		return &spec.Swagger{}, nil
	}
	sortByPriority(specs)
	for _, specInfo := range specs {
		if specToReturn == nil {
			specToReturn = &spec.Swagger{}
			*specToReturn = *specInfo.spec
			// Paths and Definitions are set by MergeSpecsIgnorePathConflict
			specToReturn.Paths = nil
			specToReturn.Definitions = nil
		}
		if err := aggregator.MergeSpecsIgnorePathConflict(specToReturn, specInfo.spec); err != nil {
			return nil, err
		}
	}

	return specToReturn, nil
}

// updateOpenAPISpec aggregates all OpenAPI specs.  It is not thread-safe. The caller is responsible to hold proper locks.
func (s *specAggregator) updateOpenAPISpec() error {
	if s.openAPIVersionedService == nil {
		return nil
	}
	specToServe, err := s.buildOpenAPISpec()
	if err != nil {
		return err
	}
	return s.openAPIVersionedService.UpdateSpec(specToServe)
}

// tryUpdatingServiceSpecs tries updating openAPISpecs map with specified specInfo, and keeps the map intact
// if the update fails.
func (s *specAggregator) tryUpdatingServiceSpecs(specInfo *openAPISpecInfo) error {
	if specInfo == nil {
		return fmt.Errorf("invalid input: specInfo must be non-nil")
	}
	orgSpecInfo, exists := s.openAPISpecs[specInfo.apiService.Name]
	// Skip aggregation if OpenAPI spec didn't change
	if exists && orgSpecInfo != nil && orgSpecInfo.etag == specInfo.etag {
		return nil
	}
	s.openAPISpecs[specInfo.apiService.Name] = specInfo
	if err := s.updateOpenAPISpec(); err != nil {
		if exists {
			s.openAPISpecs[specInfo.apiService.Name] = orgSpecInfo
		} else {
			delete(s.openAPISpecs, specInfo.apiService.Name)
		}
		return err
	}
	return nil
}

// tryDeleteServiceSpecs tries delete specified specInfo from openAPISpecs map, and keeps the map intact
// if the update fails.
func (s *specAggregator) tryDeleteServiceSpecs(apiServiceName string) error {
	orgSpecInfo, exists := s.openAPISpecs[apiServiceName]
	if !exists {
		return nil
	}
	delete(s.openAPISpecs, apiServiceName)
	if err := s.updateOpenAPISpec(); err != nil {
		s.openAPISpecs[apiServiceName] = orgSpecInfo
		return err
	}
	return nil
}

// UpdateAPIServiceSpec updates the api service's OpenAPI spec. It is thread safe.
func (s *specAggregator) UpdateAPIServiceSpec(apiServiceName string, spec *spec.Swagger, etag string) error {
	s.rwMutex.Lock()
	defer s.rwMutex.Unlock()

	specInfo, existingService := s.openAPISpecs[apiServiceName]
	if !existingService {
		return fmt.Errorf("APIService %q does not exists", apiServiceName)
	}

	// For APIServices (non-local) specs, only merge their /apis/ prefixed endpoint as it is the only paths
	// proxy handler delegates.
	if specInfo.apiService.Spec.Service != nil {
		spec = aggregator.FilterSpecByPathsWithoutSideEffects(spec, []string{"/apis/"})
	}

	return s.tryUpdatingServiceSpecs(&openAPISpecInfo{
		apiService: specInfo.apiService,
		spec:       spec,
		handler:    specInfo.handler,
		etag:       etag,
	})
}

// AddUpdateAPIService adds or updates the api service. It is thread safe.
func (s *specAggregator) AddUpdateAPIService(handler http.Handler, apiService *apiregistration.APIService) error {
	s.rwMutex.Lock()
	defer s.rwMutex.Unlock()

	if apiService.Spec.Service == nil {
		// All local specs should be already aggregated using local delegate chain
		return nil
	}

	newSpec := &openAPISpecInfo{
		apiService: *apiService,
		handler:    handler,
	}
	if specInfo, existingService := s.openAPISpecs[apiService.Name]; existingService {
		newSpec.etag = specInfo.etag
		newSpec.spec = specInfo.spec
	}
	return s.tryUpdatingServiceSpecs(newSpec)
}

// RemoveAPIServiceSpec removes an api service from OpenAPI aggregation. If it does not exist, no error is returned.
// It is thread safe.
func (s *specAggregator) RemoveAPIServiceSpec(apiServiceName string) error {
	s.rwMutex.Lock()
	defer s.rwMutex.Unlock()

	if _, existingService := s.openAPISpecs[apiServiceName]; !existingService {
		return nil
	}

	return s.tryDeleteServiceSpecs(apiServiceName)
}

// GetAPIServiceSpec returns api service spec info
func (s *specAggregator) GetAPIServiceInfo(apiServiceName string) (handler http.Handler, etag string, exists bool) {
	s.rwMutex.RLock()
	defer s.rwMutex.RUnlock()

	if info, existingService := s.openAPISpecs[apiServiceName]; existingService {
		return info.handler, info.etag, true
	}
	return nil, "", false
}
<commit_msg>UPSTREAM: 76385: kube-aggregator: fix spec info update<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package aggregator

import (
	"fmt"
	"net/http"
	"strings"
	"sync"
	"time"

	restful "github.com/emicklei/go-restful"
	"github.com/go-openapi/spec"

	"k8s.io/apiserver/pkg/server"
	"k8s.io/kube-aggregator/pkg/apis/apiregistration"
	"k8s.io/kube-openapi/pkg/aggregator"
	"k8s.io/kube-openapi/pkg/builder"
	"k8s.io/kube-openapi/pkg/common"
	"k8s.io/kube-openapi/pkg/handler"
)

// SpecAggregator calls out to http handlers of APIServices and merges specs. It keeps state of the last
// known specs including the http etag.
type SpecAggregator interface {
	AddUpdateAPIService(handler http.Handler, apiService *apiregistration.APIService) error
	UpdateAPIServiceSpec(apiServiceName string, spec *spec.Swagger, etag string) error
	RemoveAPIServiceSpec(apiServiceName string) error
	GetAPIServiceInfo(apiServiceName string) (handler http.Handler, etag string, exists bool)
	GetAPIServiceNames() []string
}

const (
	aggregatorUser                = "system:aggregator"
	specDownloadTimeout           = 60 * time.Second
	localDelegateChainNamePrefix  = "k8s_internal_local_delegation_chain_"
	localDelegateChainNamePattern = localDelegateChainNamePrefix + "%010d"

	// A randomly generated UUID to differentiate local and remote eTags.
	locallyGeneratedEtagPrefix = "\"6E8F849B434D4B98A569B9D7718876E9-"
)

// IsLocalAPIService returns true for local specs from delegates.
func IsLocalAPIService(apiServiceName string) bool {
	return strings.HasPrefix(apiServiceName, localDelegateChainNamePrefix)
}

// GetAPIServicesName returns the names of APIServices recorded in specAggregator.openAPISpecs.
// We use this function to pass the names of local APIServices to the controller in this package,
// so that the controller can periodically sync the OpenAPI spec from delegation API servers.
func (s *specAggregator) GetAPIServiceNames() []string {
	names := make([]string, len(s.openAPISpecs))
	for key := range s.openAPISpecs {
		names = append(names, key)
	}
	return names
}

// BuildAndRegisterAggregator registered OpenAPI aggregator handler. This function is not thread safe as it only being called on startup.
func BuildAndRegisterAggregator(downloader *Downloader, delegationTarget server.DelegationTarget, webServices []*restful.WebService,
	config *common.Config, pathHandler common.PathHandler) (SpecAggregator, error) {
	s := &specAggregator{
		openAPISpecs: map[string]*openAPISpecInfo{},
	}

	i := 0
	// Build Aggregator's spec
	aggregatorOpenAPISpec, err := builder.BuildOpenAPISpec(webServices, config)
	if err != nil {
		return nil, err
	}

	// Reserving non-name spec for aggregator's Spec.
	s.addLocalSpec(aggregatorOpenAPISpec, nil, fmt.Sprintf(localDelegateChainNamePattern, i), "")
	i++
	for delegate := delegationTarget; delegate != nil; delegate = delegate.NextDelegate() {
		handler := delegate.UnprotectedHandler()
		if handler == nil {
			continue
		}
		delegateSpec, etag, _, err := downloader.Download(handler, "")
		if err != nil {
			return nil, err
		}
		if delegateSpec == nil {
			continue
		}
		s.addLocalSpec(delegateSpec, handler, fmt.Sprintf(localDelegateChainNamePattern, i), etag)
		i++
	}

	// Build initial spec to serve.
	specToServe, err := s.buildOpenAPISpec()
	if err != nil {
		return nil, err
	}

	// Install handler
	s.openAPIVersionedService, err = handler.RegisterOpenAPIVersionedService(
		specToServe, "/openapi/v2", pathHandler)
	if err != nil {
		return nil, err
	}

	return s, nil
}

type specAggregator struct {
	// mutex protects all members of this struct.
	rwMutex sync.RWMutex

	// Map of API Services' OpenAPI specs by their name
	openAPISpecs map[string]*openAPISpecInfo

	// provided for dynamic OpenAPI spec
	openAPIVersionedService *handler.OpenAPIService
}

var _ SpecAggregator = &specAggregator{}

// This function is not thread safe as it only being called on startup.
func (s *specAggregator) addLocalSpec(spec *spec.Swagger, localHandler http.Handler, name, etag string) {
	localAPIService := apiregistration.APIService{}
	localAPIService.Name = name
	s.openAPISpecs[name] = &openAPISpecInfo{
		etag:       etag,
		apiService: localAPIService,
		handler:    localHandler,
		spec:       spec,
	}
}

// openAPISpecInfo is used to store OpenAPI spec with its priority.
// It can be used to sort specs with their priorities.
type openAPISpecInfo struct {
	apiService apiregistration.APIService

	// Specification of this API Service. If null then the spec is not loaded yet.
	spec    *spec.Swagger
	handler http.Handler
	etag    string
}

// buildOpenAPISpec aggregates all OpenAPI specs.  It is not thread-safe. The caller is responsible to hold proper locks.
func (s *specAggregator) buildOpenAPISpec() (specToReturn *spec.Swagger, err error) {
	specs := []openAPISpecInfo{}
	for _, specInfo := range s.openAPISpecs {
		if specInfo.spec == nil {
			continue
		}
		specs = append(specs, *specInfo)
	}
	if len(specs) == 0 {
		return &spec.Swagger{}, nil
	}
	sortByPriority(specs)
	for _, specInfo := range specs {
		if specToReturn == nil {
			specToReturn = &spec.Swagger{}
			*specToReturn = *specInfo.spec
			// Paths and Definitions are set by MergeSpecsIgnorePathConflict
			specToReturn.Paths = nil
			specToReturn.Definitions = nil
		}
		if err := aggregator.MergeSpecsIgnorePathConflict(specToReturn, specInfo.spec); err != nil {
			return nil, err
		}
	}

	return specToReturn, nil
}

// updateOpenAPISpec aggregates all OpenAPI specs.  It is not thread-safe. The caller is responsible to hold proper locks.
func (s *specAggregator) updateOpenAPISpec() error {
	if s.openAPIVersionedService == nil {
		return nil
	}
	specToServe, err := s.buildOpenAPISpec()
	if err != nil {
		return err
	}
	return s.openAPIVersionedService.UpdateSpec(specToServe)
}

// tryUpdatingServiceSpecs tries updating openAPISpecs map with specified specInfo, and keeps the map intact
// if the update fails.
func (s *specAggregator) tryUpdatingServiceSpecs(specInfo *openAPISpecInfo) error {
	if specInfo == nil {
		return fmt.Errorf("invalid input: specInfo must be non-nil")
	}
	origSpecInfo, existedBefore := s.openAPISpecs[specInfo.apiService.Name]
	s.openAPISpecs[specInfo.apiService.Name] = specInfo

	// Skip aggregation if OpenAPI spec didn't change
	if existedBefore && origSpecInfo != nil && origSpecInfo.etag == specInfo.etag {
		return nil
	}
	if err := s.updateOpenAPISpec(); err != nil {
		if existedBefore {
			s.openAPISpecs[specInfo.apiService.Name] = origSpecInfo
		} else {
			delete(s.openAPISpecs, specInfo.apiService.Name)
		}
		return err
	}
	return nil
}

// tryDeleteServiceSpecs tries delete specified specInfo from openAPISpecs map, and keeps the map intact
// if the update fails.
func (s *specAggregator) tryDeleteServiceSpecs(apiServiceName string) error {
	orgSpecInfo, exists := s.openAPISpecs[apiServiceName]
	if !exists {
		return nil
	}
	delete(s.openAPISpecs, apiServiceName)
	if err := s.updateOpenAPISpec(); err != nil {
		s.openAPISpecs[apiServiceName] = orgSpecInfo
		return err
	}
	return nil
}

// UpdateAPIServiceSpec updates the api service's OpenAPI spec. It is thread safe.
func (s *specAggregator) UpdateAPIServiceSpec(apiServiceName string, spec *spec.Swagger, etag string) error {
	s.rwMutex.Lock()
	defer s.rwMutex.Unlock()

	specInfo, existingService := s.openAPISpecs[apiServiceName]
	if !existingService {
		return fmt.Errorf("APIService %q does not exists", apiServiceName)
	}

	// For APIServices (non-local) specs, only merge their /apis/ prefixed endpoint as it is the only paths
	// proxy handler delegates.
	if specInfo.apiService.Spec.Service != nil {
		spec = aggregator.FilterSpecByPathsWithoutSideEffects(spec, []string{"/apis/"})
	}

	return s.tryUpdatingServiceSpecs(&openAPISpecInfo{
		apiService: specInfo.apiService,
		spec:       spec,
		handler:    specInfo.handler,
		etag:       etag,
	})
}

// AddUpdateAPIService adds or updates the api service. It is thread safe.
func (s *specAggregator) AddUpdateAPIService(handler http.Handler, apiService *apiregistration.APIService) error {
	s.rwMutex.Lock()
	defer s.rwMutex.Unlock()

	if apiService.Spec.Service == nil {
		// All local specs should be already aggregated using local delegate chain
		return nil
	}

	newSpec := &openAPISpecInfo{
		apiService: *apiService,
		handler:    handler,
	}
	if specInfo, existingService := s.openAPISpecs[apiService.Name]; existingService {
		newSpec.etag = specInfo.etag
		newSpec.spec = specInfo.spec
	}
	return s.tryUpdatingServiceSpecs(newSpec)
}

// RemoveAPIServiceSpec removes an api service from OpenAPI aggregation. If it does not exist, no error is returned.
// It is thread safe.
func (s *specAggregator) RemoveAPIServiceSpec(apiServiceName string) error {
	s.rwMutex.Lock()
	defer s.rwMutex.Unlock()

	if _, existingService := s.openAPISpecs[apiServiceName]; !existingService {
		return nil
	}

	return s.tryDeleteServiceSpecs(apiServiceName)
}

// GetAPIServiceSpec returns api service spec info
func (s *specAggregator) GetAPIServiceInfo(apiServiceName string) (handler http.Handler, etag string, exists bool) {
	s.rwMutex.RLock()
	defer s.rwMutex.RUnlock()

	if info, existingService := s.openAPISpecs[apiServiceName]; existingService {
		return info.handler, info.etag, true
	}
	return nil, "", false
}
<|endoftext|>