<commit_before>//
// Copyright (c) 2015-2022 Snowplow Analytics Ltd. All rights reserved.
//
// This program is licensed to you under the Apache License Version 2.0,
// and you may not use this file except in compliance with the Apache License Version 2.0.
// You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the Apache License Version 2.0 is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
//
package main

import (
	"context"
	"database/sql"
	"fmt"
	"log"
	"os"
	"strings"
	"time"

	"github.com/olekukonko/tablewriter"
	"github.com/pkg/errors"
	sf "github.com/snowflakedb/gosnowflake"
)

// Specific for Snowflake db
const (
	loginTimeout  = 5 * time.Second                // by default is 60
	multiStmtName = "multiple statement execution" // https://github.com/snowflakedb/gosnowflake/blob/e909f00ff624a7e60d4f91718f6adc92cbd0d80f/connection.go#L57-L61
)

type SnowFlakeTarget struct {
	Target
	Client *sql.DB
	Dsn    string
}

func (sft SnowFlakeTarget) IsConnectable() bool {
	client := sft.Client
	err := client.Ping()
	return err == nil
}

func NewSnowflakeTarget(target Target) *SnowFlakeTarget {
	// Note: region connection parameter is deprecated
	var region string
	if target.Region == "us-west-1" {
		region = ""
	} else {
		region = target.Region
	}

	configStr, err := sf.DSN(&sf.Config{
		Region:       region,
		Account:      target.Account,
		User:         target.Username,
		Password:     target.Password,
		Database:     target.Database,
		Warehouse:    target.Warehouse,
		LoginTimeout: loginTimeout,
	})

	if err != nil {
		log.Fatal(err)
	}

	db, err := sql.Open("snowflake", configStr)

	if err != nil {
		log.Fatal(err)
	}

	return &SnowFlakeTarget{target, db, configStr}
}

func (sft SnowFlakeTarget) GetTarget() Target {
	return sft.Target
}

// Run a query against the target
func (sft SnowFlakeTarget) RunQuery(query ReadyQuery, dryRun bool, showQueryOutput bool) QueryStatus {
	var affected int64 = 0
	var err error

	if dryRun {
		if sft.IsConnectable() {
			log.Printf("SUCCESS: Able to connect to target database, %s\n.", sft.Account)
		} else {
			log.Printf("ERROR: Cannot connect to target database, %s\n.", sft.Account)
		}

		return QueryStatus{query, query.Path, 0, nil}
	}

	// Enable grabbing the queryID
	queryIdChannel := make(chan string)
	ctxWithQueryIdChan := sf.WithQueryIDChan(context.Background(), queryIdChannel)

	// Kick off a goroutine to grab the queryId when we get it from the driver (there should be one queryID per script)
	var queryId string
	qId := &queryId
	go func() {
		*qId = <-queryIdChannel
	}()

	// 0 allows arbitrary number of statements
	ctx, err := sf.WithMultiStatement(ctxWithQueryIdChan, 0)
	if err != nil {
		log.Printf("ERROR: Could not initialise query script.")
		return QueryStatus{query, query.Path, 0, err}
	}
	script := query.Script

	if len(strings.TrimSpace(script)) > 0 {
		if showQueryOutput {
			rows, err := sft.Client.QueryContext(ctx, script)
			if err != nil {
				return QueryStatus{query, query.Path, int(affected), err}
			}
			defer rows.Close()

			err = printSfTable(rows)
			if err != nil {
				log.Printf("ERROR: %s.", err)
				return QueryStatus{query, query.Path, int(affected), err}
			}

			for rows.NextResultSet() {
				err = printSfTable(rows)
				if err != nil {
					log.Printf("ERROR: %s.", err)
					return QueryStatus{query, query.Path, int(affected), err}
				}
			}
		} else {
			res, err := sft.Client.ExecContext(ctx, script)
			if err != nil {
				switch err.Error() {
				// If the error message is `-00001: `, the DB failed to return accurate status. Request the status and proceed accordingly.
				case "-00001: ":
					fmt.Println("INFO: Encountered -1 status. Polling for query result with queryId: ", queryId)
					pollResult := pollForQueryStatus(sft, queryId)
					return QueryStatus{query, query.Path, int(affected), pollResult}
				default:
					return QueryStatus{query, query.Path, int(affected), errors.Wrap(err, fmt.Sprintf("QueryID: %s", queryId))}
				}
			}
			aff, _ := res.RowsAffected()
			affected += aff
		}
	}

	return QueryStatus{query, query.Path, int(affected), err}
}

func printSfTable(rows *sql.Rows) error {
	outputBuffer := make([][]string, 0, 10)
	cols, err := rows.Columns()
	if err != nil {
		return errors.New("Unable to read columns")
	}

	// check to prevent rows.Next() on multi-statement
	// see also: https://github.com/snowflakedb/gosnowflake/issues/365
	for _, c := range cols {
		if c == multiStmtName {
			return errors.New("Unable to showQueryOutput for multi-statement queries")
		}
	}

	vals := make([]interface{}, len(cols))
	rawResult := make([][]byte, len(cols))
	for i := range rawResult {
		vals[i] = &rawResult[i]
	}

	for rows.Next() {
		err = rows.Scan(vals...)
		if err != nil {
			return errors.New("Unable to read row")
		}

		if len(vals) > 0 {
			outputBuffer = append(outputBuffer, stringify(rawResult))
		}
	}

	if len(outputBuffer) > 0 {
		log.Printf("QUERY OUTPUT:\n")
		table := tablewriter.NewWriter(os.Stdout)
		table.SetHeader(cols)
		table.SetBorders(tablewriter.Border{Left: true, Top: false, Right: true, Bottom: false})
		table.SetCenterSeparator("|")

		for _, row := range outputBuffer {
			table.Append(row)
		}

		table.Render() // Send output
	}
	return nil
}

func stringify(row [][]byte) []string {
	var line []string
	for _, element := range row {
		line = append(line, fmt.Sprint(string(element)))
	}
	return line
}

// Blocking function to poll for the true status of a query which didn't return a result.
func pollForQueryStatus(sft SnowFlakeTarget, queryId string) error {
	// Get the snoflake driver and open a connection
	sfd := sft.Client.Driver()
	conn, err := sfd.Open(sft.Dsn)
	if err != nil {
		return errors.Wrap(err, "Failed to open connection to poll for query result.")
	}
	// Poll Snowflake for actual query status
	for {
		qStatus, err := conn.(sf.SnowflakeConnection).GetQueryStatus(context.Background(), queryId)

		switch {
		case err != nil && strings.Contains(err.Error(), "279301:"): // The driver returns an error containing this code when the query is still running.
			break
		case err != nil:
			// Any other error is genuine, return the error.
			return err
		case qStatus != nil && qStatus.ErrorCode == 0:
			// A non-nil qStatus means the query completed. If the ErrorCode field is 0, we have no error.
			return nil
		case qStatus != nil:
			// If qStatus is non-nil but has a non-zero error code, return the relevant info as an error.
			return errors.New(qStatus.ErrorMessage)
		default:
			break
		}
		// Give it a minute before polling again.
		time.Sleep(60 * time.Second)
	}
}
<commit_msg>Add Application to JDBC driver configuration for identifying Snowplow jobs (closes #198)<commit_after>//
// Copyright (c) 2015-2022 Snowplow Analytics Ltd. All rights reserved.
//
// This program is licensed to you under the Apache License Version 2.0,
// and you may not use this file except in compliance with the Apache License Version 2.0.
// You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the Apache License Version 2.0 is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
//
package main

import (
	"context"
	"database/sql"
	"fmt"
	"log"
	"os"
	"strings"
	"time"

	"github.com/olekukonko/tablewriter"
	"github.com/pkg/errors"
	sf "github.com/snowflakedb/gosnowflake"
)

// Specific for Snowflake db
const (
	snowplowAppName = `Snowplow_OSS`
	loginTimeout    = 5 * time.Second                // by default is 60
	multiStmtName   = "multiple statement execution" // https://github.com/snowflakedb/gosnowflake/blob/e909f00ff624a7e60d4f91718f6adc92cbd0d80f/connection.go#L57-L61
)

type SnowFlakeTarget struct {
	Target
	Client *sql.DB
	Dsn    string
}

func (sft SnowFlakeTarget) IsConnectable() bool {
	client := sft.Client
	err := client.Ping()
	return err == nil
}

func NewSnowflakeTarget(target Target) *SnowFlakeTarget {
	// Note: region connection parameter is deprecated
	var region string
	if target.Region == "us-west-1" {
		region = ""
	} else {
		region = target.Region
	}

	config := &sf.Config{
		Region:       region,
		Account:      target.Account,
		User:         target.Username,
		Password:     target.Password,
		Database:     target.Database,
		Warehouse:    target.Warehouse,
		LoginTimeout: loginTimeout,
	}
	if envAppName := os.Getenv(`SNOWPLOW_SQL_RUNNER_SNOWFLAKE_APP_NAME`); envAppName != `` {
		config.Application = `Snowplow_` + envAppName
	} else {
		config.Application = snowplowAppName
	}

	configStr, err := sf.DSN(config)

	if err != nil {
		log.Fatal(err)
	}

	db, err := sql.Open("snowflake", configStr)

	if err != nil {
		log.Fatal(err)
	}

	return &SnowFlakeTarget{target, db, configStr}
}

func (sft SnowFlakeTarget) GetTarget() Target {
	return sft.Target
}

// Run a query against the target
func (sft SnowFlakeTarget) RunQuery(query ReadyQuery, dryRun bool, showQueryOutput bool) QueryStatus {
	var affected int64 = 0
	var err error

	if dryRun {
		if sft.IsConnectable() {
			log.Printf("SUCCESS: Able to connect to target database, %s\n.", sft.Account)
		} else {
			log.Printf("ERROR: Cannot connect to target database, %s\n.", sft.Account)
		}

		return QueryStatus{query, query.Path, 0, nil}
	}

	// Enable grabbing the queryID
	queryIdChannel := make(chan string)
	ctxWithQueryIdChan := sf.WithQueryIDChan(context.Background(), queryIdChannel)

	// Kick off a goroutine to grab the queryId when we get it from the driver (there should be one queryID per script)
	var queryId string
	qId := &queryId
	go func() {
		*qId = <-queryIdChannel
	}()

	// 0 allows arbitrary number of statements
	ctx, err := sf.WithMultiStatement(ctxWithQueryIdChan, 0)
	if err != nil {
		log.Printf("ERROR: Could not initialise query script.")
		return QueryStatus{query, query.Path, 0, err}
	}
	script := query.Script

	if len(strings.TrimSpace(script)) > 0 {
		if showQueryOutput {
			rows, err := sft.Client.QueryContext(ctx, script)
			if err != nil {
				return QueryStatus{query, query.Path, int(affected), err}
			}
			defer rows.Close()

			err = printSfTable(rows)
			if err != nil {
				log.Printf("ERROR: %s.", err)
				return QueryStatus{query, query.Path, int(affected), err}
			}

			for rows.NextResultSet() {
				err = printSfTable(rows)
				if err != nil {
					log.Printf("ERROR: %s.", err)
					return QueryStatus{query, query.Path, int(affected), err}
				}
			}
		} else {
			res, err := sft.Client.ExecContext(ctx, script)
			if err != nil {
				switch err.Error() {
				// If the error message is `-00001: `, the DB failed to return accurate status. Request the status and proceed accordingly.
				case "-00001: ":
					fmt.Println("INFO: Encountered -1 status. Polling for query result with queryId: ", queryId)
					pollResult := pollForQueryStatus(sft, queryId)
					return QueryStatus{query, query.Path, int(affected), pollResult}
				default:
					return QueryStatus{query, query.Path, int(affected), errors.Wrap(err, fmt.Sprintf("QueryID: %s", queryId))}
				}
			}
			aff, _ := res.RowsAffected()
			affected += aff
		}
	}

	return QueryStatus{query, query.Path, int(affected), err}
}

func printSfTable(rows *sql.Rows) error {
	outputBuffer := make([][]string, 0, 10)
	cols, err := rows.Columns()
	if err != nil {
		return errors.New("Unable to read columns")
	}

	// check to prevent rows.Next() on multi-statement
	// see also: https://github.com/snowflakedb/gosnowflake/issues/365
	for _, c := range cols {
		if c == multiStmtName {
			return errors.New("Unable to showQueryOutput for multi-statement queries")
		}
	}

	vals := make([]interface{}, len(cols))
	rawResult := make([][]byte, len(cols))
	for i := range rawResult {
		vals[i] = &rawResult[i]
	}

	for rows.Next() {
		err = rows.Scan(vals...)
		if err != nil {
			return errors.New("Unable to read row")
		}

		if len(vals) > 0 {
			outputBuffer = append(outputBuffer, stringify(rawResult))
		}
	}

	if len(outputBuffer) > 0 {
		log.Printf("QUERY OUTPUT:\n")
		table := tablewriter.NewWriter(os.Stdout)
		table.SetHeader(cols)
		table.SetBorders(tablewriter.Border{Left: true, Top: false, Right: true, Bottom: false})
		table.SetCenterSeparator("|")

		for _, row := range outputBuffer {
			table.Append(row)
		}

		table.Render() // Send output
	}
	return nil
}

func stringify(row [][]byte) []string {
	var line []string
	for _, element := range row {
		line = append(line, fmt.Sprint(string(element)))
	}
	return line
}

// Blocking function to poll for the true status of a query which didn't return a result.
func pollForQueryStatus(sft SnowFlakeTarget, queryId string) error {
	// Get the snoflake driver and open a connection
	sfd := sft.Client.Driver()
	conn, err := sfd.Open(sft.Dsn)
	if err != nil {
		return errors.Wrap(err, "Failed to open connection to poll for query result.")
	}
	// Poll Snowflake for actual query status
	for {
		qStatus, err := conn.(sf.SnowflakeConnection).GetQueryStatus(context.Background(), queryId)

		switch {
		case err != nil && strings.Contains(err.Error(), "279301:"): // The driver returns an error containing this code when the query is still running.
			break
		case err != nil:
			// Any other error is genuine, return the error.
			return err
		case qStatus != nil && qStatus.ErrorCode == 0:
			// A non-nil qStatus means the query completed. If the ErrorCode field is 0, we have no error.
			return nil
		case qStatus != nil:
			// If qStatus is non-nil but has a non-zero error code, return the relevant info as an error.
			return errors.New(qStatus.ErrorMessage)
		default:
			break
		}
		// Give it a minute before polling again.
		time.Sleep(60 * time.Second)
	}
}
<|endoftext|>


<commit_before>package compress

import "strings"

var incompressibleTypes = map[string]struct{}{
	"":                             {},
	"application/pdf":              {},
	"application/x-gzip":           {},
	"application/x-rar-compressed": {},
	"application/zip":              {},
	"audio/aac":                    {},
	"audio/mp4":                    {},
	"audio/mpeg":                   {},
	"audio/wav":                    {},
	"audio/webm":                   {},
	"image/gif":                    {},
	"image/jpeg":                   {},
	"image/png":                    {},
	"video/h264":                   {},
	"video/mp4":                    {},
	"video/mpeg":                   {},
	"video/webm":                   {},
	"video/x-flv":                  {},
}

// compressibleContentType indicates whether the content of ct type can be compressed.
func compressibleContentType(ct string) bool {
	_, ok := incompressibleTypes[strings.ToLower(ct)]
	return !ok
}
<commit_msg>Update incompressibleTypes<commit_after>package compress

import "strings"

var incompressibleTypes = map[string]struct{}{
	"": {},
	"application/octet-stream":     {},
	"application/pdf":              {},
	"application/x-font-woff":      {},
	"application/x-gzip":           {},
	"application/x-rar-compressed": {},
	"application/zip":              {},
	"audio/aac":                    {},
	"audio/mp4":                    {},
	"audio/mpeg":                   {},
	"audio/wav":                    {},
	"audio/webm":                   {},
	"image/gif":                    {},
	"image/jpeg":                   {},
	"image/png":                    {},
	"video/h264":                   {},
	"video/mp4":                    {},
	"video/mpeg":                   {},
	"video/webm":                   {},
	"video/x-flv":                  {},
}

// compressibleContentType indicates whether the content of ct type can be compressed.
func compressibleContentType(ct string) bool {
	_, ok := incompressibleTypes[strings.ToLower(ct)]
	return !ok
}
<|endoftext|>


<commit_before>package marketplace

import (
	"fmt"
	"strings"
	"time"

	g "github.com/onsi/ginkgo"
	o "github.com/onsi/gomega"

	exutil "github.com/openshift/origin/test/extended/util"
	"k8s.io/apimachinery/pkg/util/wait"
	e2e "k8s.io/kubernetes/test/e2e/framework"
)

var _ = g.Describe("[Feature:Marketplace] Marketplace resources with labels provider displayName", func() {

	defer g.GinkgoRecover()

	var (
		oc            = exutil.NewCLI("marketplace", exutil.KubeConfigPath())
		allNs         = "openshift-operators"
		marketplaceNs = "openshift-marketplace"
		ResourceWait  = 60 * time.Second

		opsrcYamltem = exutil.FixturePath("testdata", "marketplace", "opsrc", "02-opsrc.yaml")
		cscYamltem   = exutil.FixturePath("testdata", "marketplace", "csc", "02-csc.yaml")
	)

	g.AfterEach(func() {
		//clear the sub,csv resource
		allresourcelist := [][]string{
			{"operatorsource", "opsrctestlabel", marketplaceNs},
			{"catalogsourceconfig", "csctestlabel", marketplaceNs},
		}

		for _, source := range allresourcelist {
			err := clearResources(oc, source[0], source[1], source[2])
			o.Expect(err).NotTo(o.HaveOccurred())
		}
	})

	//OCP-21728 check the publisher,display,labels of opsrc&csc
	g.It("[ocp-21728]create opsrc with labels", func() {

		//create one opsrc with label
		opsrcYaml, err := oc.AsAdmin().Run("process").Args("--ignore-unknown-parameters=true", "-f", opsrcYamltem, "-p", "NAME=opsrctestlabel", "NAMESPACE=marketplace_e2e", "LABEL=optestlabel", "DISPLAYNAME=optestlabel", "PUBLISHER=optestlabel", fmt.Sprintf("MARKETPLACE=%s", marketplaceNs)).OutputToFile("config.json")
		o.Expect(err).NotTo(o.HaveOccurred())

		err = createResources(oc, opsrcYaml)
		o.Expect(err).NotTo(o.HaveOccurred())

		//wait for the opsrc is created finished
		err = wait.Poll(5*time.Second, ResourceWait, func() (bool, error) {
			output, err := oc.AsAdmin().Run("get").Args("operatorsource", "opsrctestlabel", "-o=jsonpath={.status.currentPhase.phase.message}", "-n", marketplaceNs).Output()
			if err != nil {
				e2e.Failf("Failed to create opsrctestlabel, error:%v", err)
				return false, err
			}
			if strings.Contains(output, "has been successfully reconciled") {
				return true, nil
			}
			return false, nil
		})

		o.Expect(err).NotTo(o.HaveOccurred())

		opsrcResourceList := [][]string{
			{"operatorsource", "opsrctestlabel", "-o=jsonpath={.metadata.labels.opsrc-provider}", marketplaceNs},
			{"operatorsource", "opsrctestlabel", "-o=jsonpath={.spec.displayName}", marketplaceNs},
			{"operatorsource", "opsrctestlabel", "-o=jsonpath={.spec.publisher}", marketplaceNs},
			{"catalogsource", "opsrctestlabel", "-o=jsonpath={.metadata.labels.opsrc-provider}", marketplaceNs},
			{"catalogsource", "opsrctestlabel", "-o=jsonpath={.spec.displayName}", marketplaceNs},
			{"catalogsource", "opsrctestlabel", "-o=jsonpath={.spec.publisher}", marketplaceNs},
		}
		//check the displayname,provider,labels of opsrc & catalogsource
		for _, source := range opsrcResourceList {
			msg, _ := getResourceByPath(oc, source[0], source[1], source[2], source[3])
			o.Expect(msg).Should(o.ContainSubstring("optestlabel"))
		}
		//create one csc with provider&display&labels
		cscYaml, err := oc.AsAdmin().Run("process").Args("--ignore-unknown-parameters=true", "-f", cscYamltem, "-p", "NAME=csctestlabel", fmt.Sprintf("NAMESPACE=%s", allNs), fmt.Sprintf("MARKETPLACE=%s", marketplaceNs), "PACKAGES=camel-k-marketplace-e2e-tests", "DISPLAYNAME=csctestlabel", "PUBLISHER=csctestlabel").OutputToFile("config.json")
		err = createResources(oc, cscYaml)
		o.Expect(err).NotTo(o.HaveOccurred())

		//wait for the csc is created finished
		err = wait.Poll(5*time.Second, ResourceWait, func() (bool, error) {
			output, err := oc.AsAdmin().Run("get").Args("catalogsourceconfig", "csctestlabel", "-o=jsonpath={.status.currentPhase.phase.message}", "-n", marketplaceNs).Output()
			if err != nil {
				e2e.Failf("Failed to create csctestlabel, error:%v", err)
				return false, err
			}
			if strings.Contains(output, "has been successfully reconciled") {
				return true, nil
			}
			return false, nil
		})

		o.Expect(err).NotTo(o.HaveOccurred())

		cscResourceList := [][]string{
			{"catalogsourceconfig", "csctestlabel", "-o=jsonpath={.spec.csDisplayName}", marketplaceNs},
			{"catalogsourceconfig", "csctestlabel", "-o=jsonpath={.spec.csPublisher}", marketplaceNs},
			{"catalogsource", "csctestlabel", "-o=jsonpath={.spec.displayName}", allNs},
			{"catalogsource", "csctestlabel", "-o=jsonpath={.spec.publisher}", allNs},
		}
		//check the displayname,provider oc csc & catalogsource
		for _, source := range cscResourceList {
			msg, _ := getResourceByPath(oc, source[0], source[1], source[2], source[3])
			o.Expect(msg).Should(o.ContainSubstring("csctestlabel"))
		}

		//get the packagelist of opsrctestlabel
		packageListOpsrc1, _ := getResourceByPath(oc, "operatorsource", "opsrctestlabel", "-o=jsonpath={.status.packages}", marketplaceNs)
		packageList := strings.Split(packageListOpsrc1, ",")

		//get the packagelist with label of opsrctestlabel
		packageListOpsrc2, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("packagemanifests", "-lopsrc-provider=optestlabel", "-o=name", "-n", marketplaceNs).Output()
		for _, packages := range packageList {
			o.Expect(packageListOpsrc2).Should(o.ContainSubstring(packages))
		}
	})
})
<commit_msg>change the resourcetime<commit_after>package marketplace

import (
	"fmt"
	"strings"
	"time"

	g "github.com/onsi/ginkgo"
	o "github.com/onsi/gomega"

	exutil "github.com/openshift/origin/test/extended/util"
	"k8s.io/apimachinery/pkg/util/wait"
	e2e "k8s.io/kubernetes/test/e2e/framework"
)

var _ = g.Describe("[Feature:Marketplace] Marketplace resources with labels provider displayName", func() {

	defer g.GinkgoRecover()

	var (
		oc            = exutil.NewCLI("marketplace", exutil.KubeConfigPath())
		allNs         = "openshift-operators"
		marketplaceNs = "openshift-marketplace"
		resourceWait  = 60 * time.Second

		opsrcYamltem = exutil.FixturePath("testdata", "marketplace", "opsrc", "02-opsrc.yaml")
		cscYamltem   = exutil.FixturePath("testdata", "marketplace", "csc", "02-csc.yaml")
	)

	g.AfterEach(func() {
		//clear the sub,csv resource
		allresourcelist := [][]string{
			{"operatorsource", "opsrctestlabel", marketplaceNs},
			{"catalogsourceconfig", "csctestlabel", marketplaceNs},
		}

		for _, source := range allresourcelist {
			err := clearResources(oc, source[0], source[1], source[2])
			o.Expect(err).NotTo(o.HaveOccurred())
		}
	})

	//OCP-21728 check the publisher,display,labels of opsrc&csc
	g.It("[ocp-21728]create opsrc with labels", func() {

		//create one opsrc with label
		opsrcYaml, err := oc.AsAdmin().Run("process").Args("--ignore-unknown-parameters=true", "-f", opsrcYamltem, "-p", "NAME=opsrctestlabel", "NAMESPACE=marketplace_e2e", "LABEL=optestlabel", "DISPLAYNAME=optestlabel", "PUBLISHER=optestlabel", fmt.Sprintf("MARKETPLACE=%s", marketplaceNs)).OutputToFile("config.json")
		o.Expect(err).NotTo(o.HaveOccurred())

		err = createResources(oc, opsrcYaml)
		o.Expect(err).NotTo(o.HaveOccurred())

		//wait for the opsrc is created finished
		err = wait.Poll(5*time.Second, ResourceWait, func() (bool, error) {
			output, err := oc.AsAdmin().Run("get").Args("operatorsource", "opsrctestlabel", "-o=jsonpath={.status.currentPhase.phase.message}", "-n", marketplaceNs).Output()
			if err != nil {
				e2e.Failf("Failed to create opsrctestlabel, error:%v", err)
				return false, err
			}
			if strings.Contains(output, "has been successfully reconciled") {
				return true, nil
			}
			return false, nil
		})

		o.Expect(err).NotTo(o.HaveOccurred())

		opsrcResourceList := [][]string{
			{"operatorsource", "opsrctestlabel", "-o=jsonpath={.metadata.labels.opsrc-provider}", marketplaceNs},
			{"operatorsource", "opsrctestlabel", "-o=jsonpath={.spec.displayName}", marketplaceNs},
			{"operatorsource", "opsrctestlabel", "-o=jsonpath={.spec.publisher}", marketplaceNs},
			{"catalogsource", "opsrctestlabel", "-o=jsonpath={.metadata.labels.opsrc-provider}", marketplaceNs},
			{"catalogsource", "opsrctestlabel", "-o=jsonpath={.spec.displayName}", marketplaceNs},
			{"catalogsource", "opsrctestlabel", "-o=jsonpath={.spec.publisher}", marketplaceNs},
		}
		//check the displayname,provider,labels of opsrc & catalogsource
		for _, source := range opsrcResourceList {
			msg, _ := getResourceByPath(oc, source[0], source[1], source[2], source[3])
			o.Expect(msg).Should(o.ContainSubstring("optestlabel"))
		}
		//create one csc with provider&display&labels
		cscYaml, err := oc.AsAdmin().Run("process").Args("--ignore-unknown-parameters=true", "-f", cscYamltem, "-p", "NAME=csctestlabel", fmt.Sprintf("NAMESPACE=%s", allNs), fmt.Sprintf("MARKETPLACE=%s", marketplaceNs), "PACKAGES=camel-k-marketplace-e2e-tests", "DISPLAYNAME=csctestlabel", "PUBLISHER=csctestlabel").OutputToFile("config.json")
		err = createResources(oc, cscYaml)
		o.Expect(err).NotTo(o.HaveOccurred())

		//wait for the csc is created finished
		err = wait.Poll(5*time.Second, ResourceWait, func() (bool, error) {
			output, err := oc.AsAdmin().Run("get").Args("catalogsourceconfig", "csctestlabel", "-o=jsonpath={.status.currentPhase.phase.message}", "-n", marketplaceNs).Output()
			if err != nil {
				e2e.Failf("Failed to create csctestlabel, error:%v", err)
				return false, err
			}
			if strings.Contains(output, "has been successfully reconciled") {
				return true, nil
			}
			return false, nil
		})

		o.Expect(err).NotTo(o.HaveOccurred())

		cscResourceList := [][]string{
			{"catalogsourceconfig", "csctestlabel", "-o=jsonpath={.spec.csDisplayName}", marketplaceNs},
			{"catalogsourceconfig", "csctestlabel", "-o=jsonpath={.spec.csPublisher}", marketplaceNs},
			{"catalogsource", "csctestlabel", "-o=jsonpath={.spec.displayName}", allNs},
			{"catalogsource", "csctestlabel", "-o=jsonpath={.spec.publisher}", allNs},
		}
		//check the displayname,provider oc csc & catalogsource
		for _, source := range cscResourceList {
			msg, _ := getResourceByPath(oc, source[0], source[1], source[2], source[3])
			o.Expect(msg).Should(o.ContainSubstring("csctestlabel"))
		}

		//get the packagelist of opsrctestlabel
		packageListOpsrc1, _ := getResourceByPath(oc, "operatorsource", "opsrctestlabel", "-o=jsonpath={.status.packages}", marketplaceNs)
		packageList := strings.Split(packageListOpsrc1, ",")

		//get the packagelist with label of opsrctestlabel
		packageListOpsrc2, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("packagemanifests", "-lopsrc-provider=optestlabel", "-o=name", "-n", marketplaceNs).Output()
		for _, packages := range packageList {
			o.Expect(packageListOpsrc2).Should(o.ContainSubstring(packages))
		}
	})
})
<|endoftext|>


<commit_before>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package ingress

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"

	networkingv1beta1 "k8s.io/api/networking/v1beta1"
)

func TestIngressToManifest(t *testing.T) {
	ing := &networkingv1beta1.Ingress{}
	// Create a temp dir.
	tmpDir, err := ioutil.TempDir("", "kubemci")
	if err != nil {
		t.Fatalf("unexpected error in creating temp dir: %s", err)
	}
	defer os.RemoveAll(tmpDir)
	ingPath := filepath.Join(tmpDir, "ing.yaml")

	// Write the ingress to a file and ensure that there is no error.
	if err := ingressToManifest(ing, ingPath); err != nil {
		t.Fatalf("Error in creating file: %s", err)
	}
	// Writing it again should not return an error.
	if err := ingressToManifest(ing, ingPath); err != nil {
		t.Fatalf("Error in creating file: %s", err)
	}
}
<commit_msg>Update copyright date for ingress_utils_test.go<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package ingress

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"

	networkingv1beta1 "k8s.io/api/networking/v1beta1"
)

func TestIngressToManifest(t *testing.T) {
	ing := &networkingv1beta1.Ingress{}
	// Create a temp dir.
	tmpDir, err := ioutil.TempDir("", "kubemci")
	if err != nil {
		t.Fatalf("unexpected error in creating temp dir: %s", err)
	}
	defer os.RemoveAll(tmpDir)
	ingPath := filepath.Join(tmpDir, "ing.yaml")

	// Write the ingress to a file and ensure that there is no error.
	if err := ingressToManifest(ing, ingPath); err != nil {
		t.Fatalf("Error in creating file: %s", err)
	}
	// Writing it again should not return an error.
	if err := ingressToManifest(ing, ingPath); err != nil {
		t.Fatalf("Error in creating file: %s", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"math/rand"
	"strings"
)

func main() {
	const distance = 57600000
	spaceline := ""
	trip := ""

	fmt.Printf("%-18v %4v %v %7v\n", "Spaceline", "Days", "Round-trip", "Price")
	fmt.Println("===============================================")
	for count :=0; count < 10; count++ {

		speed := rand.Intn(16) + 15
		days := (distance / speed) / 86164
		price := days / 2

		switch rand.Intn(3) {
		case 0:
			spaceline = "Space Adventures"
		case 1:
			spaceline = "SpaceX"
		case 2:
			spaceline = "Virgin Galactic"
		}

		switch rand.Intn(2) {
		case 0:
			trip = "One Way"
		case 1:
			trip = "Round Trip"
		}

		if strings.Contains(trip, "Round") {
			price = price * 2
		}

		fmt.Printf("%-18v %4v %v $%4v\n", spaceline, days, trip, price)
	}
}
<commit_msg>Updated days so they reflect total trip time, also added in proper formatting, which I had to look up.<commit_after>package main

import (
	"fmt"
	"math/rand"
	"strings"
)

func main() {
	const distance = 57600000
	spaceline := ""
	trip := ""

	fmt.Printf("%-18v %4v %v %7v\n", "Spaceline", "Days", "Round-trip", "Price")
	fmt.Println("===============================================")
	for count :=0; count < 10; count++ {

		speed := rand.Intn(16) + 15
		days := (distance / speed) / 86164
		price := days / 2

		switch rand.Intn(3) {
		case 0:
			spaceline = "Space Adventures"
		case 1:
			spaceline = "SpaceX"
		case 2:
			spaceline = "Virgin Galactic"
		}

		switch rand.Intn(2) {
		case 0:
			trip = "One Way"
		case 1:
			trip = "Round Trip"
		}

		if strings.Contains(trip, "Round") {
			price = price * 2
			days = days * 2
		}

		fmt.Printf("%-18v %4v %-10v $%4v\n", spaceline, days, trip, price)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/route53"
	"github.com/koding/logging"
)

const (
	// callerReferance is used as unique id for idempotency on aws side
	callerReferance         = "tunnelproxy_dev_1"
	hostedZoneName          = "tunnelproxy.koding.com"
	hostedZoneComment       = "Hosted zone for tunnel proxies"
	validStateForHostedZone = "INSYNC" // taken from aws response
)

var (
	errHostedZoneNotFound           = errors.New("hosted zone not found")
	errDeadlineReachedForChangeInfo = errors.New("deadline for change info")
)

type RecordManager struct {
	svc        *route53.Route53
	hostedZone *route53.HostedZone
	log        logging.Logger
	region     string
}

func New(config *aws.Config, log logging.Logger) (*RecordManager, error) {
	return &RecordManager{
		svc:    route53.New(config),
		log:    log.New("recordmanager"),
		region: "us-east-1", // TODO make configurable
	}, nil
}

func (r *RecordManager) Init() error {
	r.log.Debug("init started")

	hostedZoneLogger := r.log.New("HostedZone")
	hostedZoneLogger.Debug("working...")

	err := r.getHostedZone(hostedZoneLogger)
	if err != nil && err != errHostedZoneNotFound {
		return err
	}

	if err == errHostedZoneNotFound {
		hostedZoneLogger.Debug("Not found, creating...")
		err := r.createHostedZone(hostedZoneLogger)
		if err != nil {
			return err
		}
	}

	hostedZoneLogger.Debug("hosted zone is ready")
	hostedZoneLogger.Info("RecordManager is ready")
	return err
}

func (r *RecordManager) getHostedZone(hostedZoneLogger logging.Logger) error {
	iteration := 0
	// try to get our hosted zone
	for {
		// just be paranoid about remove api calls, dont harden too much
		if iteration == maxIteration {
			return errMaxIterationReached
		}

		log := hostedZoneLogger.New("iteration", iteration)

		iteration++

		// for pagination
		var nextMarker *string

		log.Debug("fetching hosted zone")
		listHostedZonesResp, err := r.svc.ListHostedZones(
			&route53.ListHostedZonesInput{
				Marker: nextMarker,
			}, // we dont have anything to filter
		)
		if err != nil {
			return err
		}

		if listHostedZonesResp == nil {
			return errors.New("malformed response")
		}

		for _, hostedZone := range listHostedZonesResp.HostedZones {
			if *hostedZone.CallerReference == callerReferance {
				r.hostedZone = hostedZone
				log.Debug("hosted zone found")
				return nil
			}
		}

		// if our result set is truncated we can try to fecth again, but if we
		// reach to end, nothing to do left
		if !*listHostedZonesResp.IsTruncated {
			return errHostedZoneNotFound
		}

		// assign next marker
		nextMarker = listHostedZonesResp.NextMarker
	}
}

// createHostedZone creates hosted zone and makes sure that it is in to be used
// state
func (r *RecordManager) createHostedZone(hostedZoneLogger logging.Logger) error {
	hostedZoneLogger.Debug("create hosted zone started")

	resp, err := r.svc.CreateHostedZone(&route53.CreateHostedZoneInput{
		CallerReference: aws.String(callerReferance),
		Name:            aws.String(hostedZoneName),
		HostedZoneConfig: &route53.HostedZoneConfig{
			Comment: aws.String(hostedZoneComment),
		},
	})

	if err != nil {
		return err
	}

	if resp == nil {
		return errors.New("malformed response, resp is nil")
	}

	changeInfo := resp.ChangeInfo
	deadline := time.After(time.Minute * 4)

	// make sure it propagated
	for {
		// if our change propagated, we can return
		if *changeInfo.Status == validStateForHostedZone {
			hostedZoneLogger.Debug("hosted zone status is valid")
			break
		}

		select {
		case <-deadline:
			return errDeadlineReachedForChangeInfo
		default:
			time.Sleep(time.Second * 3) // poor man's throttling
			hostedZoneLogger.New("changeInfoID", *changeInfo.ID).Debug("fetching latest status")
			getChangeResp, err := r.svc.GetChange(&route53.GetChangeInput{
				ID: changeInfo.ID,
			})
			if err != nil {
				return err
			}

			if getChangeResp == nil {
				return errors.New("malformed response, getChangeResp is nil")
			}

			changeInfo = getChangeResp.ChangeInfo
		}
	}

	r.hostedZone = resp.HostedZone

	return nil
}

func (r *RecordManager) createRecordSet() error {
	if r.hostedZone == nil {
		return errors.New("hosted zone is not set")
	}

	params := &route53.ChangeResourceRecordSetsInput{
		ChangeBatch: &route53.ChangeBatch{
			Changes: []*route53.Change{
				&route53.Change{
					Action: aws.String("UPSERT"),
					ResourceRecordSet: &route53.ResourceRecordSet{
						Name:   aws.String(hostedZoneName),
						Type:   aws.String("A"),
						Region: aws.String(r.region),
						ResourceRecords: []*route53.ResourceRecord{
							// TODO these will be actual ip adreesses of proxy machines
							&route53.ResourceRecord{
								Value: aws.String("52.7.21.41"),
							},
							&route53.ResourceRecord{
								Value: aws.String("52.1.117.108"),
							},
						},
						// use region name as identifer
						SetIdentifier: aws.String(r.region),
						TTL:           aws.Long(1),
					},
				},
			},
			Comment: aws.String(
				fmt.Sprintf(
					"Record set for zone: %s region: %s",
					hostedZoneName,
					r.region,
				),
			),
		},
		HostedZoneID: r.hostedZone.ID,
	}

	_, err := r.svc.ChangeResourceRecordSets(params)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Tunnel: proper naming for records<commit_after>package main

import (
	"errors"
	"fmt"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/route53"
	"github.com/koding/logging"
)

const (
	// callerReferance is used as unique id for idempotency on aws side
	callerReferance         = "tunnelproxy_dev_1"
	hostedZoneName          = "tunnelproxy.koding.com"
	hostedZoneComment       = "Hosted zone for tunnel proxies"
	validStateForHostedZone = "INSYNC" // taken from aws response
)

var (
	errHostedZoneNotFound           = errors.New("hosted zone not found")
	errDeadlineReachedForChangeInfo = errors.New("deadline for change info")
)

// RecordManager manages Route53 records
type RecordManager struct {
	// aws services
	route53 *route53.Route53

	// application wide parameters
	hostedZone *route53.HostedZone
	region     string

	// general usage
	log logging.Logger
}

// New creates a RecordManager
func NewRecordManager(config *aws.Config, log logging.Logger, region string) (*RecordManager, error) {
	return &RecordManager{
		route53: route53.New(config),
		log:     log.New("recordmanager"),
		region:  region,
	}, nil
}

// Init initializes record manager's prerequisites
func (r *RecordManager) Init() error {
	r.log.Debug("init started")

	if err := r.ensureHostedZone(); err != nil {
		return err
	}

	r.log.Info("RecordManager is ready")
	return nil
}

func (r *RecordManager) ensureHostedZone() error {
	hostedZoneLogger := r.log.New("HostedZone")
	hostedZoneLogger.Debug("working...")

	err := r.getHostedZone(hostedZoneLogger)
	if err != nil && err != errHostedZoneNotFound {
		return err
	}

	if err == errHostedZoneNotFound {
		hostedZoneLogger.Debug("Not found, creating...")
		err := r.createHostedZone(hostedZoneLogger)
		if err != nil {
			return err
		}
	}

	hostedZoneLogger.Debug("hosted zone is ready")

	return nil
}

func (r *RecordManager) getHostedZone(hostedZoneLogger logging.Logger) error {
	iteration := 0
	// try to get our hosted zone
	for {
		// just be paranoid about remove api calls, dont harden too much
		if iteration == maxIteration {
			return errMaxIterationReached
		}

		log := hostedZoneLogger.New("iteration", iteration)

		iteration++

		// for pagination
		var nextMarker *string

		log.Debug("fetching hosted zone")
		listHostedZonesResp, err := r.route53.ListHostedZones(
			&route53.ListHostedZonesInput{
				Marker: nextMarker,
			}, // we dont have anything to filter
		)
		if err != nil {
			return err
		}

		if listHostedZonesResp == nil {
			return errors.New("malformed response")
		}

		for _, hostedZone := range listHostedZonesResp.HostedZones {
			if *hostedZone.CallerReference == callerReferance {
				r.hostedZone = hostedZone
				log.Debug("hosted zone found")
				return nil
			}
		}

		// if our result set is truncated we can try to fecth again, but if we
		// reach to end, nothing to do left
		if !*listHostedZonesResp.IsTruncated {
			return errHostedZoneNotFound
		}

		// assign next marker
		nextMarker = listHostedZonesResp.NextMarker
	}
}

// createHostedZone creates hosted zone and makes sure that it is in to be used
// state
func (r *RecordManager) createHostedZone(hostedZoneLogger logging.Logger) error {
	hostedZoneLogger.Debug("create hosted zone started")

	resp, err := r.route53.CreateHostedZone(&route53.CreateHostedZoneInput{
		CallerReference: aws.String(callerReferance),
		Name:            aws.String(hostedZoneName),
		HostedZoneConfig: &route53.HostedZoneConfig{
			Comment: aws.String(hostedZoneComment),
		},
	})

	if err != nil {
		return err
	}

	if resp == nil {
		return errors.New("malformed response, resp is nil")
	}

	changeInfo := resp.ChangeInfo
	deadline := time.After(time.Minute * 4)

	// make sure it propagated
	for {
		// if our change propagated, we can return
		if *changeInfo.Status == validStateForHostedZone {
			hostedZoneLogger.Debug("hosted zone status is valid")
			break
		}

		select {
		case <-deadline:
			return errDeadlineReachedForChangeInfo
		default:
			time.Sleep(time.Second * 3) // poor man's throttling
			hostedZoneLogger.New("changeInfoID", *changeInfo.ID).Debug("fetching latest status")
			getChangeResp, err := r.route53.GetChange(&route53.GetChangeInput{
				ID: changeInfo.ID,
			})
			if err != nil {
				return err
			}

			if getChangeResp == nil {
				return errors.New("malformed response, getChangeResp is nil")
			}

			changeInfo = getChangeResp.ChangeInfo
		}
	}

	r.hostedZone = resp.HostedZone

	return nil
}

func (r *RecordManager) createRecordSet() error {
	if r.hostedZone == nil {
		return errors.New("hosted zone is not set")
	}

	params := &route53.ChangeResourceRecordSetsInput{
		ChangeBatch: &route53.ChangeBatch{
			Changes: []*route53.Change{
				&route53.Change{
					Action: aws.String("UPSERT"),
					ResourceRecordSet: &route53.ResourceRecordSet{
						Name:   aws.String(hostedZoneName),
						Type:   aws.String("A"),
						Region: aws.String(r.region),
						ResourceRecords: []*route53.ResourceRecord{
							// TODO these will be actual ip adreesses of proxy machines
							&route53.ResourceRecord{
								Value: aws.String("52.7.21.41"),
							},
							&route53.ResourceRecord{
								Value: aws.String("52.1.117.108"),
							},
						},
						// use region name as identifer
						SetIdentifier: aws.String(r.region),
						TTL:           aws.Long(1),
					},
				},
			},
			Comment: aws.String(
				fmt.Sprintf(
					"Record set for zone: %s region: %s",
					hostedZoneName,
					r.region,
				),
			),
		},
		HostedZoneID: r.hostedZone.ID,
	}

	_, err := r.route53.ChangeResourceRecordSets(params)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

// Package tailer provides a class that is responsible for tailing log files
// and extracting new log lines to be passed into the virtual machines.
package tailer

// For regular files, mtail gets notified on modifications (i.e. appends) to
// log files that are being watched, in order to read the new lines. Log files
// can also be rotated, so mtail is also notified of creates in the log file
// directory.

import (
	"errors"
	"expvar"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"sync"
	"syscall"
	"time"
	"unicode/utf8"

	"github.com/golang/glog"

	"github.com/google/mtail/watcher"

	"github.com/spf13/afero"
)

var (
	logCount     = expvar.NewInt("log_count")
	logErrors    = expvar.NewMap("log_errors_total")
	logRotations = expvar.NewMap("log_rotations_total")
)

// Tailer receives notification of changes from a Watcher and extracts new log
// lines from files. It also handles new log file creation events and log
// rotations.
type Tailer struct {
	w watcher.Watcher

	watched     map[string]struct{}   // Names of logs being watched.
	watchedLock sync.RWMutex          // protects `watched'
	lines       chan<- string         // Logfile lines being emitted.
	files       map[string]afero.File // File handles for each pathname.
	filesLock   sync.Mutex            // protects `files'
	partials    map[string]string     // Accumulator for the currently read line for each pathname.

	fs afero.Fs // mockable filesystem interface
}

// Options configures a Tailer
type Options struct {
	Lines chan<- string
	W     watcher.Watcher // Not required, will use watcher.LogWatcher if it is zero.
	FS    afero.Fs        // Not required, will use afero.OsFs if it is zero.
}

// New returns a new Tailer, configured with the supplied Options
func New(o Options) (*Tailer, error) {
	if o.Lines == nil {
		return nil, errors.New("tailer needs lines")
	}
	fs := o.FS
	if fs == nil {
		fs = &afero.OsFs{}
	}
	w := o.W
	if w == nil {
		var err error
		w, err = watcher.NewLogWatcher()
		if err != nil {
			return nil, fmt.Errorf("Couldn't create a watcher for tailer: %s", err)
		}
	}
	t := &Tailer{
		w:        w,
		watched:  make(map[string]struct{}),
		lines:    o.Lines,
		files:    make(map[string]afero.File),
		partials: make(map[string]string),
		fs:       fs,
	}
	go t.run()
	return t, nil
}

// addWatched adds a path to the list of watched items.
func (t *Tailer) addWatched(path string) {
	t.watchedLock.Lock()
	defer t.watchedLock.Unlock()
	t.watched[path] = struct{}{}
}

// isWatching indicates if the path is being watched. It includes both
// filenames and directories.
func (t *Tailer) isWatching(path string) bool {
	t.watchedLock.RLock()
	defer t.watchedLock.RUnlock()
	_, ok := t.watched[path]
	return ok
}

// Tail registers a file path to be tailed.
func (t *Tailer) Tail(pathname string) {
	fullpath, err := filepath.Abs(pathname)
	if err != nil {
		glog.Infof("Failed to find absolute path for %q: %s\n", pathname, err)
		return
	}
	if !t.isWatching(fullpath) {
		t.addWatched(fullpath)
		logCount.Add(1)
		t.openLogPath(fullpath, false)
	}
}

// TailFile registers a file descriptor to be tailed.
func (t *Tailer) TailFile(f afero.File) error {
	logCount.Add(1)
	return t.startNewFile(f, false)
}

// handleLogUpdate reads all available bytes from an already opened file
// identified by pathname, and sends them to be processed on the lines channel.
func (t *Tailer) handleLogUpdate(pathname string) {
	t.filesLock.Lock()
	fd, ok := t.files[pathname]
	t.filesLock.Unlock()
	if !ok {
		glog.Warningf("No file descriptor found for %q, but is being watched", pathname)
		return
	}
	var err error
	t.partials[pathname], err = t.read(fd, t.partials[pathname])
	if err != nil && err != io.EOF {
		glog.Info(err)
	}
}

// read reads blocks of 4096 bytes from the File, sending lines to the
// channel as it encounters newlines.  If EOF is encountered, the partial line
// is returned to be concatenated with on the next call.
func (t *Tailer) read(f afero.File, partialIn string) (partialOut string, err error) {
	partial := partialIn
	b := make([]byte, 0, 4096)
	for {
		n, err := f.Read(b[:cap(b)])
		b = b[:n]
		if err != nil {
			return partial, err
		}

		for i, width := 0, 0; i < len(b) && i < n; i += width {
			var rune rune
			rune, width = utf8.DecodeRune(b[i:])
			switch {
			case rune != '\n':
				partial += string(rune)
			default:
				// send off line for processing
				t.lines <- partial
				// reset accumulator
				partial = ""
			}
		}
	}
}

// inode returns the inode number of a file, or 0 if the file has no underlying Sys implementation.
func inode(f os.FileInfo) uint64 {
	s := f.Sys()
	if s == nil {
		return 0
	}
	switch s := s.(type) {
	case *syscall.Stat_t:
		return s.Ino
	default:
		return 0
	}
}

// handleLogCreate handles both new and rotated log files.
func (t *Tailer) handleLogCreate(pathname string) {
	t.filesLock.Lock()
	fd, ok := t.files[pathname]
	t.filesLock.Unlock()
	if ok {
		s1, err := fd.Stat()
		if err != nil {
			glog.Infof("Stat failed on %q: %s", t.files[pathname].Name(), err)
			return
		}
		s2, err := t.fs.Stat(pathname)
		if err != nil {
			glog.Infof("Stat failed on %q: %s", pathname, err)
			return
		}
		if inode(s1) != inode(s2) {
			logRotations.Add(pathname, 1)
			// flush the old log, pathname is still an index into t.files with the old inode.
			t.handleLogUpdate(pathname)
			fd.Close()
			err := t.w.Remove(pathname)
			if err != nil {
				glog.Info("Failed removing watches on", pathname)
			}
			t.openLogPath(pathname, true)
		} else {
			glog.Infof("Path %s already being watched, and inode not changed.",
				pathname)
		}
	} else {
		// Freshly opened log file, never seen before.
		t.openLogPath(pathname, true)
	}
}

// openLogPath opens a new log file at pathname, and optionally seeks to the
// start or end of the file. Rotated logs should read from the start, but logs
// opened for the first time read from the end.
func (t *Tailer) openLogPath(pathname string, seenBefore bool) {
	d := path.Dir(pathname)
	if !t.isWatching(d) {
		err := t.w.Add(d)
		if err != nil {
			glog.Infof("Adding a create watch failed on %q: %s", d, err)
		}
		t.addWatched(d)
	}

	retries := 3
	retryDelay := 1 * time.Millisecond
	var f afero.File
	var err error
	for retries > 0 {
		f, err = t.fs.Open(pathname)
		if err == nil {
			break
		}
		// Doesn't exist yet. We're watching the directory, so we'll pick it up
		// again on create; return successfully.
		if os.IsNotExist(err) {
			return
		}
		glog.Infof("Failed to open %q for reading: %s", pathname, err)
		logErrors.Add(pathname, 1)
		// seenBefore indicates also that we're rotating a file that previously worked, so retry.
		if seenBefore {
			retries = retries - 1
			time.Sleep(retryDelay)
			retryDelay = retryDelay + retryDelay
		} else {
			return
		}
	}
	err = t.startNewFile(f, seenBefore)
	if err != nil {
		glog.Error(err)
	}
}

func (t *Tailer) startNewFile(f afero.File, seekStart bool) error {
	fi, err := f.Stat()
	if err != nil {
		// Stat failed, log error and return.
		logErrors.Add(f.Name(), 1)
		return fmt.Errorf("Failed to stat %q: %s", f.Name(), err)
	}
	switch m := fi.Mode(); {
	case m&os.ModeType == 0:
		if seekStart {
			f.Seek(0, os.SEEK_SET)
		} else {
			f.Seek(0, os.SEEK_END)
		}
		err = t.w.Add(f.Name())
		if err != nil {
			return fmt.Errorf("Adding a change watch failed on %q: %s", f.Name(), err)
		}
		// In case the new log has been written to already, attempt to read the
		// first lines.
		t.partials[f.Name()], err = t.read(f, "")
		if err != nil {
			if err == io.EOF {
				// Don't worry about EOF on first read, that's expected.
				break
			}
			return err
		}
	case m&os.ModeType == os.ModeNamedPipe:
		go t.readForever(f)
	default:
		return fmt.Errorf("Can't open files with mode %v: %s", m&os.ModeType, f.Name())
	}
	t.filesLock.Lock()
	t.files[f.Name()] = f
	t.filesLock.Unlock()
	glog.Infof("Tailing %s", f.Name())

	return nil
}

// start is the main event loop for the Tailer.
// It receives notification of log file changes from the watcher channel, and
// handles them.
func (t *Tailer) run() {
	for e := range t.w.Events() {
		switch e := e.(type) {
		case watcher.UpdateEvent:
			if t.isWatching(e.Pathname) {
				t.handleLogUpdate(e.Pathname)
			}
		case watcher.CreateEvent:
			if t.isWatching(e.Pathname) {
				t.handleLogCreate(e.Pathname)
			}
		default:
			glog.Infof("Unexpected event %#v", e)
		}
	}
	glog.Infof("Shutting down tailer.")
	close(t.lines)
}

func (t *Tailer) readForever(f afero.File) {
	var err error
	partial := ""
	for {
		partial, err = t.read(f, partial)
		// We want to exit at EOF, because the FD has been closed.
		if err != nil {
			glog.Infof("%s: %s", err, f.Name())
			return
		}
	}
}

// Close signals termination to the watcher.
func (t *Tailer) Close() {
	t.w.Close()
}
<commit_msg>Don't log "unexpected" when a delete event arrives.<commit_after>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

// Package tailer provides a class that is responsible for tailing log files
// and extracting new log lines to be passed into the virtual machines.
package tailer

// For regular files, mtail gets notified on modifications (i.e. appends) to
// log files that are being watched, in order to read the new lines. Log files
// can also be rotated, so mtail is also notified of creates in the log file
// directory.

import (
	"errors"
	"expvar"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"sync"
	"syscall"
	"time"
	"unicode/utf8"

	"github.com/golang/glog"

	"github.com/google/mtail/watcher"

	"github.com/spf13/afero"
)

var (
	logCount     = expvar.NewInt("log_count")
	logErrors    = expvar.NewMap("log_errors_total")
	logRotations = expvar.NewMap("log_rotations_total")
)

// Tailer receives notification of changes from a Watcher and extracts new log
// lines from files. It also handles new log file creation events and log
// rotations.
type Tailer struct {
	w watcher.Watcher

	watched     map[string]struct{}   // Names of logs being watched.
	watchedLock sync.RWMutex          // protects `watched'
	lines       chan<- string         // Logfile lines being emitted.
	files       map[string]afero.File // File handles for each pathname.
	filesLock   sync.Mutex            // protects `files'
	partials    map[string]string     // Accumulator for the currently read line for each pathname.

	fs afero.Fs // mockable filesystem interface
}

// Options configures a Tailer
type Options struct {
	Lines chan<- string
	W     watcher.Watcher // Not required, will use watcher.LogWatcher if it is zero.
	FS    afero.Fs        // Not required, will use afero.OsFs if it is zero.
}

// New returns a new Tailer, configured with the supplied Options
func New(o Options) (*Tailer, error) {
	if o.Lines == nil {
		return nil, errors.New("tailer needs lines")
	}
	fs := o.FS
	if fs == nil {
		fs = &afero.OsFs{}
	}
	w := o.W
	if w == nil {
		var err error
		w, err = watcher.NewLogWatcher()
		if err != nil {
			return nil, fmt.Errorf("Couldn't create a watcher for tailer: %s", err)
		}
	}
	t := &Tailer{
		w:        w,
		watched:  make(map[string]struct{}),
		lines:    o.Lines,
		files:    make(map[string]afero.File),
		partials: make(map[string]string),
		fs:       fs,
	}
	go t.run()
	return t, nil
}

// addWatched adds a path to the list of watched items.
func (t *Tailer) addWatched(path string) {
	t.watchedLock.Lock()
	defer t.watchedLock.Unlock()
	t.watched[path] = struct{}{}
}

// isWatching indicates if the path is being watched. It includes both
// filenames and directories.
func (t *Tailer) isWatching(path string) bool {
	t.watchedLock.RLock()
	defer t.watchedLock.RUnlock()
	_, ok := t.watched[path]
	return ok
}

// Tail registers a file path to be tailed.
func (t *Tailer) Tail(pathname string) {
	fullpath, err := filepath.Abs(pathname)
	if err != nil {
		glog.Infof("Failed to find absolute path for %q: %s\n", pathname, err)
		return
	}
	if !t.isWatching(fullpath) {
		t.addWatched(fullpath)
		logCount.Add(1)
		t.openLogPath(fullpath, false)
	}
}

// TailFile registers a file descriptor to be tailed.
func (t *Tailer) TailFile(f afero.File) error {
	logCount.Add(1)
	return t.startNewFile(f, false)
}

// handleLogUpdate reads all available bytes from an already opened file
// identified by pathname, and sends them to be processed on the lines channel.
func (t *Tailer) handleLogUpdate(pathname string) {
	t.filesLock.Lock()
	fd, ok := t.files[pathname]
	t.filesLock.Unlock()
	if !ok {
		glog.Warningf("No file descriptor found for %q, but is being watched", pathname)
		return
	}
	var err error
	t.partials[pathname], err = t.read(fd, t.partials[pathname])
	if err != nil && err != io.EOF {
		glog.Info(err)
	}
}

// read reads blocks of 4096 bytes from the File, sending lines to the
// channel as it encounters newlines.  If EOF is encountered, the partial line
// is returned to be concatenated with on the next call.
func (t *Tailer) read(f afero.File, partialIn string) (partialOut string, err error) {
	partial := partialIn
	b := make([]byte, 0, 4096)
	for {
		n, err := f.Read(b[:cap(b)])
		b = b[:n]
		if err != nil {
			return partial, err
		}

		for i, width := 0, 0; i < len(b) && i < n; i += width {
			var rune rune
			rune, width = utf8.DecodeRune(b[i:])
			switch {
			case rune != '\n':
				partial += string(rune)
			default:
				// send off line for processing
				t.lines <- partial
				// reset accumulator
				partial = ""
			}
		}
	}
}

// inode returns the inode number of a file, or 0 if the file has no underlying Sys implementation.
func inode(f os.FileInfo) uint64 {
	s := f.Sys()
	if s == nil {
		return 0
	}
	switch s := s.(type) {
	case *syscall.Stat_t:
		return s.Ino
	default:
		return 0
	}
}

// handleLogCreate handles both new and rotated log files.
func (t *Tailer) handleLogCreate(pathname string) {
	t.filesLock.Lock()
	fd, ok := t.files[pathname]
	t.filesLock.Unlock()
	if ok {
		s1, err := fd.Stat()
		if err != nil {
			glog.Infof("Stat failed on %q: %s", t.files[pathname].Name(), err)
			return
		}
		s2, err := t.fs.Stat(pathname)
		if err != nil {
			glog.Infof("Stat failed on %q: %s", pathname, err)
			return
		}
		if inode(s1) != inode(s2) {
			logRotations.Add(pathname, 1)
			// flush the old log, pathname is still an index into t.files with the old inode.
			t.handleLogUpdate(pathname)
			fd.Close()
			err := t.w.Remove(pathname)
			if err != nil {
				glog.Info("Failed removing watches on", pathname)
			}
			t.openLogPath(pathname, true)
		} else {
			glog.Infof("Path %s already being watched, and inode not changed.",
				pathname)
		}
	} else {
		// Freshly opened log file, never seen before.
		t.openLogPath(pathname, true)
	}
}

// openLogPath opens a new log file at pathname, and optionally seeks to the
// start or end of the file. Rotated logs should read from the start, but logs
// opened for the first time read from the end.
func (t *Tailer) openLogPath(pathname string, seenBefore bool) {
	d := path.Dir(pathname)
	if !t.isWatching(d) {
		err := t.w.Add(d)
		if err != nil {
			glog.Infof("Adding a create watch failed on %q: %s", d, err)
		}
		t.addWatched(d)
	}

	retries := 3
	retryDelay := 1 * time.Millisecond
	var f afero.File
	var err error
	for retries > 0 {
		f, err = t.fs.Open(pathname)
		if err == nil {
			break
		}
		// Doesn't exist yet. We're watching the directory, so we'll pick it up
		// again on create; return successfully.
		if os.IsNotExist(err) {
			return
		}
		glog.Infof("Failed to open %q for reading: %s", pathname, err)
		logErrors.Add(pathname, 1)
		// seenBefore indicates also that we're rotating a file that previously worked, so retry.
		if seenBefore {
			retries = retries - 1
			time.Sleep(retryDelay)
			retryDelay = retryDelay + retryDelay
		} else {
			return
		}
	}
	err = t.startNewFile(f, seenBefore)
	if err != nil {
		glog.Error(err)
	}
}

func (t *Tailer) startNewFile(f afero.File, seekStart bool) error {
	fi, err := f.Stat()
	if err != nil {
		// Stat failed, log error and return.
		logErrors.Add(f.Name(), 1)
		return fmt.Errorf("Failed to stat %q: %s", f.Name(), err)
	}
	switch m := fi.Mode(); {
	case m&os.ModeType == 0:
		if seekStart {
			f.Seek(0, os.SEEK_SET)
		} else {
			f.Seek(0, os.SEEK_END)
		}
		err = t.w.Add(f.Name())
		if err != nil {
			return fmt.Errorf("Adding a change watch failed on %q: %s", f.Name(), err)
		}
		// In case the new log has been written to already, attempt to read the
		// first lines.
		t.partials[f.Name()], err = t.read(f, "")
		if err != nil {
			if err == io.EOF {
				// Don't worry about EOF on first read, that's expected.
				break
			}
			return err
		}
	case m&os.ModeType == os.ModeNamedPipe:
		go t.readForever(f)
	default:
		return fmt.Errorf("Can't open files with mode %v: %s", m&os.ModeType, f.Name())
	}
	t.filesLock.Lock()
	t.files[f.Name()] = f
	t.filesLock.Unlock()
	glog.Infof("Tailing %s", f.Name())

	return nil
}

// start is the main event loop for the Tailer.
// It receives notification of log file changes from the watcher channel, and
// handles them.
func (t *Tailer) run() {
	for e := range t.w.Events() {
		switch e := e.(type) {
		case watcher.UpdateEvent:
			if t.isWatching(e.Pathname) {
				t.handleLogUpdate(e.Pathname)
			}
		case watcher.CreateEvent:
			if t.isWatching(e.Pathname) {
				t.handleLogCreate(e.Pathname)
			}
		case watcher.DeleteEvent:
		default:
			glog.Infof("Unexpected event %#v", e)
		}
	}
	glog.Infof("Shutting down tailer.")
	close(t.lines)
}

func (t *Tailer) readForever(f afero.File) {
	var err error
	partial := ""
	for {
		partial, err = t.read(f, partial)
		// We want to exit at EOF, because the FD has been closed.
		if err != nil {
			glog.Infof("%s: %s", err, f.Name())
			return
		}
	}
}

// Close signals termination to the watcher.
func (t *Tailer) Close() {
	t.w.Close()
}
<|endoftext|>


<commit_before>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

// Package tailer provides a class that is responsible for tailing log files
// and extracting new log lines to be passed into the virtual machines.
package tailer

// For regular files, mtail gets notified on modifications (i.e. appends) to
// log files that are being watched, in order to read the new lines. Log files
// can also be rotated, so mtail is also notified of creates in the log file
// directory.

import (
	"expvar"
	"html/template"
	"io"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/golang/glog"
	"github.com/pkg/errors"

	"github.com/google/mtail/logline"
	"github.com/google/mtail/watcher"

	"github.com/spf13/afero"
)

var (
	// logCount records the number of logs that are being tailed
	logCount = expvar.NewInt("log_count")
)

// Tailer receives notification of changes from a Watcher and extracts new log
// lines from files. It also handles new log file creation events and log
// rotations.
type Tailer struct {
	lines chan<- *logline.LogLine // Logfile lines being emitted.
	w     watcher.Watcher
	fs    afero.Fs // mockable filesystem interface

	handlesMu sync.RWMutex     // protects `handles'
	handles   map[string]*File // File handles for each pathname.

	globPatternsMu sync.RWMutex        // protects `globPatterns'
	globPatterns   map[string]struct{} // glob patterns to match newly created files in dir paths against

	runDone chan struct{} // Signals termination of the run goroutine.

	eventsHandle int // record the handle with which to add new log files to the watcher

	pollTicker *time.Ticker

	oneShot bool
}

// OneShot puts the tailer in one-shot mode.
func OneShot(t *Tailer) error {
	t.oneShot = true
	return nil
}

// PollInterval sets the time interval between polls of the watched log files.
func PollInterval(interval time.Duration) func(*Tailer) error {
	return func(t *Tailer) error {
		if interval > 0 {
			t.pollTicker = time.NewTicker(interval)
		}
		return nil
	}
}

// New creates a new Tailer.
func New(lines chan<- *logline.LogLine, fs afero.Fs, w watcher.Watcher, options ...func(*Tailer) error) (*Tailer, error) {
	if lines == nil {
		return nil, errors.New("can't create tailer without lines channel")
	}
	if fs == nil {
		return nil, errors.New("can't create tailer without FS")
	}
	if w == nil {
		return nil, errors.New("can't create tailer without W")
	}
	t := &Tailer{
		lines:        lines,
		w:            w,
		fs:           fs,
		handles:      make(map[string]*File),
		globPatterns: make(map[string]struct{}),
		runDone:      make(chan struct{}),
	}
	if err := t.SetOption(options...); err != nil {
		return nil, err
	}
	handle, eventsChan := t.w.Events()
	t.eventsHandle = handle
	go t.run(eventsChan)
	return t, nil
}

// SetOption takes one or more option functions and applies them in order to Tailer.
func (t *Tailer) SetOption(options ...func(*Tailer) error) error {
	for _, option := range options {
		if err := option(t); err != nil {
			return err
		}
	}
	return nil
}

// setHandle sets a file handle under it's pathname
func (t *Tailer) setHandle(pathname string, f *File) error {
	absPath, err := filepath.Abs(pathname)
	if err != nil {
		return errors.Wrapf(err, "Failed to lookup abspath of %q", pathname)
	}
	t.handlesMu.Lock()
	defer t.handlesMu.Unlock()
	t.handles[absPath] = f
	return nil
}

// handleForPath retrives a file handle for a pathname.
func (t *Tailer) handleForPath(pathname string) (*File, bool) {
	absPath, err := filepath.Abs(pathname)
	if err != nil {
		glog.V(2).Infof("Couldn't resolve path %q: %s", pathname, err)
		return nil, false
	}
	t.handlesMu.Lock()
	defer t.handlesMu.Unlock()
	fd, ok := t.handles[absPath]
	return fd, ok
}

func (t *Tailer) hasHandle(pathname string) bool {
	_, ok := t.handleForPath(pathname)
	return ok
}

// AddPattern adds a pattern to the list of patterns to filter filenames against.
func (t *Tailer) AddPattern(pattern string) {
	glog.V(2).Infof("AddPattern: %s", pattern)
	t.globPatternsMu.Lock()
	t.globPatterns[pattern] = struct{}{}
	t.globPatternsMu.Unlock()
}

// TailPattern registers a pattern to be tailed.  If pattern is a plain
// file then it is watched for updates and opened.  If pattern is a glob, then
// all paths that match the glob are opened and watched, and the directories
// containing those matches, if any, are watched.
func (t *Tailer) TailPattern(pattern string) error {
	matches, err := afero.Glob(t.fs, pattern)
	if err != nil {
		return err
	}
	t.AddPattern(pattern)
	glog.V(1).Infof("glob matches: %v", matches)
	// TODO(jaq): Error if there are no matches, or do we just assume that it's OK?
	// mtail_test.go assumes that it's ok.  Figure out why.
	// if len(matches) == 0 {
	// 	return errors.Errorf("No matches for pattern %q", pattern)
	// }
	for _, pathname := range matches {
		err := t.TailPath(pathname)
		if err != nil {
			return errors.Wrapf(err, "attempting to tail %q", pathname)
		}
	}
	// Add a watch on the containing directory, so we know when a rotation
	// occurs or something shows up that matches this pattern.  TODO(jaq): this
	// seems fallible.
	return t.watchDirname(pattern)
}

// TailPath registers a filesystem pathname to be tailed.
func (t *Tailer) TailPath(pathname string) error {
	if t.hasHandle(pathname) {
		glog.V(2).Infof("already watching %q", pathname)
		return nil
	}
	if err := t.w.Add(pathname, t.eventsHandle); err != nil {
		return err
	}
	// New file at start of program, seek to EOF.
	return t.openLogPath(pathname, false)
}

// handleLogEvent is dispatched when an Event is received, causing the tailer
// to read all available bytes from an already-opened file and send each log
// line onto lines channel.  Because we handle rotations and truncates when
// reaching EOF in the file reader itself, we don't care what the signal is
// from the filewatcher.
func (t *Tailer) handleLogEvent(pathname string) {
	glog.V(2).Infof("handleLogUpdate %s", pathname)
	fd, ok := t.handleForPath(pathname)
	if !ok {
		glog.V(1).Infof("No file handle found for %q, but is being watched", pathname)
		// We want to open files we have watches on in case the file was
		// unreadable before now; but we have to copmare against the glob to be
		// sure we don't just add all the files in a watched directory as they
		// get modified.
		t.handleCreateGlob(pathname)
		return
	}
	doFollow(fd)
}

// doFollow performs the Follow on an existing file descriptor, logging any errors
func doFollow(fd *File) {
	err := fd.Follow()
	if err != nil && err != io.EOF {
		glog.Info(err)
	}
}

// pollHandles walks the handles map and polls them all in series.
func (t *Tailer) pollHandles() {
	t.handlesMu.RLock()
	defer t.handlesMu.RUnlock()
	for _, fd := range t.handles {
		doFollow(fd)
	}
}

// watchDirname adds the directory containing a path to be watched.
func (t *Tailer) watchDirname(pathname string) error {
	absPath, err := filepath.Abs(pathname)
	if err != nil {
		return err
	}
	d := filepath.Dir(absPath)
	return t.w.Add(d, t.eventsHandle)
}

// openLogPath opens a log file named by pathname.
func (t *Tailer) openLogPath(pathname string, seekToStart bool) error {
	glog.V(2).Infof("openlogPath %s %v", pathname, seekToStart)
	if err := t.watchDirname(pathname); err != nil {
		return err
	}
	f, err := NewFile(t.fs, pathname, t.lines, seekToStart || t.oneShot)
	if err != nil {
		// Doesn't exist yet. We're watching the directory, so we'll pick it up
		// again on create; return successfully.
		if os.IsNotExist(err) {
			glog.V(1).Infof("pathname %q doesn't exist (yet?)", f.Pathname)
			return nil
		}
		return err
	}
	glog.V(2).Infof("Adding a file watch on %q", f.Pathname)
	if err := t.w.Add(f.Pathname, t.eventsHandle); err != nil {
		return err
	}
	if err := t.setHandle(pathname, f); err != nil {
		return err
	}
	if err := f.Read(); err != nil && err != io.EOF {
		return err
	}
	glog.Infof("Tailing %s", f.Pathname)
	logCount.Add(1)
	return nil
}

// handleCreateGlob matches the pathname against the glob patterns and starts tailing the file.
func (t *Tailer) handleCreateGlob(pathname string) {
	t.globPatternsMu.RLock()
	defer t.globPatternsMu.RUnlock()

	for pattern := range t.globPatterns {
		matched, err := filepath.Match(pattern, pathname)
		if err != nil {
			glog.Warningf("Unexpected bad pattern %q not detected earlier", pattern)
			continue
		}
		if !matched {
			glog.V(2).Infof("No match for %q", pathname)
			continue
		}
		glog.V(1).Infof("New file %q matched existing glob %q", pathname, pattern)
		// If this file was just created, read from the start of the file.
		if err := t.openLogPath(pathname, true); err != nil {
			glog.Infof("Failed to tail new file %q: %s", pathname, err)
		}
		glog.V(2).Infof("Started tailing %q", pathname)
	}
}

// run the main event loop for the Tailer.  It receives notification of
// log file changes from the watcher channel, and dispatches the log event
// handler.
func (t *Tailer) run(events <-chan watcher.Event) {
	defer close(t.runDone)
	defer close(t.lines)

	var ticks <-chan time.Time
	if t.pollTicker != nil {
		ticks = t.pollTicker.C
		defer t.pollTicker.Stop()
	}

	for {
		select {
		case e, ok := <-events:
			if !ok {
				glog.Infof("Shutting down tailer.")
				return
			}

			glog.V(2).Infof("Event type %#v", e)
			t.handleLogEvent(e.Pathname)

		case <-ticks:
			t.pollHandles()
		}
	}
}

// Close signals termination to the watcher.
func (t *Tailer) Close() error {
	if err := t.w.Close(); err != nil {
		return err
	}
	<-t.runDone
	return nil
}

const tailerTemplate = `

Log Tailer

Patterns

{{range $name, $val := $.Patterns}}

{{$name}}

{{end}}

Log files watched

pathname
errors
rotations
truncations
lines read

{{range $name, $val := $.Handles}}

{{$name}}

{{index $.Errors $name}}
{{index $.Rotations $name}}
{{index $.Truncs $name}}
{{index $.Lines $name}}

{{end}}

`

// WriteStatusHTML emits the Tailer's state in HTML format to the io.Writer w.
func (t *Tailer) WriteStatusHTML(w io.Writer) error {
	tpl, err := template.New("tailer").Parse(tailerTemplate)
	if err != nil {
		return err
	}
	t.handlesMu.RLock()
	defer t.handlesMu.RUnlock()
	t.globPatternsMu.RLock()
	defer t.globPatternsMu.RUnlock()
	data := struct {
		Handles   map[string]*File
		Patterns  map[string]struct{}
		Rotations map[string]string
		Lines     map[string]string
		Errors    map[string]string
		Truncs    map[string]string
	}{
		t.handles,
		t.globPatterns,
		make(map[string]string),
		make(map[string]string),
		make(map[string]string),
		make(map[string]string),
	}
	for _, pair := range []struct {
		v string
		m map[string]string
	}{
		{"log_errors_total", data.Errors},
		{"log_rotations_total", data.Rotations},
		{"log_lines_total", data.Lines},
		{"log_truncates_total", data.Truncs},
	} {
		expvar.Get(pair.v).(*expvar.Map).Do(func(kv expvar.KeyValue) {
			pair.m[kv.Key] = kv.Value.String()
		})
	}
	return tpl.Execute(w, data)
}
<commit_msg>Safer expvar extraction now the globals are in scope.<commit_after>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

// Package tailer provides a class that is responsible for tailing log files
// and extracting new log lines to be passed into the virtual machines.
package tailer

// For regular files, mtail gets notified on modifications (i.e. appends) to
// log files that are being watched, in order to read the new lines. Log files
// can also be rotated, so mtail is also notified of creates in the log file
// directory.

import (
	"expvar"
	"html/template"
	"io"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/golang/glog"
	"github.com/pkg/errors"

	"github.com/google/mtail/logline"
	"github.com/google/mtail/watcher"

	"github.com/spf13/afero"
)

var (
	// logCount records the number of logs that are being tailed
	logCount = expvar.NewInt("log_count")
)

// Tailer receives notification of changes from a Watcher and extracts new log
// lines from files. It also handles new log file creation events and log
// rotations.
type Tailer struct {
	lines chan<- *logline.LogLine // Logfile lines being emitted.
	w     watcher.Watcher
	fs    afero.Fs // mockable filesystem interface

	handlesMu sync.RWMutex     // protects `handles'
	handles   map[string]*File // File handles for each pathname.

	globPatternsMu sync.RWMutex        // protects `globPatterns'
	globPatterns   map[string]struct{} // glob patterns to match newly created files in dir paths against

	runDone chan struct{} // Signals termination of the run goroutine.

	eventsHandle int // record the handle with which to add new log files to the watcher

	pollTicker *time.Ticker

	oneShot bool
}

// OneShot puts the tailer in one-shot mode.
func OneShot(t *Tailer) error {
	t.oneShot = true
	return nil
}

// PollInterval sets the time interval between polls of the watched log files.
func PollInterval(interval time.Duration) func(*Tailer) error {
	return func(t *Tailer) error {
		if interval > 0 {
			t.pollTicker = time.NewTicker(interval)
		}
		return nil
	}
}

// New creates a new Tailer.
func New(lines chan<- *logline.LogLine, fs afero.Fs, w watcher.Watcher, options ...func(*Tailer) error) (*Tailer, error) {
	if lines == nil {
		return nil, errors.New("can't create tailer without lines channel")
	}
	if fs == nil {
		return nil, errors.New("can't create tailer without FS")
	}
	if w == nil {
		return nil, errors.New("can't create tailer without W")
	}
	t := &Tailer{
		lines:        lines,
		w:            w,
		fs:           fs,
		handles:      make(map[string]*File),
		globPatterns: make(map[string]struct{}),
		runDone:      make(chan struct{}),
	}
	if err := t.SetOption(options...); err != nil {
		return nil, err
	}
	handle, eventsChan := t.w.Events()
	t.eventsHandle = handle
	go t.run(eventsChan)
	return t, nil
}

// SetOption takes one or more option functions and applies them in order to Tailer.
func (t *Tailer) SetOption(options ...func(*Tailer) error) error {
	for _, option := range options {
		if err := option(t); err != nil {
			return err
		}
	}
	return nil
}

// setHandle sets a file handle under it's pathname
func (t *Tailer) setHandle(pathname string, f *File) error {
	absPath, err := filepath.Abs(pathname)
	if err != nil {
		return errors.Wrapf(err, "Failed to lookup abspath of %q", pathname)
	}
	t.handlesMu.Lock()
	defer t.handlesMu.Unlock()
	t.handles[absPath] = f
	return nil
}

// handleForPath retrives a file handle for a pathname.
func (t *Tailer) handleForPath(pathname string) (*File, bool) {
	absPath, err := filepath.Abs(pathname)
	if err != nil {
		glog.V(2).Infof("Couldn't resolve path %q: %s", pathname, err)
		return nil, false
	}
	t.handlesMu.Lock()
	defer t.handlesMu.Unlock()
	fd, ok := t.handles[absPath]
	return fd, ok
}

func (t *Tailer) hasHandle(pathname string) bool {
	_, ok := t.handleForPath(pathname)
	return ok
}

// AddPattern adds a pattern to the list of patterns to filter filenames against.
func (t *Tailer) AddPattern(pattern string) {
	glog.V(2).Infof("AddPattern: %s", pattern)
	t.globPatternsMu.Lock()
	t.globPatterns[pattern] = struct{}{}
	t.globPatternsMu.Unlock()
}

// TailPattern registers a pattern to be tailed.  If pattern is a plain
// file then it is watched for updates and opened.  If pattern is a glob, then
// all paths that match the glob are opened and watched, and the directories
// containing those matches, if any, are watched.
func (t *Tailer) TailPattern(pattern string) error {
	matches, err := afero.Glob(t.fs, pattern)
	if err != nil {
		return err
	}
	t.AddPattern(pattern)
	glog.V(1).Infof("glob matches: %v", matches)
	// TODO(jaq): Error if there are no matches, or do we just assume that it's OK?
	// mtail_test.go assumes that it's ok.  Figure out why.
	// if len(matches) == 0 {
	// 	return errors.Errorf("No matches for pattern %q", pattern)
	// }
	for _, pathname := range matches {
		err := t.TailPath(pathname)
		if err != nil {
			return errors.Wrapf(err, "attempting to tail %q", pathname)
		}
	}
	// Add a watch on the containing directory, so we know when a rotation
	// occurs or something shows up that matches this pattern.  TODO(jaq): this
	// seems fallible.
	return t.watchDirname(pattern)
}

// TailPath registers a filesystem pathname to be tailed.
func (t *Tailer) TailPath(pathname string) error {
	if t.hasHandle(pathname) {
		glog.V(2).Infof("already watching %q", pathname)
		return nil
	}
	if err := t.w.Add(pathname, t.eventsHandle); err != nil {
		return err
	}
	// New file at start of program, seek to EOF.
	return t.openLogPath(pathname, false)
}

// handleLogEvent is dispatched when an Event is received, causing the tailer
// to read all available bytes from an already-opened file and send each log
// line onto lines channel.  Because we handle rotations and truncates when
// reaching EOF in the file reader itself, we don't care what the signal is
// from the filewatcher.
func (t *Tailer) handleLogEvent(pathname string) {
	glog.V(2).Infof("handleLogUpdate %s", pathname)
	fd, ok := t.handleForPath(pathname)
	if !ok {
		glog.V(1).Infof("No file handle found for %q, but is being watched", pathname)
		// We want to open files we have watches on in case the file was
		// unreadable before now; but we have to copmare against the glob to be
		// sure we don't just add all the files in a watched directory as they
		// get modified.
		t.handleCreateGlob(pathname)
		return
	}
	doFollow(fd)
}

// doFollow performs the Follow on an existing file descriptor, logging any errors
func doFollow(fd *File) {
	err := fd.Follow()
	if err != nil && err != io.EOF {
		glog.Info(err)
	}
}

// pollHandles walks the handles map and polls them all in series.
func (t *Tailer) pollHandles() {
	t.handlesMu.RLock()
	defer t.handlesMu.RUnlock()
	for _, fd := range t.handles {
		doFollow(fd)
	}
}

// watchDirname adds the directory containing a path to be watched.
func (t *Tailer) watchDirname(pathname string) error {
	absPath, err := filepath.Abs(pathname)
	if err != nil {
		return err
	}
	d := filepath.Dir(absPath)
	return t.w.Add(d, t.eventsHandle)
}

// openLogPath opens a log file named by pathname.
func (t *Tailer) openLogPath(pathname string, seekToStart bool) error {
	glog.V(2).Infof("openlogPath %s %v", pathname, seekToStart)
	if err := t.watchDirname(pathname); err != nil {
		return err
	}
	f, err := NewFile(t.fs, pathname, t.lines, seekToStart || t.oneShot)
	if err != nil {
		// Doesn't exist yet. We're watching the directory, so we'll pick it up
		// again on create; return successfully.
		if os.IsNotExist(err) {
			glog.V(1).Infof("pathname %q doesn't exist (yet?)", f.Pathname)
			return nil
		}
		return err
	}
	glog.V(2).Infof("Adding a file watch on %q", f.Pathname)
	if err := t.w.Add(f.Pathname, t.eventsHandle); err != nil {
		return err
	}
	if err := t.setHandle(pathname, f); err != nil {
		return err
	}
	if err := f.Read(); err != nil && err != io.EOF {
		return err
	}
	glog.Infof("Tailing %s", f.Pathname)
	logCount.Add(1)
	return nil
}

// handleCreateGlob matches the pathname against the glob patterns and starts tailing the file.
func (t *Tailer) handleCreateGlob(pathname string) {
	t.globPatternsMu.RLock()
	defer t.globPatternsMu.RUnlock()

	for pattern := range t.globPatterns {
		matched, err := filepath.Match(pattern, pathname)
		if err != nil {
			glog.Warningf("Unexpected bad pattern %q not detected earlier", pattern)
			continue
		}
		if !matched {
			glog.V(2).Infof("No match for %q", pathname)
			continue
		}
		glog.V(1).Infof("New file %q matched existing glob %q", pathname, pattern)
		// If this file was just created, read from the start of the file.
		if err := t.openLogPath(pathname, true); err != nil {
			glog.Infof("Failed to tail new file %q: %s", pathname, err)
		}
		glog.V(2).Infof("Started tailing %q", pathname)
	}
}

// run the main event loop for the Tailer.  It receives notification of
// log file changes from the watcher channel, and dispatches the log event
// handler.
func (t *Tailer) run(events <-chan watcher.Event) {
	defer close(t.runDone)
	defer close(t.lines)

	var ticks <-chan time.Time
	if t.pollTicker != nil {
		ticks = t.pollTicker.C
		defer t.pollTicker.Stop()
	}

	for {
		select {
		case e, ok := <-events:
			if !ok {
				glog.Infof("Shutting down tailer.")
				return
			}

			glog.V(2).Infof("Event type %#v", e)
			t.handleLogEvent(e.Pathname)

		case <-ticks:
			t.pollHandles()
		}
	}
}

// Close signals termination to the watcher.
func (t *Tailer) Close() error {
	if err := t.w.Close(); err != nil {
		return err
	}
	<-t.runDone
	return nil
}

const tailerTemplate = `

Log Tailer

Patterns

{{range $name, $val := $.Patterns}}

{{$name}}

{{end}}

Log files watched

pathname
errors
rotations
truncations
lines read

{{range $name, $val := $.Handles}}

{{$name}}

{{index $.Errors $name}}
{{index $.Rotations $name}}
{{index $.Truncs $name}}
{{index $.Lines $name}}

{{end}}

`

// WriteStatusHTML emits the Tailer's state in HTML format to the io.Writer w.
func (t *Tailer) WriteStatusHTML(w io.Writer) error {
	tpl, err := template.New("tailer").Parse(tailerTemplate)
	if err != nil {
		return err
	}
	t.handlesMu.RLock()
	defer t.handlesMu.RUnlock()
	t.globPatternsMu.RLock()
	defer t.globPatternsMu.RUnlock()
	data := struct {
		Handles   map[string]*File
		Patterns  map[string]struct{}
		Rotations map[string]string
		Lines     map[string]string
		Errors    map[string]string
		Truncs    map[string]string
	}{
		t.handles,
		t.globPatterns,
		make(map[string]string),
		make(map[string]string),
		make(map[string]string),
		make(map[string]string),
	}
	for _, pair := range []struct {
		v *expvar.Map
		m map[string]string
	}{
		{logErrors, data.Errors},
		{logRotations, data.Rotations},
		{logTruncs, data.Truncs},
		{lineCount, data.Lines},
	} {
		pair.v.Do(func(kv expvar.KeyValue) {
			pair.m[kv.Key] = kv.Value.String()
		})
	}
	return tpl.Execute(w, data)
}
<|endoftext|>


<commit_before>package tally

import (
	"sync"
	"sync/atomic"
)

// Tally is a counter structure.
type Tally struct {
	c   uint64
	mux sync.Mutex
}

// Increments the counter by one.
// This operation is atomic.
func (t *Tally) Inc() {
	atomic.AddUint64(&t.c, 1)
}

/*
// Decrements the counter by one.
// This operation is atomic.
// Note that decrement will wrap.
func (t *Tally) Dec() {
	atomic.AddUint64(&t.c, ^uint64(0))
}
*/

// Returns the current value of the counter.
// This operation is atomic.
func (t *Tally) Get() uint64 {
	return atomic.LoadUint64(&t.c)
}

// Sets the counter to value.
// This operation is atomic.
func (t *Tally) Set(i uint64) {
	atomic.StoreUint64(&t.c, i)
}

// Sets the counter to 0.
// This operation is atomic.
func (t *Tally) Reset() {
	atomic.StoreUint64(&t.c, 0)
}

// Increments and returns the next counter value.
// This uses locking, to ensure that values are
// safely incremented and returned.
func (t *Tally) SafeNext() uint64 {
	t.mux.Lock()
	t.c++
	i := t.c
	t.mux.Unlock()
	return i
}
<commit_msg>add dec<commit_after>package tally

import (
	"sync"
	"sync/atomic"
)

// Tally is a counter structure.
type Tally struct {
	c   uint64
	mux sync.Mutex
}

// Increments the counter by one.
// This operation is atomic.
func (t *Tally) Inc() {
	atomic.AddUint64(&t.c, 1)
}

// Decrements the counter by one.
// This operation is atomic.
// Note that decrement will wrap.
func (t *Tally) Dec() {
	atomic.AddUint64(&t.c, ^uint64(0))
}

// Returns the current value of the counter.
// This operation is atomic.
func (t *Tally) Get() uint64 {
	return atomic.LoadUint64(&t.c)
}

// Sets the counter to value.
// This operation is atomic.
func (t *Tally) Set(i uint64) {
	atomic.StoreUint64(&t.c, i)
}

// Sets the counter to 0.
// This operation is atomic.
func (t *Tally) Reset() {
	atomic.StoreUint64(&t.c, 0)
}

// Increments and returns the next counter value.
// This uses locking, to ensure that values are
// safely incremented and returned.
func (t *Tally) SafeNext() uint64 {
	t.mux.Lock()
	t.c++
	i := t.c
	t.mux.Unlock()
	return i
}
<|endoftext|>


<commit_before>// Copyright 2015 Matthew Holt and The Caddy Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package rewrite

import (
	"net/http"
	"net/url"
	"strings"

	"github.com/caddyserver/caddy/v2"
	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
)

func init() {
	caddy.RegisterModule(Rewrite{})
}

// Rewrite is a middleware which can rewrite HTTP requests.
type Rewrite struct {
	Method   string `json:"method,omitempty"`
	URI      string `json:"uri,omitempty"`
	Rehandle bool   `json:"rehandle,omitempty"`
}

// CaddyModule returns the Caddy module information.
func (Rewrite) CaddyModule() caddy.ModuleInfo {
	return caddy.ModuleInfo{
		Name: "http.handlers.rewrite",
		New:  func() caddy.Module { return new(Rewrite) },
	}
}

func (rewr Rewrite) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {
	repl := r.Context().Value(caddy.ReplacerCtxKey).(caddy.Replacer)
	var rehandleNeeded bool

	if rewr.Method != "" {
		method := r.Method
		r.Method = strings.ToUpper(repl.ReplaceAll(rewr.Method, ""))
		if r.Method != method {
			rehandleNeeded = true
		}
	}

	if rewr.URI != "" {
		oldURI := r.RequestURI
		newURI := repl.ReplaceAll(rewr.URI, "")

		u, err := url.Parse(newURI)
		if err != nil {
			return caddyhttp.Error(http.StatusInternalServerError, err)
		}

		r.RequestURI = newURI
		r.URL.Path = u.Path
		if u.RawQuery != "" {
			r.URL.RawQuery = u.RawQuery
		}
		if u.Fragment != "" {
			r.URL.Fragment = u.Fragment
		}

		if newURI != oldURI {
			rehandleNeeded = true
		}
	}

	if rehandleNeeded && rewr.Rehandle {
		return caddyhttp.ErrRehandle
	}

	return next.ServeHTTP(w, r)
}

// Interface guard
var _ caddyhttp.MiddlewareHandler = (*Rewrite)(nil)
<commit_msg>rewrite: Options to strip prefix/suffix and issue redirects<commit_after>// Copyright 2015 Matthew Holt and The Caddy Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package rewrite

import (
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"strings"

	"github.com/caddyserver/caddy/v2"
	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
)

func init() {
	caddy.RegisterModule(Rewrite{})
}

// Rewrite is a middleware which can rewrite HTTP requests.
type Rewrite struct {
	Method string `json:"method,omitempty"`
	URI    string `json:"uri,omitempty"`

	StripPathPrefix string `json:"strip_path_prefix,omitempty"`
	StripPathSuffix string `json:"strip_path_suffix,omitempty"`

	HTTPRedirect caddyhttp.WeakString `json:"http_redirect,omitempty"`
	Rehandle     bool                 `json:"rehandle,omitempty"`
}

// CaddyModule returns the Caddy module information.
func (Rewrite) CaddyModule() caddy.ModuleInfo {
	return caddy.ModuleInfo{
		Name: "http.handlers.rewrite",
		New:  func() caddy.Module { return new(Rewrite) },
	}
}

// Validate ensures rewr's configuration is valid.
func (rewr Rewrite) Validate() error {
	if rewr.HTTPRedirect != "" && rewr.Rehandle {
		return fmt.Errorf("cannot be configured to both write a redirect response and rehandle internally")
	}
	return nil
}

func (rewr Rewrite) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {
	repl := r.Context().Value(caddy.ReplacerCtxKey).(caddy.Replacer)
	var changed bool

	// rewrite the method
	if rewr.Method != "" {
		method := r.Method
		r.Method = strings.ToUpper(repl.ReplaceAll(rewr.Method, ""))
		if r.Method != method {
			changed = true
		}
	}

	// rewrite the URI
	if rewr.URI != "" {
		oldURI := r.RequestURI
		newURI := repl.ReplaceAll(rewr.URI, "")

		u, err := url.Parse(newURI)
		if err != nil {
			return caddyhttp.Error(http.StatusInternalServerError, err)
		}

		r.RequestURI = newURI
		r.URL.Path = u.Path
		if u.RawQuery != "" {
			r.URL.RawQuery = u.RawQuery
		}
		if u.Fragment != "" {
			r.URL.Fragment = u.Fragment
		}

		if newURI != oldURI {
			changed = true
		}
	}

	// strip path prefix or suffix
	if rewr.StripPathPrefix != "" {
		prefix := repl.ReplaceAll(rewr.StripPathPrefix, "")
		r.URL.Path = strings.TrimPrefix(r.URL.Path, prefix)
		newURI := r.URL.String()
		if newURI != r.RequestURI {
			changed = true
		}
		r.RequestURI = newURI
	}
	if rewr.StripPathSuffix != "" {
		suffix := repl.ReplaceAll(rewr.StripPathSuffix, "")
		r.URL.Path = strings.TrimSuffix(r.URL.Path, suffix)
		newURI := r.URL.String()
		if newURI != r.RequestURI {
			changed = true
		}
		r.RequestURI = newURI
	}

	if changed && rewr.Rehandle {
		return caddyhttp.ErrRehandle
	}

	if changed && rewr.HTTPRedirect != "" {
		statusCode, err := strconv.Atoi(repl.ReplaceAll(rewr.HTTPRedirect.String(), ""))
		if err != nil {
			return caddyhttp.Error(http.StatusInternalServerError, err)
		}
		w.Header().Set("Location", r.RequestURI)
		w.WriteHeader(statusCode)
		return nil
	}

	return next.ServeHTTP(w, r)
}

// Interface guard
var _ caddyhttp.MiddlewareHandler = (*Rewrite)(nil)
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"github.com/outbrain/orchestrator/app"
	"github.com/outbrain/orchestrator/config"
	"github.com/outbrain/log"
)

func main() {
	configFile := flag.String("config", "", "config file name")
	command := flag.String("c", "", "command (move-up|make-child-of)")
	instance := flag.String("i", "", "instance, host:port")
	sibling := flag.String("s", "", "sibling instance, host:port")
	owner := flag.String("owner", "", "operation owner")
	reason := flag.String("reason", "", "operation reason")
	discovery := flag.Bool("discovery", false, "auto discovery mode")
	verbose := flag.Bool("verbose", false, "verbose")
	debug := flag.Bool("debug", false, "debug mode (very verbose)")
	flag.Parse();

	log.SetLevel(log.ERROR)
	if *verbose {
		log.SetLevel(log.INFO)
	}
	if *debug {
		log.SetLevel(log.DEBUG)
	}

	log.Info("starting")

	if len(*configFile) > 0 {
		config.ForceRead(*configFile)
	} else {
		config.Read("/etc/orchestrator.conf.json", "conf/orchestrator.conf.json", "orchestrator.conf.json")
	}

	switch {
		case len(flag.Args()) == 0 || flag.Arg(0) == "cli":
			app.Cli(*command, *instance, *sibling, *owner, *reason)
		case flag.Arg(0) == "http":
			app.Http(*discovery)
		default:
			log.Error("Usage: orchestrator --options... [cli|http]")
	}
}
<commit_msg>discvery options defaults true<commit_after>package main

import (
	"flag"
	"github.com/outbrain/orchestrator/app"
	"github.com/outbrain/orchestrator/config"
	"github.com/outbrain/log"
)

func main() {
	configFile := flag.String("config", "", "config file name")
	command := flag.String("c", "", "command (move-up|make-child-of)")
	instance := flag.String("i", "", "instance, host:port")
	sibling := flag.String("s", "", "sibling instance, host:port")
	owner := flag.String("owner", "", "operation owner")
	reason := flag.String("reason", "", "operation reason")
	discovery := flag.Bool("discovery", true, "auto discovery mode")
	verbose := flag.Bool("verbose", false, "verbose")
	debug := flag.Bool("debug", false, "debug mode (very verbose)")
	flag.Parse();

	log.SetLevel(log.ERROR)
	if *verbose {
		log.SetLevel(log.INFO)
	}
	if *debug {
		log.SetLevel(log.DEBUG)
	}

	log.Info("starting")

	if len(*configFile) > 0 {
		config.ForceRead(*configFile)
	} else {
		config.Read("/etc/orchestrator.conf.json", "conf/orchestrator.conf.json", "orchestrator.conf.json")
	}

	switch {
		case len(flag.Args()) == 0 || flag.Arg(0) == "cli":
			app.Cli(*command, *instance, *sibling, *owner, *reason)
		case flag.Arg(0) == "http":
			app.Http(*discovery)
		default:
			log.Error("Usage: orchestrator --options... [cli|http]")
	}
}
<|endoftext|>