<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package cron implements handlers for cron jobs.
package cron

import (
	"context"
	"encoding/json"
	"net/http"
	"path"
	"path/filepath"
	"strings"
	"time"

	"cloud.google.com/go/datastore"
	"github.com/pkg/errors"

	"clusterfuzz/go/base/buckets"
	"clusterfuzz/go/base/config"
	"clusterfuzz/go/base/logs"
	"clusterfuzz/go/cloud/db"
	"clusterfuzz/go/cloud/db/types"
	"clusterfuzz/go/cloud/gcs"
)

const (
	dateLayout = "20060102"
)

type latestReportInfo struct {
	FuzzerStatsDir    string `json:"fuzzer_stats_dir"`
	HTMLReportURL     string `json:"html_report_url"`
	ReportDate        string `json:"report_date"`
	ReportSummaryPath string `json:"report_summary_path"`
}

// These structs are needed for parsing summary files exported by llvm-cov.
type reportInfo struct {
	Platform string `json:"platform"`
	Revision int    `json:"revision"`
	Root     string `json:"root"`
	Source   string `json:"source"`
}

type coverageInfo struct {
	Covered int `json:"covered"`
	Total   int `json:"count"`
}

type coverageTotals struct {
	ByFunction coverageInfo `json:"functions"`
	ByLine     coverageInfo `json:"lines"`
	ByRegion   coverageInfo `json:"regions"`
}

type coverageData struct {
	Totals coverageTotals `json:"totals"`
}

type coverageSummary struct {
	Data []coverageData `json:"data"`
}

func latestReportInfoDir(bucket string) string {
	return buckets.BuildURL(gcs.Scheme, bucket, "latest_report_info/")
}

func constructKey(fuzzer, date string) string {
	return fuzzer + "-" + date
}

// coverageInformation reads coverage summary file from GCS and constructs
// CoverageInformation entity out of it.
func coverageInformation(ctx context.Context, info latestReportInfo, summaryPath, name string) (*datastore.Key, *types.CoverageInformation, error) {
	var stats coverageSummary
	err := readJSON(ctx, summaryPath, &stats)
	if err != nil {
		return nil, nil, err
	}

	dateObj, err := time.Parse(dateLayout, info.ReportDate)
	if err != nil {
		logs.Errorf("Failed to parse date %s: %+v", info.ReportDate, err)
		return nil, nil, errors.Wrapf(err, "Incorrect date format: %s.", info.ReportDate)
	}

	var coverageInfo types.CoverageInformation
	key := &datastore.Key{
		Kind: "CoverageInformation",
		Name: constructKey(name, info.ReportDate),
	}

	// Ignore the error as we either get an existing entity or create a new one.
	_ = db.Get(ctx, key, &coverageInfo)
	coverageInfo.Fuzzer = name
	coverageInfo.Date = dateObj
	coverageInfo.FunctionsCovered = stats.Data[0].Totals.ByFunction.Covered
	coverageInfo.FunctionsTotal = stats.Data[0].Totals.ByFunction.Total
	coverageInfo.EdgesCovered = stats.Data[0].Totals.ByRegion.Covered
	coverageInfo.EdgesTotal = stats.Data[0].Totals.ByRegion.Total

	// Link to a per project report as long as we don't have per fuzzer reports.
	coverageInfo.HTMLReportURL = info.HTMLReportURL

	return key, &coverageInfo, nil
}

func basename(url string) string {
	base := path.Base(url)
	return strings.TrimSuffix(base, filepath.Ext(base))
}

func projectQualifiedFuzzerName(fuzzer, project string) string {
	// TODO(crbug.com/879288): use initialize env vars from local config.
	if project == "chromium" {
		return fuzzer
	}

	prefix := project + "_"
	if strings.HasPrefix(fuzzer, prefix) {
		return fuzzer
	}

	return prefix + fuzzer
}

// processFuzzerStats processes individual fuzzer stats file and constructs
// types.CoverageInformation object.
func processFuzzerStats(ctx context.Context, objectInfo buckets.ObjectInfo, info latestReportInfo, project string) (*datastore.Key, *types.CoverageInformation, error) {
	fuzzer := projectQualifiedFuzzerName(basename(objectInfo.Name), project)
	logs.Logf("Processing fuzzer stats for %s (%s)", fuzzer, objectInfo.FullPath())
	return coverageInformation(ctx, info, objectInfo.FullPath(), fuzzer)
}

func processGCSDir(ctx context.Context, url string, cb func(context.Context, buckets.ObjectInfo) error) error {
	it, err := buckets.ListObjects(ctx, url, false)
	if err != nil {
		return errors.Wrapf(err, "Failed to list %s.", url)
	}

	var info buckets.ObjectInfo
	for it.Next(&info) {
		err = cb(ctx, info)
		if err != nil {
			return err
		}
	}

	if err = it.Err(); err != nil {
		return errors.Wrapf(err, "Failed to iterate through %s.", url)
	}

	return nil
}

// processProjectStats processess total stats for a single project.
func processProjectStats(ctx context.Context, info latestReportInfo, project string) (*datastore.Key, *types.CoverageInformation, error) {
	logs.Logf("Processing total stats for %s project (%s)", project, info.ReportSummaryPath)

	// Using project name as a fuzzer_name should not cause any problems, as we
	// use project qualified names for fuzz targets and won't have any collisions.
	return coverageInformation(ctx, info, info.ReportSummaryPath, project)
}

// processProject processess latest report info for a single project.
func processProject(ctx context.Context, objectInfo buckets.ObjectInfo) error {
	project := basename(objectInfo.Name)
	logs.Logf("Processing coverage for %s project.", project)

	var info latestReportInfo
	err := readJSON(ctx, objectInfo.FullPath(), &info)
	if err != nil {
		return err
	}

	// Iterate through info.FuzzerStatsDir and prepare coverage information for
	// invididual fuzz targets.
	var keys []*datastore.Key
	var entities []*types.CoverageInformation
	processGCSDir(ctx, info.FuzzerStatsDir, func(ctx context.Context, objectInfo buckets.ObjectInfo) error {
		key, entity, err := processFuzzerStats(ctx, objectInfo, info, project)
		if err != nil {
			return err
		}

		keys = append(keys, key)
		entities = append(entities, entity)
		return nil
	})

	logs.Logf("Processed coverage for %d targets in %s project.", len(keys), project)

	key, entity, err := processProjectStats(ctx, info, project)
	if err != nil {
		logs.Errorf("Failed to processProjectStats for %s: %+v", project, err)
	} else {
		keys = append(keys, key)
		entities = append(entities, entity)
	}

	_, err = db.PutMulti(ctx, keys, entities)
	return err
}

func readJSON(ctx context.Context, url string, data interface{}) error {
	obj, err := buckets.ReadObject(ctx, url)
	if err != nil {
		return errors.Wrapf(err, "Failed to read %s into json.", url)
	}

	err = json.NewDecoder(obj).Decode(&data)
	if err != nil {
		return errors.Wrapf(err, "Failed to decode %s as json.", url)
	}

	return nil
}

// FuzzerCoverage gets the latest code coverage stats and links to reports.
func FuzzerCoverage(w http.ResponseWriter, r *http.Request) {
	cfg := config.NewProjectConfig()
	bucket := cfg.GetString("coverage.reports.bucket")
	url := latestReportInfoDir(bucket)
	err := processGCSDir(r.Context(), url, processProject)
	if err != nil {
		logs.Errorf("Failed to processProject in %s: %+v", url, err)
	} else {
		logs.Logf("FuzzerCoverage task finished successfully.")
	}
}
<commit_msg>Add an extra log message in the beginning of the FuzzerCoverage cron task. (#1102)<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package cron implements handlers for cron jobs.
package cron

import (
	"context"
	"encoding/json"
	"net/http"
	"path"
	"path/filepath"
	"strings"
	"time"

	"cloud.google.com/go/datastore"
	"github.com/pkg/errors"

	"clusterfuzz/go/base/buckets"
	"clusterfuzz/go/base/config"
	"clusterfuzz/go/base/logs"
	"clusterfuzz/go/cloud/db"
	"clusterfuzz/go/cloud/db/types"
	"clusterfuzz/go/cloud/gcs"
)

const (
	dateLayout = "20060102"
)

type latestReportInfo struct {
	FuzzerStatsDir    string `json:"fuzzer_stats_dir"`
	HTMLReportURL     string `json:"html_report_url"`
	ReportDate        string `json:"report_date"`
	ReportSummaryPath string `json:"report_summary_path"`
}

// These structs are needed for parsing summary files exported by llvm-cov.
type reportInfo struct {
	Platform string `json:"platform"`
	Revision int    `json:"revision"`
	Root     string `json:"root"`
	Source   string `json:"source"`
}

type coverageInfo struct {
	Covered int `json:"covered"`
	Total   int `json:"count"`
}

type coverageTotals struct {
	ByFunction coverageInfo `json:"functions"`
	ByLine     coverageInfo `json:"lines"`
	ByRegion   coverageInfo `json:"regions"`
}

type coverageData struct {
	Totals coverageTotals `json:"totals"`
}

type coverageSummary struct {
	Data []coverageData `json:"data"`
}

func latestReportInfoDir(bucket string) string {
	return buckets.BuildURL(gcs.Scheme, bucket, "latest_report_info/")
}

func constructKey(fuzzer, date string) string {
	return fuzzer + "-" + date
}

// coverageInformation reads coverage summary file from GCS and constructs
// CoverageInformation entity out of it.
func coverageInformation(ctx context.Context, info latestReportInfo, summaryPath, name string) (*datastore.Key, *types.CoverageInformation, error) {
	var stats coverageSummary
	err := readJSON(ctx, summaryPath, &stats)
	if err != nil {
		return nil, nil, err
	}

	dateObj, err := time.Parse(dateLayout, info.ReportDate)
	if err != nil {
		logs.Errorf("Failed to parse date %s: %+v", info.ReportDate, err)
		return nil, nil, errors.Wrapf(err, "Incorrect date format: %s.", info.ReportDate)
	}

	var coverageInfo types.CoverageInformation
	key := &datastore.Key{
		Kind: "CoverageInformation",
		Name: constructKey(name, info.ReportDate),
	}

	// Ignore the error as we either get an existing entity or create a new one.
	_ = db.Get(ctx, key, &coverageInfo)
	coverageInfo.Fuzzer = name
	coverageInfo.Date = dateObj
	coverageInfo.FunctionsCovered = stats.Data[0].Totals.ByFunction.Covered
	coverageInfo.FunctionsTotal = stats.Data[0].Totals.ByFunction.Total
	coverageInfo.EdgesCovered = stats.Data[0].Totals.ByRegion.Covered
	coverageInfo.EdgesTotal = stats.Data[0].Totals.ByRegion.Total

	// Link to a per project report as long as we don't have per fuzzer reports.
	coverageInfo.HTMLReportURL = info.HTMLReportURL

	return key, &coverageInfo, nil
}

func basename(url string) string {
	base := path.Base(url)
	return strings.TrimSuffix(base, filepath.Ext(base))
}

func projectQualifiedFuzzerName(fuzzer, project string) string {
	// TODO(crbug.com/879288): use initialize env vars from local config.
	if project == "chromium" {
		return fuzzer
	}

	prefix := project + "_"
	if strings.HasPrefix(fuzzer, prefix) {
		return fuzzer
	}

	return prefix + fuzzer
}

// processFuzzerStats processes individual fuzzer stats file and constructs
// types.CoverageInformation object.
func processFuzzerStats(ctx context.Context, objectInfo buckets.ObjectInfo, info latestReportInfo, project string) (*datastore.Key, *types.CoverageInformation, error) {
	fuzzer := projectQualifiedFuzzerName(basename(objectInfo.Name), project)
	logs.Logf("Processing fuzzer stats for %s (%s)", fuzzer, objectInfo.FullPath())
	return coverageInformation(ctx, info, objectInfo.FullPath(), fuzzer)
}

func processGCSDir(ctx context.Context, url string, cb func(context.Context, buckets.ObjectInfo) error) error {
	it, err := buckets.ListObjects(ctx, url, false)
	if err != nil {
		return errors.Wrapf(err, "Failed to list %s.", url)
	}

	var info buckets.ObjectInfo
	for it.Next(&info) {
		err = cb(ctx, info)
		if err != nil {
			return err
		}
	}

	if err = it.Err(); err != nil {
		return errors.Wrapf(err, "Failed to iterate through %s.", url)
	}

	return nil
}

// processProjectStats processess total stats for a single project.
func processProjectStats(ctx context.Context, info latestReportInfo, project string) (*datastore.Key, *types.CoverageInformation, error) {
	logs.Logf("Processing total stats for %s project (%s)", project, info.ReportSummaryPath)

	// Using project name as a fuzzer_name should not cause any problems, as we
	// use project qualified names for fuzz targets and won't have any collisions.
	return coverageInformation(ctx, info, info.ReportSummaryPath, project)
}

// processProject processess latest report info for a single project.
func processProject(ctx context.Context, objectInfo buckets.ObjectInfo) error {
	project := basename(objectInfo.Name)
	logs.Logf("Processing coverage for %s project.", project)

	var info latestReportInfo
	err := readJSON(ctx, objectInfo.FullPath(), &info)
	if err != nil {
		return err
	}

	// Iterate through info.FuzzerStatsDir and prepare coverage information for
	// invididual fuzz targets.
	var keys []*datastore.Key
	var entities []*types.CoverageInformation
	processGCSDir(ctx, info.FuzzerStatsDir, func(ctx context.Context, objectInfo buckets.ObjectInfo) error {
		key, entity, err := processFuzzerStats(ctx, objectInfo, info, project)
		if err != nil {
			return err
		}

		keys = append(keys, key)
		entities = append(entities, entity)
		return nil
	})

	logs.Logf("Processed coverage for %d targets in %s project.", len(keys), project)

	key, entity, err := processProjectStats(ctx, info, project)
	if err != nil {
		logs.Errorf("Failed to processProjectStats for %s: %+v", project, err)
	} else {
		keys = append(keys, key)
		entities = append(entities, entity)
	}

	_, err = db.PutMulti(ctx, keys, entities)
	return err
}

func readJSON(ctx context.Context, url string, data interface{}) error {
	obj, err := buckets.ReadObject(ctx, url)
	if err != nil {
		return errors.Wrapf(err, "Failed to read %s into json.", url)
	}

	err = json.NewDecoder(obj).Decode(&data)
	if err != nil {
		return errors.Wrapf(err, "Failed to decode %s as json.", url)
	}

	return nil
}

// FuzzerCoverage gets the latest code coverage stats and links to reports.
func FuzzerCoverage(w http.ResponseWriter, r *http.Request) {
	logs.Logf("FuzzerCoverage task started.")
	cfg := config.NewProjectConfig()
	bucket := cfg.GetString("coverage.reports.bucket")
	url := latestReportInfoDir(bucket)
	err := processGCSDir(r.Context(), url, processProject)
	if err != nil {
		logs.Errorf("Failed to processProject in %s: %+v", url, err)
	} else {
		logs.Logf("FuzzerCoverage task finished successfully.")
	}
}
<|endoftext|>


<commit_before>package gojsonschema

import (
	"fmt"
)

func DebugDisplayJsonSchema(d *JsonSchemaDocument) {
	debugDisplayJsonSchemaRecursive(d.rootSchema, 0)
}

func debugDisplayJsonSchemaRecursive(s *JsonSchema, level int) {
	for i := 0; i != level; i++ {
		fmt.Printf(" ")
	}

	if s.property == nil {
		fmt.Printf("(nil)")
	} else {
		fmt.Printf(*s.property)
	}

	if s.ref != nil {
		fmt.Printf( " | ref %s", s.ref )
	}

	fmt.Printf("\n")

	for i := range s.definitionsChildren {
		debugDisplayJsonSchemaRecursive(s.definitionsChildren[i], level+1)
	}

	for i := range s.propertiesChildren {
		debugDisplayJsonSchemaRecursive(s.propertiesChildren[i], level+1)
	}

	if s.itemsChild != nil {
		debugDisplayJsonSchemaRecursive(s.itemsChild, level+1)
	}

}
<commit_msg>added id in display<commit_after>package gojsonschema

import (
	"fmt"
)

func DebugDisplayJsonSchema(d *JsonSchemaDocument) {
	debugDisplayJsonSchemaRecursive(d.rootSchema, 0)
}

func debugDisplayJsonSchemaRecursive(s *JsonSchema, level int) {
	for i := 0; i != level; i++ {
		fmt.Printf(" ")
	}

	if s.property == nil {
		fmt.Printf("(nil)")
	} else {
		fmt.Printf(*s.property)
	}

	if s.ref != nil {
		fmt.Printf( " | ref %s", s.ref )
	}

	if s.id != nil {
		fmt.Printf( " | id %s", s.id )
	}

	fmt.Printf("\n")

	for i := range s.definitionsChildren {
		debugDisplayJsonSchemaRecursive(s.definitionsChildren[i], level+1)
	}

	for i := range s.propertiesChildren {
		debugDisplayJsonSchemaRecursive(s.propertiesChildren[i], level+1)
	}

	if s.itemsChild != nil {
		debugDisplayJsonSchemaRecursive(s.itemsChild, level+1)
	}

}
<|endoftext|>


<commit_before>/**
 * (C) Copyright 2014, Deft Labs
 */

package deftlabsds

import (
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
	"deftlabs.com/log"
)

type DataSource struct {
	DbName string
	CollectionName string
	Mongo *Mongo
	slogger.Logger
}

// Insert a document into a collection with the base configured write concern.
func (self *DataSource) Insert(doc interface{}) error { return self.Mongo.Collection(self.DbName, self.CollectionName).Insert(doc) }

// Insert a document into a collection with the passed write concern.
func (self *DataSource) InsertSafe(doc interface{}, safeMode *mgo.Safe) error {
	session := self.SessionClone()

	defer session.Close()

	session.SetSafe(safeMode)
	return session.DB(self.DbName).C(self.CollectionName).Insert(doc)
}

// Finds one document or returns nil. If the document is not found an error of type mgo.ErrNotFound is
// returned. The result must be a pointer.
func (self *DataSource) FindOne(query *bson.M, result interface{}) error {
	return self.Collection().Find(query).One(result)
}

// Delete one or more documents from the collection. If the document(s) is/are not found, no error
// is returned.
func (self *DataSource) Delete(selector interface{}) error {
	_, err := self.Collection().RemoveAll(selector)
	return err
}

// Returns the collection from the session.
func (self *DataSource) Collection() *mgo.Collection { return self.Mongo.Collection(self.DbName, self.CollectionName) }

// Returns the database from the session.
func (self *DataSource) Db() *mgo.Database { return self.Mongo.Db(self.DbName) }

// Returns the session struct.
func (self *DataSource) Session() *mgo.Session { return self.Mongo.session }

// Returns a clone of the session struct.
func (self *DataSource) SessionClone() *mgo.Session { return self.Mongo.session.Clone() }

// Returns a copy of the session struct.
func (self *DataSource) SessionCopy() *mgo.Session { return self.Mongo.session.Clone() }

<commit_msg>added index conv methods<commit_after>/**
 * (C) Copyright 2014, Deft Labs
 */

package deftlabsds

import (
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
	"deftlabs.com/log"
)

type DataSource struct {
	DbName string
	CollectionName string
	Mongo *Mongo
	slogger.Logger
}

// Insert a document into a collection with the base configured write concern.
func (self *DataSource) Insert(doc interface{}) error { return self.Mongo.Collection(self.DbName, self.CollectionName).Insert(doc) }

// Insert a document into a collection with the passed write concern.
func (self *DataSource) InsertSafe(doc interface{}, safeMode *mgo.Safe) error {
	session := self.SessionClone()

	defer session.Close()

	session.SetSafe(safeMode)
	return session.DB(self.DbName).C(self.CollectionName).Insert(doc)
}

// Finds one document or returns nil. If the document is not found an error of type mgo.ErrNotFound is
// returned. The result must be a pointer.
func (self *DataSource) FindOne(query *bson.M, result interface{}) error {
	return self.Collection().Find(query).One(result)
}

// Delete one or more documents from the collection. If the document(s) is/are not found, no error
// is returned.
func (self *DataSource) Delete(selector interface{}) error {
	_, err := self.Collection().RemoveAll(selector)
	return err
}

// Ensure a unique, non-sparse index is created. This does not create in the background. This does
// NOT drop duplicates if they exist. Duplicates will cause an error.
func (self *DataSource) EnsureUniqueIndex(fields []string) error {
	return self.Collection().EnsureIndex(mgo.Index{
		Key: fields,
		Unique: true,
		DropDups: true,
		Background: false,
		Sparse: false,
	})
}

// Ensure a non-unique, non-sparse index is created. This does not create in the background.
func (self *DataSource) EnsureIndex(fields []string) error {
	return self.Collection().EnsureIndex(mgo.Index{
		Key: fields,
		Unique: false,
		DropDups: true,
		Background: false,
		Sparse: false,
	})
}

// Ensure a non-unique, sparse index is created. This does not create in the background.
func (self *DataSource) EnsureSparseIndex(fields []string) error {
	return self.Collection().EnsureIndex(mgo.Index{
		Key: fields,
		Unique: false,
		DropDups: true,
		Background: false,
		Sparse: true,
	})
}

// Ensure a unique, sparse index is created. This does not create in the background. This does
// NOT drop duplicates if they exist. Duplicates will cause an error.
func (self *DataSource) EnsureUniqueSparseIndex(fields []string) error {
	return self.Collection().EnsureIndex(mgo.Index{
		Key: fields,
		Unique: true,
		DropDups: false,
		Background: false,
		Sparse: true,
	})
}

// Returns the collection from the session.
func (self *DataSource) Collection() *mgo.Collection { return self.Mongo.Collection(self.DbName, self.CollectionName) }

// Returns the database from the session.
func (self *DataSource) Db() *mgo.Database { return self.Mongo.Db(self.DbName) }

// Returns the session struct.
func (self *DataSource) Session() *mgo.Session { return self.Mongo.session }

// Returns a clone of the session struct.
func (self *DataSource) SessionClone() *mgo.Session { return self.Mongo.session.Clone() }

// Returns a copy of the session struct.
func (self *DataSource) SessionCopy() *mgo.Session { return self.Mongo.session.Clone() }

<|endoftext|>


<commit_before>// vcfanno is a command-line application and an api for annotating intervals (bed or vcf).
package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	_ "net/http/pprof"
	"os"
	"runtime"
	"runtime/pprof"

	"time"

	"github.com/BurntSushi/toml"
	"github.com/brentp/bix"
	"github.com/brentp/irelate"
	"github.com/brentp/irelate/interfaces"
	"github.com/brentp/irelate/parsers"
	. "github.com/brentp/vcfanno/api"
	. "github.com/brentp/vcfanno/shared"
	"github.com/brentp/vcfgo"
	"github.com/brentp/xopen"
)

const VERSION = "0.0.8"

func main() {
	fmt.Fprintf(os.Stderr, `
=============================================
vcfanno version %s [built with %s]

see: https://github.com/brentp/vcfanno
=============================================
`, VERSION, runtime.Version())

	ends := flag.Bool("ends", false, "annotate the start and end as well as the interval itself.")
	notstrict := flag.Bool("permissive-overlap", false, "annotate with an overlapping variant even it doesn't"+
		" share the same ref and alt alleles. Default is to require exact match between variants.")
	js := flag.String("js", "", "optional path to a file containing custom javascript functions to be used as ops")
	lexsort := flag.Bool("lexicographical", false, "expect chromosomes in order of 1,10,11 ... 19, 2, 20... "+
		" default is 1, 10, 11, ..., 19, 2, 20... . All files must be in the same order.")
	region := flag.String("region", "", "optional region (chrom:start-end) to restrict annnotation. Useful for parallelization")
	base := flag.String("base-path", "", "optional base-path to prepend to annotation files in the config")
	flag.Parse()
	inFiles := flag.Args()
	if len(inFiles) != 2 {
		fmt.Printf(`Usage:
%s config.toml intput.vcf > annotated.vcf

To run a server:

%s server

`, os.Args[0], os.Args[0])
		flag.PrintDefaults()
		return
	}
	queryFile := inFiles[1]
	if !(xopen.Exists(queryFile) || queryFile == "-") {
		fmt.Fprintf(os.Stderr, "\nERROR: can't find query file: %s\n", queryFile)
		os.Exit(2)
	}

	var config Config
	if _, err := toml.DecodeFile(inFiles[0], &config); err != nil {
		panic(err)
	}
	config.Base = *base
	for _, a := range config.Annotation {
		err := CheckAnno(&a)
		if err != nil {
			log.Fatal("CheckAnno err:", err)
		}
	}
	sources, e := config.Sources()
	if e != nil {
		log.Fatal(e)
	}

	log.Printf("found %d sources from %d files\n", len(sources), len(config.Annotation))
	go func() {
		log.Println(http.ListenAndServe("localhost:6060", nil))
	}()

	jsString := ReadJs(*js)
	strict := !*notstrict
	var a = NewAnnotator(sources, jsString, *ends, strict, !*lexsort, *region)

	var out io.Writer = os.Stdout
	defer os.Stdout.Close()

	var rdr *vcfgo.Reader
	var err error
	var q io.Reader

	if *region == "" {
		q, err = xopen.Ropen(queryFile)
		if err != nil {
			log.Fatal(err)
		}
	} else {
		bx, err := bix.New(queryFile)
		if err != nil {
			log.Fatal(err)
		}
		chrom, start, end, err := irelate.RegionToParts(*region)
		if err != nil {
			log.Fatal(err)
		}
		q, err = bx.Query(chrom, start, end, true)
		if err != nil {
			log.Fatal(err)
		}
	}

	qs, rdr, err := parsers.VCFIterator(q)
	if err != nil {
		log.Fatal(err)
	}

	a.UpdateHeader(rdr)

	files, err := a.SetupStreams()
	if err != nil {
		log.Fatal(err)
	}

	fn := func(v interfaces.Relatable) {
		a.AnnotateOne(v, a.Strict)
	}

	stream := irelate.PIRelate(5000, 40000, qs, fn, files...)

	out, err = vcfgo.NewWriter(out, rdr.Header)
	if err != nil {
		log.Fatal(err)
	}

	start := time.Now()
	n := 0

	if os.Getenv("IRELATE_PROFILE") == "TRUE" {
		log.Println("profiling to: irelate.pprof")
		f, err := os.Create("irelate.pprof")
		if err != nil {
			panic(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	//for interval := range a.Annotate(stream) {
	for interval := range stream {
		fmt.Fprintf(out, "%s\n", interval)
		_ = interval
		n++
	}
	printTime(start, n)
	if rdr != nil {
		if e := rdr.Error(); e != nil {
			log.Println(e)
		}
	}

}

func printTime(start time.Time, n int) {
	dur := time.Since(start)
	duri, duru := dur.Seconds(), "second"
	if duri > float64(600) {
		duri, duru = dur.Minutes(), "minute"
	}
	log.Printf("annotated %d variants in %.2f %ss (%.1f / %s)", n, duri, duru, float64(n)/duri, duru)
}
<commit_msg>cleanup<commit_after>// vcfanno is a command-line application and an api for annotating intervals (bed or vcf).
package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	_ "net/http/pprof"
	"os"
	"runtime"
	"runtime/pprof"

	"time"

	"github.com/BurntSushi/toml"
	"github.com/brentp/bix"
	"github.com/brentp/irelate"
	"github.com/brentp/irelate/interfaces"
	"github.com/brentp/irelate/parsers"
	. "github.com/brentp/vcfanno/api"
	. "github.com/brentp/vcfanno/shared"
	"github.com/brentp/vcfgo"
	"github.com/brentp/xopen"
)

const VERSION = "0.0.8"

func main() {
	fmt.Fprintf(os.Stderr, `
=============================================
vcfanno version %s [built with %s]

see: https://github.com/brentp/vcfanno
=============================================
`, VERSION, runtime.Version())

	ends := flag.Bool("ends", false, "annotate the start and end as well as the interval itself.")
	notstrict := flag.Bool("permissive-overlap", false, "annotate with an overlapping variant even it doesn't"+
		" share the same ref and alt alleles. Default is to require exact match between variants.")
	js := flag.String("js", "", "optional path to a file containing custom javascript functions to be used as ops")
	lexsort := flag.Bool("lexicographical", false, "expect chromosomes in order of 1,10,11 ... 19, 2, 20... "+
		" default is 1, 10, 11, ..., 19, 2, 20... . All files must be in the same order.")
	region := flag.String("region", "", "optional region (chrom:start-end) to restrict annnotation. Useful for parallelization")
	base := flag.String("base-path", "", "optional base-path to prepend to annotation files in the config")
	flag.Parse()
	inFiles := flag.Args()
	if len(inFiles) != 2 {
		fmt.Printf(`Usage:
%s config.toml intput.vcf > annotated.vcf

To run a server:

%s server

`, os.Args[0], os.Args[0])
		flag.PrintDefaults()
		return
	}
	queryFile := inFiles[1]
	if !(xopen.Exists(queryFile) || queryFile == "-") {
		fmt.Fprintf(os.Stderr, "\nERROR: can't find query file: %s\n", queryFile)
		os.Exit(2)
	}

	var config Config
	if _, err := toml.DecodeFile(inFiles[0], &config); err != nil {
		panic(err)
	}
	config.Base = *base
	for _, a := range config.Annotation {
		err := CheckAnno(&a)
		if err != nil {
			log.Fatal("CheckAnno err:", err)
		}
	}
	sources, e := config.Sources()
	if e != nil {
		log.Fatal(e)
	}

	log.Printf("found %d sources from %d files\n", len(sources), len(config.Annotation))
	go func() {
		log.Println(http.ListenAndServe("localhost:6060", nil))
	}()

	jsString := ReadJs(*js)
	strict := !*notstrict
	var a = NewAnnotator(sources, jsString, *ends, strict, !*lexsort, *region)

	var out io.Writer = os.Stdout
	defer os.Stdout.Close()

	var rdr *vcfgo.Reader
	var err error
	var q io.Reader

	if *region == "" {
		q, err = xopen.Ropen(queryFile)
		if err != nil {
			log.Fatal(err)
		}
	} else {
		bx, err := bix.New(queryFile)
		if err != nil {
			log.Fatal(err)
		}
		chrom, start, end, err := irelate.RegionToParts(*region)
		if err != nil {
			log.Fatal(err)
		}
		q, err = bx.Query(chrom, start, end, true)
		if err != nil {
			log.Fatal(err)
		}
	}

	qs, rdr, err := parsers.VCFIterator(q)
	if err != nil {
		log.Fatal(err)
	}

	a.UpdateHeader(rdr)

	files, err := a.SetupStreams()
	if err != nil {
		log.Fatal(err)
	}

	fn := func(v interfaces.Relatable) {
		a.AnnotateOne(v, a.Strict)
	}

	stream := irelate.PIRelate(5000, 60000, qs, fn, files...)

	out, err = vcfgo.NewWriter(out, rdr.Header)
	if err != nil {
		log.Fatal(err)
	}

	start := time.Now()
	n := 0

	if os.Getenv("IRELATE_PROFILE") == "TRUE" {
		log.Println("profiling to: irelate.pprof")
		f, err := os.Create("irelate.pprof")
		if err != nil {
			panic(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	for interval := range stream {
		fmt.Fprintf(out, "%s\n", interval)
		n++
	}
	printTime(start, n)
	if rdr != nil {
		if e := rdr.Error(); e != nil {
			log.Println(e)
		}
	}

}

func printTime(start time.Time, n int) {
	dur := time.Since(start)
	duri, duru := dur.Seconds(), "second"
	if duri > float64(600) {
		duri, duru = dur.Minutes(), "minute"
	}
	log.Printf("annotated %d variants in %.2f %ss (%.1f / %s)", n, duri, duru, float64(n)/duri, duru)
}
<|endoftext|>


<commit_before>package network // import "github.com/docker/docker/integration/network"

import (
	"context"
	"testing"
	"time"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/filters"
	swarmtypes "github.com/docker/docker/api/types/swarm"
	"github.com/docker/docker/client"
	"github.com/docker/docker/integration/internal/network"
	"github.com/docker/docker/integration/internal/swarm"
	"gotest.tools/assert"
	"gotest.tools/poll"
	"gotest.tools/skip"
)

const defaultSwarmPort = 2477

func TestInspectNetwork(t *testing.T) {
	skip.If(t, testEnv.OSType == "windows", "FIXME")
	defer setupTest(t)()
	d := swarm.NewSwarm(t, testEnv)
	defer d.Stop(t)
	client := d.NewClientT(t)
	defer client.Close()

	overlayName := "overlay1"
	overlayID := network.CreateNoError(t, context.Background(), client, overlayName,
		network.WithDriver("overlay"),
		network.WithCheckDuplicate(),
	)

	var instances uint64 = 4
	serviceName := "TestService" + t.Name()

	serviceID := swarm.CreateService(t, d,
		swarm.ServiceWithReplicas(instances),
		swarm.ServiceWithName(serviceName),
		swarm.ServiceWithNetwork(overlayName),
	)

	poll.WaitOn(t, serviceRunningTasksCount(client, serviceID, instances), swarm.ServicePoll)

	_, _, err := client.ServiceInspectWithRaw(context.Background(), serviceID, types.ServiceInspectOptions{})
	assert.NilError(t, err)

	// Test inspect verbose with full NetworkID
	networkVerbose, err := client.NetworkInspect(context.Background(), overlayID, types.NetworkInspectOptions{
		Verbose: true,
	})
	assert.NilError(t, err)
	assert.Assert(t, validNetworkVerbose(networkVerbose, serviceName, instances))

	// Test inspect verbose with partial NetworkID
	networkVerbose, err = client.NetworkInspect(context.Background(), overlayID[0:11], types.NetworkInspectOptions{
		Verbose: true,
	})
	assert.NilError(t, err)
	assert.Assert(t, validNetworkVerbose(networkVerbose, serviceName, instances))

	// Test inspect verbose with Network name and swarm scope
	networkVerbose, err = client.NetworkInspect(context.Background(), overlayName, types.NetworkInspectOptions{
		Verbose: true,
		Scope:   "swarm",
	})
	assert.NilError(t, err)
	assert.Assert(t, validNetworkVerbose(networkVerbose, serviceName, instances))

	err = client.ServiceRemove(context.Background(), serviceID)
	assert.NilError(t, err)

	poll.WaitOn(t, serviceIsRemoved(client, serviceID), swarm.ServicePoll)
	poll.WaitOn(t, noTasks(client), swarm.ServicePoll)

	serviceID2 := swarm.CreateService(t, d,
		swarm.ServiceWithReplicas(instances),
		swarm.ServiceWithName(serviceName),
		swarm.ServiceWithNetwork(overlayName),
	)

	poll.WaitOn(t, serviceRunningTasksCount(client, serviceID2, instances), swarm.ServicePoll)

	err = client.ServiceRemove(context.Background(), serviceID2)
	assert.NilError(t, err)

	poll.WaitOn(t, serviceIsRemoved(client, serviceID2), swarm.ServicePoll)
	poll.WaitOn(t, noTasks(client), swarm.ServicePoll)

	err = client.NetworkRemove(context.Background(), overlayID)
	assert.NilError(t, err)

	poll.WaitOn(t, network.IsRemoved(context.Background(), client, overlayID), poll.WithTimeout(1*time.Minute), poll.WithDelay(10*time.Second))
}

func serviceRunningTasksCount(client client.ServiceAPIClient, serviceID string, instances uint64) func(log poll.LogT) poll.Result {
	return func(log poll.LogT) poll.Result {
		filter := filters.NewArgs()
		filter.Add("service", serviceID)
		tasks, err := client.TaskList(context.Background(), types.TaskListOptions{
			Filters: filter,
		})
		switch {
		case err != nil:
			return poll.Error(err)
		case len(tasks) == int(instances):
			for _, task := range tasks {
				if task.Status.State != swarmtypes.TaskStateRunning {
					return poll.Continue("waiting for tasks to enter run state")
				}
			}
			return poll.Success()
		default:
			return poll.Continue("task count at %d waiting for %d", len(tasks), instances)
		}
	}
}

func serviceIsRemoved(client client.ServiceAPIClient, serviceID string) func(log poll.LogT) poll.Result {
	return func(log poll.LogT) poll.Result {
		filter := filters.NewArgs()
		filter.Add("service", serviceID)
		_, err := client.TaskList(context.Background(), types.TaskListOptions{
			Filters: filter,
		})
		if err == nil {
			return poll.Continue("waiting for service %s to be deleted", serviceID)
		}
		return poll.Success()
	}
}

func noTasks(client client.ServiceAPIClient) func(log poll.LogT) poll.Result {
	return func(log poll.LogT) poll.Result {
		filter := filters.NewArgs()
		tasks, err := client.TaskList(context.Background(), types.TaskListOptions{
			Filters: filter,
		})
		switch {
		case err != nil:
			return poll.Error(err)
		case len(tasks) == 0:
			return poll.Success()
		default:
			return poll.Continue("task count at %d waiting for 0", len(tasks))
		}
	}
}

// Check to see if Service and Tasks info are part of the inspect verbose response
func validNetworkVerbose(network types.NetworkResource, service string, instances uint64) bool {
	if service, ok := network.Services[service]; ok {
		if len(service.Tasks) != int(instances) {
			return false
		}
	}

	if network.IPAM.Config == nil {
		return false
	}

	for _, cfg := range network.IPAM.Config {
		if cfg.Gateway == "" || cfg.Subnet == "" {
			return false
		}
	}
	return true
}
<commit_msg>Refactor TestInspectNetwork<commit_after>package network // import "github.com/docker/docker/integration/network"

import (
	"context"
	"testing"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/filters"
	swarmtypes "github.com/docker/docker/api/types/swarm"
	"github.com/docker/docker/client"
	"github.com/docker/docker/integration/internal/network"
	"github.com/docker/docker/integration/internal/swarm"
	"gotest.tools/assert"
	"gotest.tools/poll"
	"gotest.tools/skip"
)

func TestInspectNetwork(t *testing.T) {
	skip.If(t, testEnv.OSType == "windows", "FIXME")
	defer setupTest(t)()
	d := swarm.NewSwarm(t, testEnv)
	defer d.Stop(t)
	c := d.NewClientT(t)
	defer c.Close()

	networkName := "Overlay" + t.Name()
	overlayID := network.CreateNoError(t, context.Background(), c, networkName,
		network.WithDriver("overlay"),
		network.WithCheckDuplicate(),
	)

	var instances uint64 = 2
	serviceName := "TestService" + t.Name()

	serviceID := swarm.CreateService(t, d,
		swarm.ServiceWithReplicas(instances),
		swarm.ServiceWithName(serviceName),
		swarm.ServiceWithNetwork(networkName),
	)

	poll.WaitOn(t, serviceRunningTasksCount(c, serviceID, instances), swarm.ServicePoll)

	tests := []struct {
		name    string
		network string
		opts    types.NetworkInspectOptions
	}{
		{
			name:    "full network id",
			network: overlayID,
			opts: types.NetworkInspectOptions{
				Verbose: true,
			},
		},
		{
			name:    "partial network id",
			network: overlayID[0:11],
			opts: types.NetworkInspectOptions{
				Verbose: true,
			},
		},
		{
			name:    "network name",
			network: networkName,
			opts: types.NetworkInspectOptions{
				Verbose: true,
			},
		},
		{
			name:    "network name and swarm scope",
			network: networkName,
			opts: types.NetworkInspectOptions{
				Verbose: true,
				Scope:   "swarm",
			},
		},
	}
	ctx := context.Background()
	for _, tc := range tests {
		tc := tc
		t.Run(tc.name, func(t *testing.T) {
			nw, err := c.NetworkInspect(ctx, tc.network, tc.opts)
			assert.NilError(t, err)

			if service, ok := nw.Services[serviceName]; ok {
				assert.Equal(t, len(service.Tasks), int(instances))
			}

			assert.Assert(t, nw.IPAM.Config != nil)

			for _, cfg := range nw.IPAM.Config {
				assert.Assert(t, cfg.Gateway != "")
				assert.Assert(t, cfg.Subnet != "")
			}
		})
	}

	// TODO find out why removing networks is needed; other tests fail if the network is not removed, even though they run on a new daemon.
	err := c.ServiceRemove(ctx, serviceID)
	assert.NilError(t, err)
	poll.WaitOn(t, serviceIsRemoved(c, serviceID), swarm.ServicePoll)
	err = c.NetworkRemove(ctx, overlayID)
	assert.NilError(t, err)
	poll.WaitOn(t, network.IsRemoved(ctx, c, overlayID), swarm.NetworkPoll)
}

func serviceRunningTasksCount(client client.ServiceAPIClient, serviceID string, instances uint64) func(log poll.LogT) poll.Result {
	return func(log poll.LogT) poll.Result {
		tasks, err := client.TaskList(context.Background(), types.TaskListOptions{
			Filters: filters.NewArgs(
				filters.Arg("service", serviceID),
				filters.Arg("desired-state", string(swarmtypes.TaskStateRunning)),
			),
		})
		switch {
		case err != nil:
			return poll.Error(err)
		case len(tasks) == int(instances):
			for _, task := range tasks {
				if task.Status.Err != "" {
					log.Log("task error:", task.Status.Err)
				}
				if task.Status.State != swarmtypes.TaskStateRunning {
					return poll.Continue("waiting for tasks to enter run state (current status: %s)", task.Status.State)
				}
			}
			return poll.Success()
		default:
			return poll.Continue("task count for service %s at %d waiting for %d", serviceID, len(tasks), instances)
		}
	}
}

func serviceIsRemoved(client client.ServiceAPIClient, serviceID string) func(log poll.LogT) poll.Result {
	return func(log poll.LogT) poll.Result {
		filter := filters.NewArgs()
		filter.Add("service", serviceID)
		_, err := client.TaskList(context.Background(), types.TaskListOptions{
			Filters: filter,
		})
		if err == nil {
			return poll.Continue("waiting for service %s to be deleted", serviceID)
		}
		return poll.Success()
	}
}
<|endoftext|>


<commit_before>package network // import "github.com/docker/docker/integration/network"

import (
	"context"
	"testing"
	"time"

	"github.com/docker/docker/api/types"
	swarmtypes "github.com/docker/docker/api/types/swarm"
	"github.com/docker/docker/api/types/versions"
	"github.com/docker/docker/client"
	"github.com/docker/docker/integration/internal/network"
	"github.com/docker/docker/integration/internal/swarm"
	"github.com/docker/docker/internal/test/daemon"
	"gotest.tools/assert"
	"gotest.tools/icmd"
	"gotest.tools/poll"
	"gotest.tools/skip"
)

// delInterface removes given network interface
func delInterface(t *testing.T, ifName string) {
	icmd.RunCommand("ip", "link", "delete", ifName).Assert(t, icmd.Success)
	icmd.RunCommand("iptables", "-t", "nat", "--flush").Assert(t, icmd.Success)
	icmd.RunCommand("iptables", "--flush").Assert(t, icmd.Success)
}

func TestDaemonRestartWithLiveRestore(t *testing.T) {
	skip.If(t, testEnv.IsRemoteDaemon())
	skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.38"), "skip test from new feature")
	d := daemon.New(t)
	defer d.Stop(t)
	d.Start(t)
	d.Restart(t, "--live-restore=true",
		"--default-address-pool", "base=175.30.0.0/16,size=16",
		"--default-address-pool", "base=175.33.0.0/16,size=24")

	// Verify bridge network's subnet
	cli, err := d.NewClient()
	assert.Assert(t, err)
	defer cli.Close()
	out, err := cli.NetworkInspect(context.Background(), "bridge", types.NetworkInspectOptions{})
	assert.NilError(t, err)
	// Make sure docker0 doesn't get override with new IP in live restore case
	assert.Equal(t, out.IPAM.Config[0].Subnet, "172.18.0.0/16")
}

func TestDaemonDefaultNetworkPools(t *testing.T) {
	// Remove docker0 bridge and the start daemon defining the predefined address pools
	skip.If(t, testEnv.IsRemoteDaemon())
	skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.38"), "skip test from new feature")
	defaultNetworkBridge := "docker0"
	delInterface(t, defaultNetworkBridge)
	d := daemon.New(t)
	defer d.Stop(t)
	d.Start(t,
		"--default-address-pool", "base=175.30.0.0/16,size=16",
		"--default-address-pool", "base=175.33.0.0/16,size=24")

	// Verify bridge network's subnet
	cli, err := d.NewClient()
	assert.Assert(t, err)
	defer cli.Close()
	out, err := cli.NetworkInspect(context.Background(), "bridge", types.NetworkInspectOptions{})
	assert.NilError(t, err)
	assert.Equal(t, out.IPAM.Config[0].Subnet, "175.30.0.0/16")

	// Create a bridge network and verify its subnet is the second default pool
	name := "elango"
	network.CreateNoError(t, context.Background(), cli, name,
		network.WithDriver("bridge"),
	)
	out, err = cli.NetworkInspect(context.Background(), name, types.NetworkInspectOptions{})
	assert.NilError(t, err)
	assert.Equal(t, out.IPAM.Config[0].Subnet, "175.33.0.0/24")

	// Create a bridge network and verify its subnet is the third default pool
	name = "saanvi"
	network.CreateNoError(t, context.Background(), cli, name,
		network.WithDriver("bridge"),
	)
	out, err = cli.NetworkInspect(context.Background(), name, types.NetworkInspectOptions{})
	assert.NilError(t, err)
	assert.Equal(t, out.IPAM.Config[0].Subnet, "175.33.1.0/24")
	delInterface(t, defaultNetworkBridge)

}

func TestDaemonRestartWithExistingNetwork(t *testing.T) {
	skip.If(t, testEnv.IsRemoteDaemon())
	skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.38"), "skip test from new feature")
	defaultNetworkBridge := "docker0"
	d := daemon.New(t)
	d.Start(t)
	defer d.Stop(t)
	// Verify bridge network's subnet
	cli, err := d.NewClient()
	assert.Assert(t, err)
	defer cli.Close()

	// Create a bridge network
	name := "elango"
	network.CreateNoError(t, context.Background(), cli, name,
		network.WithDriver("bridge"),
	)
	out, err := cli.NetworkInspect(context.Background(), name, types.NetworkInspectOptions{})
	assert.NilError(t, err)
	networkip := out.IPAM.Config[0].Subnet

	// Restart daemon with default address pool option
	d.Restart(t,
		"--default-address-pool", "base=175.30.0.0/16,size=16",
		"--default-address-pool", "base=175.33.0.0/16,size=24")

	out1, err := cli.NetworkInspect(context.Background(), name, types.NetworkInspectOptions{})
	assert.NilError(t, err)
	assert.Equal(t, out1.IPAM.Config[0].Subnet, networkip)
	delInterface(t, defaultNetworkBridge)
}

func TestDaemonRestartWithExistingNetworkWithDefaultPoolRange(t *testing.T) {
	skip.If(t, testEnv.IsRemoteDaemon())
	skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.38"), "skip test from new feature")
	defaultNetworkBridge := "docker0"
	d := daemon.New(t)
	d.Start(t)
	defer d.Stop(t)
	// Verify bridge network's subnet
	cli, err := d.NewClient()
	assert.Assert(t, err)
	defer cli.Close()

	// Create a bridge network
	name := "elango"
	network.CreateNoError(t, context.Background(), cli, name,
		network.WithDriver("bridge"),
	)
	out, err := cli.NetworkInspect(context.Background(), name, types.NetworkInspectOptions{})
	assert.NilError(t, err)
	networkip := out.IPAM.Config[0].Subnet

	// Create a bridge network
	name = "sthira"
	network.CreateNoError(t, context.Background(), cli, name,
		network.WithDriver("bridge"),
	)
	out, err = cli.NetworkInspect(context.Background(), name, types.NetworkInspectOptions{})
	assert.NilError(t, err)
	networkip2 := out.IPAM.Config[0].Subnet

	// Restart daemon with default address pool option
	d.Restart(t,
		"--default-address-pool", "base=175.18.0.0/16,size=16",
		"--default-address-pool", "base=175.19.0.0/16,size=24")

	// Create a bridge network
	name = "saanvi"
	network.CreateNoError(t, context.Background(), cli, name,
		network.WithDriver("bridge"),
	)
	out1, err := cli.NetworkInspect(context.Background(), name, types.NetworkInspectOptions{})
	assert.NilError(t, err)

	assert.Check(t, out1.IPAM.Config[0].Subnet != networkip)
	assert.Check(t, out1.IPAM.Config[0].Subnet != networkip2)
	delInterface(t, defaultNetworkBridge)
}

func TestDaemonWithBipAndDefaultNetworkPool(t *testing.T) {
	skip.If(t, testEnv.IsRemoteDaemon())
	skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.38"), "skip test from new feature")
	defaultNetworkBridge := "docker0"
	d := daemon.New(t)
	defer d.Stop(t)
	d.Start(t, "--bip=172.60.0.1/16",
		"--default-address-pool", "base=175.30.0.0/16,size=16",
		"--default-address-pool", "base=175.33.0.0/16,size=24")

	// Verify bridge network's subnet
	cli, err := d.NewClient()
	assert.Assert(t, err)
	defer cli.Close()
	out, err := cli.NetworkInspect(context.Background(), "bridge", types.NetworkInspectOptions{})
	assert.NilError(t, err)
	// Make sure BIP IP doesn't get override with new default address pool .
	assert.Equal(t, out.IPAM.Config[0].Subnet, "172.60.0.1/16")
	delInterface(t, defaultNetworkBridge)
}

func TestServiceWithPredefinedNetwork(t *testing.T) {
	defer setupTest(t)()
	d := swarm.NewSwarm(t, testEnv)
	defer d.Stop(t)
	client := d.NewClientT(t)
	defer client.Close()

	hostName := "host"
	var instances uint64 = 1
	serviceName := "TestService" + t.Name()

	serviceID := swarm.CreateService(t, d,
		swarm.ServiceWithReplicas(instances),
		swarm.ServiceWithName(serviceName),
		swarm.ServiceWithNetwork(hostName),
	)

	poll.WaitOn(t, serviceRunningCount(client, serviceID, instances), swarm.ServicePoll)

	_, _, err := client.ServiceInspectWithRaw(context.Background(), serviceID, types.ServiceInspectOptions{})
	assert.NilError(t, err)

	err = client.ServiceRemove(context.Background(), serviceID)
	assert.NilError(t, err)
}

const ingressNet = "ingress"

func TestServiceRemoveKeepsIngressNetwork(t *testing.T) {
	defer setupTest(t)()
	d := swarm.NewSwarm(t, testEnv)
	defer d.Stop(t)
	client := d.NewClientT(t)
	defer client.Close()

	poll.WaitOn(t, swarmIngressReady(client), swarm.NetworkPoll)

	var instances uint64 = 1

	serviceID := swarm.CreateService(t, d,
		swarm.ServiceWithReplicas(instances),
		swarm.ServiceWithName(t.Name()+"-service"),
		swarm.ServiceWithEndpoint(&swarmtypes.EndpointSpec{
			Ports: []swarmtypes.PortConfig{
				{
					Protocol:    swarmtypes.PortConfigProtocolTCP,
					TargetPort:  80,
					PublishMode: swarmtypes.PortConfigPublishModeIngress,
				},
			},
		}),
	)

	poll.WaitOn(t, serviceRunningCount(client, serviceID, instances), swarm.ServicePoll)

	_, _, err := client.ServiceInspectWithRaw(context.Background(), serviceID, types.ServiceInspectOptions{})
	assert.NilError(t, err)

	err = client.ServiceRemove(context.Background(), serviceID)
	assert.NilError(t, err)

	poll.WaitOn(t, serviceIsRemoved(client, serviceID), swarm.ServicePoll)
	poll.WaitOn(t, noServices(client), swarm.ServicePoll)

	// Ensure that "ingress" is not removed or corrupted
	time.Sleep(10 * time.Second)
	netInfo, err := client.NetworkInspect(context.Background(), ingressNet, types.NetworkInspectOptions{
		Verbose: true,
		Scope:   "swarm",
	})
	assert.NilError(t, err, "Ingress network was removed after removing service!")
	assert.Assert(t, len(netInfo.Containers) != 0, "No load balancing endpoints in ingress network")
	assert.Assert(t, len(netInfo.Peers) != 0, "No peers (including self) in ingress network")
	_, ok := netInfo.Containers["ingress-sbox"]
	assert.Assert(t, ok, "ingress-sbox not present in ingress network")
}

func serviceRunningCount(client client.ServiceAPIClient, serviceID string, instances uint64) func(log poll.LogT) poll.Result {
	return func(log poll.LogT) poll.Result {
		services, err := client.ServiceList(context.Background(), types.ServiceListOptions{})
		if err != nil {
			return poll.Error(err)
		}

		if len(services) != int(instances) {
			return poll.Continue("Service count at %d waiting for %d", len(services), instances)
		}
		return poll.Success()
	}
}

func swarmIngressReady(client client.NetworkAPIClient) func(log poll.LogT) poll.Result {
	return func(log poll.LogT) poll.Result {
		netInfo, err := client.NetworkInspect(context.Background(), ingressNet, types.NetworkInspectOptions{
			Verbose: true,
			Scope:   "swarm",
		})
		if err != nil {
			return poll.Error(err)
		}
		np := len(netInfo.Peers)
		nc := len(netInfo.Containers)
		if np == 0 || nc == 0 {
			return poll.Continue("ingress not ready: %d peers and %d containers", nc, np)
		}
		_, ok := netInfo.Containers["ingress-sbox"]
		if !ok {
			return poll.Continue("ingress not ready: does not contain the ingress-sbox")
		}
		return poll.Success()
	}
}

func noServices(client client.ServiceAPIClient) func(log poll.LogT) poll.Result {
	return func(log poll.LogT) poll.Result {
		services, err := client.ServiceList(context.Background(), types.ServiceListOptions{})
		switch {
		case err != nil:
			return poll.Error(err)
		case len(services) == 0:
			return poll.Success()
		default:
			return poll.Continue("Service count at %d waiting for 0", len(services))
		}
	}
}
<commit_msg>add unique names to integration/network/service_test.go<commit_after>package network // import "github.com/docker/docker/integration/network"

import (
	"context"
	"testing"
	"time"

	"github.com/docker/docker/api/types"
	swarmtypes "github.com/docker/docker/api/types/swarm"
	"github.com/docker/docker/api/types/versions"
	"github.com/docker/docker/client"
	"github.com/docker/docker/integration/internal/network"
	"github.com/docker/docker/integration/internal/swarm"
	"github.com/docker/docker/internal/test/daemon"
	"gotest.tools/assert"
	"gotest.tools/icmd"
	"gotest.tools/poll"
	"gotest.tools/skip"
)

// delInterface removes given network interface
func delInterface(t *testing.T, ifName string) {
	icmd.RunCommand("ip", "link", "delete", ifName).Assert(t, icmd.Success)
	icmd.RunCommand("iptables", "-t", "nat", "--flush").Assert(t, icmd.Success)
	icmd.RunCommand("iptables", "--flush").Assert(t, icmd.Success)
}

func TestDaemonRestartWithLiveRestore(t *testing.T) {
	skip.If(t, testEnv.IsRemoteDaemon())
	skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.38"), "skip test from new feature")
	d := daemon.New(t)
	defer d.Stop(t)
	d.Start(t)
	d.Restart(t, "--live-restore=true",
		"--default-address-pool", "base=175.30.0.0/16,size=16",
		"--default-address-pool", "base=175.33.0.0/16,size=24")

	// Verify bridge network's subnet
	cli, err := d.NewClient()
	assert.Assert(t, err)
	defer cli.Close()
	out, err := cli.NetworkInspect(context.Background(), "bridge", types.NetworkInspectOptions{})
	assert.NilError(t, err)
	// Make sure docker0 doesn't get override with new IP in live restore case
	assert.Equal(t, out.IPAM.Config[0].Subnet, "172.18.0.0/16")
}

func TestDaemonDefaultNetworkPools(t *testing.T) {
	// Remove docker0 bridge and the start daemon defining the predefined address pools
	skip.If(t, testEnv.IsRemoteDaemon())
	skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.38"), "skip test from new feature")
	defaultNetworkBridge := "docker0"
	delInterface(t, defaultNetworkBridge)
	d := daemon.New(t)
	defer d.Stop(t)
	d.Start(t,
		"--default-address-pool", "base=175.30.0.0/16,size=16",
		"--default-address-pool", "base=175.33.0.0/16,size=24")

	// Verify bridge network's subnet
	cli, err := d.NewClient()
	assert.Assert(t, err)
	defer cli.Close()
	out, err := cli.NetworkInspect(context.Background(), "bridge", types.NetworkInspectOptions{})
	assert.NilError(t, err)
	assert.Equal(t, out.IPAM.Config[0].Subnet, "175.30.0.0/16")

	// Create a bridge network and verify its subnet is the second default pool
	name := "elango" + t.Name()
	network.CreateNoError(t, context.Background(), cli, name,
		network.WithDriver("bridge"),
	)
	out, err = cli.NetworkInspect(context.Background(), name, types.NetworkInspectOptions{})
	assert.NilError(t, err)
	assert.Equal(t, out.IPAM.Config[0].Subnet, "175.33.0.0/24")

	// Create a bridge network and verify its subnet is the third default pool
	name = "saanvi" + t.Name()
	network.CreateNoError(t, context.Background(), cli, name,
		network.WithDriver("bridge"),
	)
	out, err = cli.NetworkInspect(context.Background(), name, types.NetworkInspectOptions{})
	assert.NilError(t, err)
	assert.Equal(t, out.IPAM.Config[0].Subnet, "175.33.1.0/24")
	delInterface(t, defaultNetworkBridge)

}

func TestDaemonRestartWithExistingNetwork(t *testing.T) {
	skip.If(t, testEnv.IsRemoteDaemon())
	skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.38"), "skip test from new feature")
	defaultNetworkBridge := "docker0"
	d := daemon.New(t)
	d.Start(t)
	defer d.Stop(t)
	// Verify bridge network's subnet
	cli, err := d.NewClient()
	assert.Assert(t, err)
	defer cli.Close()

	// Create a bridge network
	name := "elango" + t.Name()
	network.CreateNoError(t, context.Background(), cli, name,
		network.WithDriver("bridge"),
	)
	out, err := cli.NetworkInspect(context.Background(), name, types.NetworkInspectOptions{})
	assert.NilError(t, err)
	networkip := out.IPAM.Config[0].Subnet

	// Restart daemon with default address pool option
	d.Restart(t,
		"--default-address-pool", "base=175.30.0.0/16,size=16",
		"--default-address-pool", "base=175.33.0.0/16,size=24")

	out1, err := cli.NetworkInspect(context.Background(), name, types.NetworkInspectOptions{})
	assert.NilError(t, err)
	assert.Equal(t, out1.IPAM.Config[0].Subnet, networkip)
	delInterface(t, defaultNetworkBridge)
}

func TestDaemonRestartWithExistingNetworkWithDefaultPoolRange(t *testing.T) {
	skip.If(t, testEnv.IsRemoteDaemon())
	skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.38"), "skip test from new feature")
	defaultNetworkBridge := "docker0"
	d := daemon.New(t)
	d.Start(t)
	defer d.Stop(t)
	// Verify bridge network's subnet
	cli, err := d.NewClient()
	assert.Assert(t, err)
	defer cli.Close()

	// Create a bridge network
	name := "elango" + t.Name()
	network.CreateNoError(t, context.Background(), cli, name,
		network.WithDriver("bridge"),
	)
	out, err := cli.NetworkInspect(context.Background(), name, types.NetworkInspectOptions{})
	assert.NilError(t, err)
	networkip := out.IPAM.Config[0].Subnet

	// Create a bridge network
	name = "sthira" + t.Name()
	network.CreateNoError(t, context.Background(), cli, name,
		network.WithDriver("bridge"),
	)
	out, err = cli.NetworkInspect(context.Background(), name, types.NetworkInspectOptions{})
	assert.NilError(t, err)
	networkip2 := out.IPAM.Config[0].Subnet

	// Restart daemon with default address pool option
	d.Restart(t,
		"--default-address-pool", "base=175.18.0.0/16,size=16",
		"--default-address-pool", "base=175.19.0.0/16,size=24")

	// Create a bridge network
	name = "saanvi" + t.Name()
	network.CreateNoError(t, context.Background(), cli, name,
		network.WithDriver("bridge"),
	)
	out1, err := cli.NetworkInspect(context.Background(), name, types.NetworkInspectOptions{})
	assert.NilError(t, err)

	assert.Check(t, out1.IPAM.Config[0].Subnet != networkip)
	assert.Check(t, out1.IPAM.Config[0].Subnet != networkip2)
	delInterface(t, defaultNetworkBridge)
}

func TestDaemonWithBipAndDefaultNetworkPool(t *testing.T) {
	skip.If(t, testEnv.IsRemoteDaemon())
	skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.38"), "skip test from new feature")
	defaultNetworkBridge := "docker0"
	d := daemon.New(t)
	defer d.Stop(t)
	d.Start(t, "--bip=172.60.0.1/16",
		"--default-address-pool", "base=175.30.0.0/16,size=16",
		"--default-address-pool", "base=175.33.0.0/16,size=24")

	// Verify bridge network's subnet
	cli, err := d.NewClient()
	assert.Assert(t, err)
	defer cli.Close()
	out, err := cli.NetworkInspect(context.Background(), "bridge", types.NetworkInspectOptions{})
	assert.NilError(t, err)
	// Make sure BIP IP doesn't get override with new default address pool .
	assert.Equal(t, out.IPAM.Config[0].Subnet, "172.60.0.1/16")
	delInterface(t, defaultNetworkBridge)
}

func TestServiceWithPredefinedNetwork(t *testing.T) {
	defer setupTest(t)()
	d := swarm.NewSwarm(t, testEnv)
	defer d.Stop(t)
	client := d.NewClientT(t)
	defer client.Close()

	hostName := "host"
	var instances uint64 = 1
	serviceName := "TestService" + t.Name()

	serviceID := swarm.CreateService(t, d,
		swarm.ServiceWithReplicas(instances),
		swarm.ServiceWithName(serviceName),
		swarm.ServiceWithNetwork(hostName),
	)

	poll.WaitOn(t, serviceRunningCount(client, serviceID, instances), swarm.ServicePoll)

	_, _, err := client.ServiceInspectWithRaw(context.Background(), serviceID, types.ServiceInspectOptions{})
	assert.NilError(t, err)

	err = client.ServiceRemove(context.Background(), serviceID)
	assert.NilError(t, err)
}

const ingressNet = "ingress"

func TestServiceRemoveKeepsIngressNetwork(t *testing.T) {
	defer setupTest(t)()
	d := swarm.NewSwarm(t, testEnv)
	defer d.Stop(t)
	client := d.NewClientT(t)
	defer client.Close()

	poll.WaitOn(t, swarmIngressReady(client), swarm.NetworkPoll)

	var instances uint64 = 1

	serviceID := swarm.CreateService(t, d,
		swarm.ServiceWithReplicas(instances),
		swarm.ServiceWithName(t.Name()+"-service"),
		swarm.ServiceWithEndpoint(&swarmtypes.EndpointSpec{
			Ports: []swarmtypes.PortConfig{
				{
					Protocol:    swarmtypes.PortConfigProtocolTCP,
					TargetPort:  80,
					PublishMode: swarmtypes.PortConfigPublishModeIngress,
				},
			},
		}),
	)

	poll.WaitOn(t, serviceRunningCount(client, serviceID, instances), swarm.ServicePoll)

	_, _, err := client.ServiceInspectWithRaw(context.Background(), serviceID, types.ServiceInspectOptions{})
	assert.NilError(t, err)

	err = client.ServiceRemove(context.Background(), serviceID)
	assert.NilError(t, err)

	poll.WaitOn(t, serviceIsRemoved(client, serviceID), swarm.ServicePoll)
	poll.WaitOn(t, noServices(client), swarm.ServicePoll)

	// Ensure that "ingress" is not removed or corrupted
	time.Sleep(10 * time.Second)
	netInfo, err := client.NetworkInspect(context.Background(), ingressNet, types.NetworkInspectOptions{
		Verbose: true,
		Scope:   "swarm",
	})
	assert.NilError(t, err, "Ingress network was removed after removing service!")
	assert.Assert(t, len(netInfo.Containers) != 0, "No load balancing endpoints in ingress network")
	assert.Assert(t, len(netInfo.Peers) != 0, "No peers (including self) in ingress network")
	_, ok := netInfo.Containers["ingress-sbox"]
	assert.Assert(t, ok, "ingress-sbox not present in ingress network")
}

func serviceRunningCount(client client.ServiceAPIClient, serviceID string, instances uint64) func(log poll.LogT) poll.Result {
	return func(log poll.LogT) poll.Result {
		services, err := client.ServiceList(context.Background(), types.ServiceListOptions{})
		if err != nil {
			return poll.Error(err)
		}

		if len(services) != int(instances) {
			return poll.Continue("Service count at %d waiting for %d", len(services), instances)
		}
		return poll.Success()
	}
}

func swarmIngressReady(client client.NetworkAPIClient) func(log poll.LogT) poll.Result {
	return func(log poll.LogT) poll.Result {
		netInfo, err := client.NetworkInspect(context.Background(), ingressNet, types.NetworkInspectOptions{
			Verbose: true,
			Scope:   "swarm",
		})
		if err != nil {
			return poll.Error(err)
		}
		np := len(netInfo.Peers)
		nc := len(netInfo.Containers)
		if np == 0 || nc == 0 {
			return poll.Continue("ingress not ready: %d peers and %d containers", nc, np)
		}
		_, ok := netInfo.Containers["ingress-sbox"]
		if !ok {
			return poll.Continue("ingress not ready: does not contain the ingress-sbox")
		}
		return poll.Success()
	}
}

func noServices(client client.ServiceAPIClient) func(log poll.LogT) poll.Result {
	return func(log poll.LogT) poll.Result {
		services, err := client.ServiceList(context.Background(), types.ServiceListOptions{})
		switch {
		case err != nil:
			return poll.Error(err)
		case len(services) == 0:
			return poll.Success()
		default:
			return poll.Continue("Service count at %d waiting for 0", len(services))
		}
	}
}
<|endoftext|>


<commit_before>package tcp_output

import (
	"context"
	"encoding/json"
	"fmt"
	"log"
	"net"
	"os"
	"time"

	"github.com/karimra/gnmic/collector"
	"github.com/karimra/gnmic/outputs"
	"github.com/prometheus/client_golang/prometheus"
	"google.golang.org/protobuf/proto"
)

const (
	defaultRetryTimer = 2 * time.Second
	numWorkers        = 1
)

func init() {
	outputs.Register("tcp", func() outputs.Output {
		return &TCPOutput{
			Cfg: &Config{},
		}
	})
}

type TCPOutput struct {
	Cfg *Config

	cancelFn context.CancelFunc
	buffer   chan []byte
	limiter  *time.Ticker
	logger   *log.Logger
	mo       *collector.MarshalOptions
}

type Config struct {
	Address       string        `mapstructure:"address,omitempty"` // ip:port
	Rate          time.Duration `mapstructure:"rate,omitempty"`
	BufferSize    uint          `mapstructure:"buffer-size,omitempty"`
	Format        string        `mapstructure:"format,omitempty"`
	KeepAlive     time.Duration `mapstructure:"keep-alive,omitempty"`
	RetryInterval time.Duration `mapstructure:"retry-interval,omitempty"`
}

func (t *TCPOutput) Init(ctx context.Context, cfg map[string]interface{}, logger *log.Logger) error {
	err := outputs.DecodeConfig(cfg, t.Cfg)
	if err != nil {
		logger.Printf("tcp output config decode failed: %v", err)
		return err
	}
	_, _, err = net.SplitHostPort(t.Cfg.Address)
	if err != nil {
		logger.Printf("tcp output config validation failed: %v", err)
		return fmt.Errorf("wrong address format: %v", err)
	}
	t.logger = log.New(os.Stderr, "tcp_output ", log.LstdFlags|log.Lmicroseconds)
	if logger != nil {
		t.logger.SetOutput(logger.Writer())
		t.logger.SetFlags(logger.Flags())
	}
	t.buffer = make(chan []byte, t.Cfg.BufferSize)
	if t.Cfg.Rate > 0 {
		t.limiter = time.NewTicker(t.Cfg.Rate)
	}
	if t.Cfg.RetryInterval == 0 {
		t.Cfg.RetryInterval = defaultRetryTimer
	}
	t.mo = &collector.MarshalOptions{Format: t.Cfg.Format}
	go func() {
		<-ctx.Done()
		t.Close()
	}()

	ctx, t.cancelFn = context.WithCancel(ctx)
	for i := 0; i < numWorkers; i++ {
		go t.start(ctx)
	}
	return nil
}
func (t *TCPOutput) Write(ctx context.Context, m proto.Message, meta outputs.Meta) {
	if m == nil {
		return
	}
	select {
	case <-ctx.Done():
		return
	default:
		b, err := t.mo.Marshal(m, meta)
		if err != nil {
			t.logger.Printf("failed marshaling proto msg: %v", err)
			return
		}
		t.buffer <- b
	}
}
func (t *TCPOutput) Close() error {
	t.cancelFn()
	if t.limiter != nil {
		t.limiter.Stop()
	}
	return nil
}
func (t *TCPOutput) Metrics() []prometheus.Collector { return nil }
func (t *TCPOutput) String() string {
	b, err := json.Marshal(t)
	if err != nil {
		return ""
	}
	return string(b)
}
func (t *TCPOutput) start(ctx context.Context) {
START:
	tcpAddr, err := net.ResolveTCPAddr("tcp", t.Cfg.Address)
	if err != nil {
		t.logger.Printf("failed to resolve address: %v", err)
		time.Sleep(t.Cfg.RetryInterval)
		goto START
	}
	conn, err := net.DialTCP("tcp", nil, tcpAddr)
	if err != nil {
		t.logger.Printf("failed to dial TCP: %v", err)
		time.Sleep(t.Cfg.RetryInterval)
		goto START
	}
	defer conn.Close()
	if t.Cfg.KeepAlive > 0 {
		conn.SetKeepAlive(true)
		conn.SetKeepAlivePeriod(t.Cfg.KeepAlive)
	}

	defer t.Close()
	for {
		select {
		case <-ctx.Done():
			return
		case b := <-t.buffer:
			if t.limiter != nil {
				<-t.limiter.C
			}
			_, err = conn.Write(b)
			if err != nil {
				t.logger.Printf("failed sending tcp bytes: %v", err)
				conn.Close()
				time.Sleep(t.Cfg.RetryInterval)
				goto START
			}
		}
	}
}
<commit_msg>add configurable number of workers to tcp output<commit_after>package tcp_output

import (
	"context"
	"encoding/json"
	"fmt"
	"log"
	"net"
	"os"
	"time"

	"github.com/karimra/gnmic/collector"
	"github.com/karimra/gnmic/outputs"
	"github.com/prometheus/client_golang/prometheus"
	"google.golang.org/protobuf/proto"
)

const (
	defaultRetryTimer = 2 * time.Second
	defaultNumWorkers = 1
)

func init() {
	outputs.Register("tcp", func() outputs.Output {
		return &TCPOutput{
			Cfg: &Config{},
		}
	})
}

type TCPOutput struct {
	Cfg *Config

	cancelFn context.CancelFunc
	buffer   chan []byte
	limiter  *time.Ticker
	logger   *log.Logger
	mo       *collector.MarshalOptions
}

type Config struct {
	Address       string        `mapstructure:"address,omitempty"` // ip:port
	Rate          time.Duration `mapstructure:"rate,omitempty"`
	BufferSize    uint          `mapstructure:"buffer-size,omitempty"`
	Format        string        `mapstructure:"format,omitempty"`
	KeepAlive     time.Duration `mapstructure:"keep-alive,omitempty"`
	RetryInterval time.Duration `mapstructure:"retry-interval,omitempty"`
	NumWorkers    int           `mapstructure:"num-workers,omitempty"`
}

func (t *TCPOutput) Init(ctx context.Context, cfg map[string]interface{}, logger *log.Logger) error {
	err := outputs.DecodeConfig(cfg, t.Cfg)
	if err != nil {
		logger.Printf("tcp output config decode failed: %v", err)
		return err
	}
	_, _, err = net.SplitHostPort(t.Cfg.Address)
	if err != nil {
		logger.Printf("tcp output config validation failed: %v", err)
		return fmt.Errorf("wrong address format: %v", err)
	}
	t.logger = log.New(os.Stderr, "tcp_output ", log.LstdFlags|log.Lmicroseconds)
	if logger != nil {
		t.logger.SetOutput(logger.Writer())
		t.logger.SetFlags(logger.Flags())
	}
	t.buffer = make(chan []byte, t.Cfg.BufferSize)
	if t.Cfg.Rate > 0 {
		t.limiter = time.NewTicker(t.Cfg.Rate)
	}
	if t.Cfg.RetryInterval == 0 {
		t.Cfg.RetryInterval = defaultRetryTimer
	}
	if t.Cfg.NumWorkers < 1 {
		t.Cfg.NumWorkers = defaultNumWorkers
	}
	t.mo = &collector.MarshalOptions{Format: t.Cfg.Format}
	go func() {
		<-ctx.Done()
		t.Close()
	}()

	ctx, t.cancelFn = context.WithCancel(ctx)
	for i := 0; i < t.Cfg.NumWorkers; i++ {
		go t.start(ctx, i)
	}
	return nil
}
func (t *TCPOutput) Write(ctx context.Context, m proto.Message, meta outputs.Meta) {
	if m == nil {
		return
	}
	select {
	case <-ctx.Done():
		return
	default:
		b, err := t.mo.Marshal(m, meta)
		if err != nil {
			t.logger.Printf("failed marshaling proto msg: %v", err)
			return
		}
		t.buffer <- b
	}
}
func (t *TCPOutput) Close() error {
	t.cancelFn()
	if t.limiter != nil {
		t.limiter.Stop()
	}
	return nil
}
func (t *TCPOutput) Metrics() []prometheus.Collector { return nil }
func (t *TCPOutput) String() string {
	b, err := json.Marshal(t)
	if err != nil {
		return ""
	}
	return string(b)
}
func (t *TCPOutput) start(ctx context.Context, idx int) {
	workerLogPrefix := fmt.Sprintf("worker-%d", idx)
START:
	tcpAddr, err := net.ResolveTCPAddr("tcp", t.Cfg.Address)
	if err != nil {
		t.logger.Printf("%s failed to resolve address: %v", workerLogPrefix, err)
		time.Sleep(t.Cfg.RetryInterval)
		goto START
	}
	conn, err := net.DialTCP("tcp", nil, tcpAddr)
	if err != nil {
		t.logger.Printf("%s failed to dial TCP: %v", workerLogPrefix, err)
		time.Sleep(t.Cfg.RetryInterval)
		goto START
	}
	defer conn.Close()
	if t.Cfg.KeepAlive > 0 {
		conn.SetKeepAlive(true)
		conn.SetKeepAlivePeriod(t.Cfg.KeepAlive)
	}
	defer t.Close()
	for {
		select {
		case <-ctx.Done():
			return
		case b := <-t.buffer:
			if t.limiter != nil {
				<-t.limiter.C
			}
			_, err = conn.Write(b)
			if err != nil {
				t.logger.Printf("%s failed sending tcp bytes: %v", workerLogPrefix, err)
				conn.Close()
				time.Sleep(t.Cfg.RetryInterval)
				goto START
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

const VERSION = "0.6.0"
<commit_msg>:+1: Bump up the version to 0.6.1-alpha1<commit_after>package main

const VERSION = "0.6.1-alpha1"
<|endoftext|>


<commit_before>package swag

// Version of swag.
const Version = "v1.7.5"
<commit_msg>chore: increment version (#1062)<commit_after>package swag

// Version of swag.
const Version = "v1.7.6"
<|endoftext|>


<commit_before>package main

// The git commit that was compiled. This will be filled in by the compiler.
var GitCommit string

// The main version number that is being run at the moment.
const Version = "0.1.1"

// A pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release
// such as "dev" (in development), "beta", "rc1", etc.
const VersionPrerelease = "dev"
<commit_msg>v0.1.1<commit_after>package main

// The git commit that was compiled. This will be filled in by the compiler.
var GitCommit string

// The main version number that is being run at the moment.
const Version = "0.1.1"

// A pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release
// such as "dev" (in development), "beta", "rc1", etc.
const VersionPrerelease = ""
<|endoftext|>


<commit_before>package mailfull

// Version is a version number.
const Version = "0.0.2"
<commit_msg>Bump version to v0.0.3<commit_after>package mailfull

// Version is a version number.
const Version = "v0.0.3"
<|endoftext|>


<commit_before>/*
 *
 * Copyright 2018 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package grpc

// Version is the current grpc version.
const Version = "1.26.0-dev"
<commit_msg>Change version to 1.27.0-dev (#3263)<commit_after>/*
 *
 * Copyright 2018 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package grpc

// Version is the current grpc version.
const Version = "1.27.0-dev"
<|endoftext|>


<commit_before>package main

// The git commit that was compiled. This will be filled in by the compiler.
var GitCommit string
var GitDescribe string

// The main version number that is being run at the moment.
const Version = "0.4.1"

// A pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release
// such as "dev" (in development), "beta", "rc1", etc.
const VersionPrerelease = "dev"
<commit_msg>Updated the version<commit_after>package main

// The git commit that was compiled. This will be filled in by the compiler.
var GitCommit string
var GitDescribe string

// The main version number that is being run at the moment.
const Version = "0.4.1"

// A pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release
// such as "dev" (in development), "beta", "rc1", etc.
const VersionPrerelease = "rc1"
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
)

const VERSION = "0.3.0"

var cmdVersion = &Command{
	Run:       runVersion,
	UsageLine: "version [OPTIONS]",
	Short:     "Show the easel version information",
	Long: `
Options:
	-h, --help     Print usage
`,
}

func init() {
}

func runVersion(args []string) int {

	if len(args) > 0 {
		fmt.Fprintln(os.Stderr, "Too many arguments given.")
		return 1
	}

	fmt.Printf("easel version %s\n", VERSION)
	return 0
}
<commit_msg>change version number<commit_after>package main

import (
	"fmt"
	"os"
)

const VERSION = "1.0.0"

var cmdVersion = &Command{
	Run:       runVersion,
	UsageLine: "version [OPTIONS]",
	Short:     "Show the easel version information",
	Long: `
Options:
	-h, --help     Print usage
`,
}

func init() {
}

func runVersion(args []string) int {
	if len(args) > 0 {
		fmt.Fprintln(os.Stderr, "Too many arguments given.")
		return 1
	}

	fmt.Printf("easel version %s\n", VERSION)
	return 0
}
<|endoftext|>


<commit_before>// Licensed to The Moov Authors under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. The Moov Authors licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

package ach

// Version Number
const Version = "v1.4.0-rc2"
<commit_msg>release v1.4.0-rc3<commit_after>// Licensed to The Moov Authors under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. The Moov Authors licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

package ach

// Version Number
const Version = "v1.4.0-rc3"
<|endoftext|>


<commit_before>package themekit

import (
	"crypto"
	_ "crypto/md5"
	"encoding/hex"
	"fmt"
	"github.com/inconshreveable/go-update"
	"net/http"
	"strconv"
	"strings"
)

var TKVersion Version = Version{Major: 0, Minor: 3, Patch: 3}
var ThemeKitVersion string = TKVersion.String()

type VersionComparisonResult int

const (
	VersionLessThan    VersionComparisonResult = -1
	VersionEqual                               = 0
	VersionGreaterThan                         = 1
)

func LibraryInfo() []string {
	return []string{
		"ThemeKit - Shopify Theme Utilities",
		ThemeKitVersion,
		"Author: Chris Saunders",
	}
}

type Version struct {
	Major int
	Minor int
	Patch int
}

func (v Version) String() string {
	return fmt.Sprintf("v%d.%d.%d", v.Major, v.Minor, v.Patch)
}

func (v Version) toArray() [3]int {
	return [3]int{v.Major, v.Minor, v.Patch}
}

// I often get confused by comparison, so comparison results are going
// to be the same as what <=> would return in Ruby.
// http://ruby-doc.org/core-1.9.3/Comparable.html
func (v Version) Compare(o Version) VersionComparisonResult {
	vAry := v.toArray()
	oAry := o.toArray()
	for i := 0; i < len(vAry); i++ {
		diff := vAry[i] - oAry[i]
		if diff < 0 {
			return VersionLessThan
		} else if diff > 0 {
			return VersionGreaterThan
		}
	}
	return VersionEqual
}

func ParseVersionString(ver string) Version {
	sanitizedVer := strings.Replace(ver, "v", "", 1)
	expandedVersionString := strings.Split(sanitizedVer, ".")
	major, _ := strconv.Atoi(expandedVersionString[0])
	minor, _ := strconv.Atoi(expandedVersionString[1])
	patch, _ := strconv.Atoi(expandedVersionString[2])
	return Version{Major: major, Minor: minor, Patch: patch}
}

func ApplyUpdate(updateURL, digest string) error {
	checksum, err := hex.DecodeString(digest)
	if err != nil {
		return err
	}

	updateFile, err := http.Get(updateURL)
	if err != nil {
		return err
	}
	defer updateFile.Body.Close()

	err = update.Apply(updateFile.Body, update.Options{
		Hash:     crypto.MD5,
		Checksum: checksum,
	})
	if err != nil {
		if rerr := update.RollbackError(err); rerr != nil {
			fmt.Println("Failed to rollback from bad update: %v", rerr)
		}
	}
	return err
}
<commit_msg>Updated version to 0.3.4<commit_after>package themekit

import (
	"crypto"
	_ "crypto/md5"
	"encoding/hex"
	"fmt"
	"github.com/inconshreveable/go-update"
	"net/http"
	"strconv"
	"strings"
)

var TKVersion Version = Version{Major: 0, Minor: 3, Patch: 4}
var ThemeKitVersion string = TKVersion.String()

type VersionComparisonResult int

const (
	VersionLessThan    VersionComparisonResult = -1
	VersionEqual                               = 0
	VersionGreaterThan                         = 1
)

func LibraryInfo() []string {
	return []string{
		"ThemeKit - Shopify Theme Utilities",
		ThemeKitVersion,
		"Author: Chris Saunders",
	}
}

type Version struct {
	Major int
	Minor int
	Patch int
}

func (v Version) String() string {
	return fmt.Sprintf("v%d.%d.%d", v.Major, v.Minor, v.Patch)
}

func (v Version) toArray() [3]int {
	return [3]int{v.Major, v.Minor, v.Patch}
}

// I often get confused by comparison, so comparison results are going
// to be the same as what <=> would return in Ruby.
// http://ruby-doc.org/core-1.9.3/Comparable.html
func (v Version) Compare(o Version) VersionComparisonResult {
	vAry := v.toArray()
	oAry := o.toArray()
	for i := 0; i < len(vAry); i++ {
		diff := vAry[i] - oAry[i]
		if diff < 0 {
			return VersionLessThan
		} else if diff > 0 {
			return VersionGreaterThan
		}
	}
	return VersionEqual
}

func ParseVersionString(ver string) Version {
	sanitizedVer := strings.Replace(ver, "v", "", 1)
	expandedVersionString := strings.Split(sanitizedVer, ".")
	major, _ := strconv.Atoi(expandedVersionString[0])
	minor, _ := strconv.Atoi(expandedVersionString[1])
	patch, _ := strconv.Atoi(expandedVersionString[2])
	return Version{Major: major, Minor: minor, Patch: patch}
}

func ApplyUpdate(updateURL, digest string) error {
	checksum, err := hex.DecodeString(digest)
	if err != nil {
		return err
	}

	updateFile, err := http.Get(updateURL)
	if err != nil {
		return err
	}
	defer updateFile.Body.Close()

	err = update.Apply(updateFile.Body, update.Options{
		Hash:     crypto.MD5,
		Checksum: checksum,
	})
	if err != nil {
		if rerr := update.RollbackError(err); rerr != nil {
			fmt.Println("Failed to rollback from bad update: %v", rerr)
		}
	}
	return err
}
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package cbgt

import (
	"fmt"
)

// The cbgt.VERSION tracks persistence versioning (schema/format of
// persisted data and configuration).  The main.VERSION from "git
// describe" that's part of an executable command, in contrast, is an
// overall "product" version.  For example, we might introduce new
// UI-only features or fix a UI typo, in which case we'd bump the
// main.VERSION number; but, if the persisted data/config format was
// unchanged, then the cbgt.VERSION number should remain unchanged.
//
// NOTE: You *must* update cbgt.VERSION if you change what's stored in
// the Cfg (such as the JSON/struct definitions or the planning
// algorithms).
const VERSION = "4.1.0"
const VERSION_KEY = "version"

// Returns true if a given version is modern enough to modify the Cfg.
// Older versions (which are running with older JSON/struct defintions
// or planning algorithms) will see false from their CheckVersion()'s.
func CheckVersion(cfg Cfg, myVersion string) (bool, error) {
	for cfg != nil {
		clusterVersion, cas, err := cfg.Get(VERSION_KEY, 0)
		if err != nil {
			return false, err
		}
		if clusterVersion == nil {
			// First time initialization, so save myVersion to cfg and
			// retry in case there was a race.
			_, err = cfg.Set(VERSION_KEY, []byte(myVersion), cas)
			if err != nil {
				return false, fmt.Errorf("version:"+
					" could not save VERSION to cfg, err: %v", err)
			}
			continue
		}
		if VersionGTE(myVersion, string(clusterVersion)) == false {
			return false, nil
		}
		if myVersion != string(clusterVersion) {
			// Found myVersion is higher than clusterVersion so save
			// myVersion to cfg and retry in case there was a race.
			_, err = cfg.Set(VERSION_KEY, []byte(myVersion), cas)
			if err != nil {
				return false, fmt.Errorf("version:"+
					" could not update VERSION in cfg, err: %v", err)
			}
			continue
		}
		return true, nil
	}

	return false, nil
}
<commit_msg>Fixed incorrect spelling of definition.<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package cbgt

import (
	"fmt"
)

// The cbgt.VERSION tracks persistence versioning (schema/format of
// persisted data and configuration).  The main.VERSION from "git
// describe" that's part of an executable command, in contrast, is an
// overall "product" version.  For example, we might introduce new
// UI-only features or fix a UI typo, in which case we'd bump the
// main.VERSION number; but, if the persisted data/config format was
// unchanged, then the cbgt.VERSION number should remain unchanged.
//
// NOTE: You *must* update cbgt.VERSION if you change what's stored in
// the Cfg (such as the JSON/struct definitions or the planning
// algorithms).
const VERSION = "4.1.0"
const VERSION_KEY = "version"

// Returns true if a given version is modern enough to modify the Cfg.
// Older versions (which are running with older JSON/struct definitions
// or planning algorithms) will see false from their CheckVersion()'s.
func CheckVersion(cfg Cfg, myVersion string) (bool, error) {
	for cfg != nil {
		clusterVersion, cas, err := cfg.Get(VERSION_KEY, 0)
		if err != nil {
			return false, err
		}
		if clusterVersion == nil {
			// First time initialization, so save myVersion to cfg and
			// retry in case there was a race.
			_, err = cfg.Set(VERSION_KEY, []byte(myVersion), cas)
			if err != nil {
				return false, fmt.Errorf("version:"+
					" could not save VERSION to cfg, err: %v", err)
			}
			continue
		}
		if VersionGTE(myVersion, string(clusterVersion)) == false {
			return false, nil
		}
		if myVersion != string(clusterVersion) {
			// Found myVersion is higher than clusterVersion so save
			// myVersion to cfg and retry in case there was a race.
			_, err = cfg.Set(VERSION_KEY, []byte(myVersion), cas)
			if err != nil {
				return false, fmt.Errorf("version:"+
					" could not update VERSION in cfg, err: %v", err)
			}
			continue
		}
		return true, nil
	}

	return false, nil
}
<|endoftext|>


<commit_before>// Package gosnowflake is a Go Snowflake Driver for Go's database/sql
//
// Copyright (c) 2017 Snowflake Computing Inc. All right reserved.
//
package gosnowflake

// SnowflakeGoDriverVersion is the version of Go Snowflake Driver
const SnowflakeGoDriverVersion = "0.2.0"
<commit_msg>Bumped up the version to 0.3.0<commit_after>// Package gosnowflake is a Go Snowflake Driver for Go's database/sql
//
// Copyright (c) 2017 Snowflake Computing Inc. All right reserved.
//
package gosnowflake

// SnowflakeGoDriverVersion is the version of Go Snowflake Driver
const SnowflakeGoDriverVersion = "0.3.0"
<|endoftext|>


<commit_before>package dispel

// Version represents the version of the API generated by dispel.
// Any visible change makes this version bump by 1.
const Version = 4
<commit_msg>bump version to 5<commit_after>package dispel

// Version represents the version of the API generated by dispel.
// Any visible change makes this version bump by 1.
const Version = 5
<|endoftext|>


<commit_before>package raygun4go

// the version of raygun4go
const packageVersion string = "1.1.1"
<commit_msg>upgrade package version to 1.2.0<commit_after>package raygun4go

// the version of raygun4go
const packageVersion string = "1.2.0"
<|endoftext|>


<commit_before>package main

const (
	// Release is the asset version of the site. Bump when any assets are
	// updated to blow away any browser caches.
	Release = "83"
)
<commit_msg>Bump assets version<commit_after>package main

const (
	// Release is the asset version of the site. Bump when any assets are
	// updated to blow away any browser caches.
	Release = "84"
)
<|endoftext|>


<commit_before>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package lameduck implements a lameducks provider. Lameduck provider fetches
// lameducks from the RTC (Runtime Configurator) service. This functionality
// allows an operator to do hitless VM upgrades. If a target is set to be in
// lameduck by the operator, it is taken out of the targets list.
package lameduck

import (
	"errors"
	"fmt"
	"net/http"
	"strings"
	"sync"
	"time"

	"cloud.google.com/go/compute/metadata"
	"github.com/google/cloudprober/logger"
	configpb "github.com/google/cloudprober/targets/lameduck/proto"
	"github.com/google/cloudprober/targets/rtc/rtcservice"
	runtimeconfig "google.golang.org/api/runtimeconfig/v1beta1"
)

// Lister is an interface for getting current lameducks.
type Lister interface {
	List() ([]string, error)
}

// global.lister is a singleton Lister. It caches data from the upstream config
// service, allowing for multiple consumers to lookup for lameducks without
// increasing load on the upstream service.
var global struct {
	mu     sync.RWMutex
	lister Lister
}

// Service provides methods to do lameduck operations on VMs.
type Service struct {
	rtc            rtcservice.Config
	opts           *configpb.Options
	expirationTime time.Duration
	l              *logger.Logger

	mu    sync.RWMutex
	names []string
}

// Updates the list of lameduck targets' names.
func (ldSvc *Service) expand() {
	resp, err := ldSvc.rtc.List()
	if err != nil {
		ldSvc.l.Errorf("targets: Error while getting the runtime config variables for lame-duck targets: %v", err)
		return
	}
	ldSvc.mu.Lock()
	ldSvc.names = ldSvc.processVars(resp)
	ldSvc.mu.Unlock()
}

// Returns the list of un-expired names of lameduck targets.
func (ldSvc *Service) processVars(vars []*runtimeconfig.Variable) []string {
	var result []string
	for _, v := range vars {
		ldSvc.l.Debugf("targets: Processing runtime-config var: %s", v.Name)

		// Variable names include the full path, including the config name.
		varParts := strings.Split(v.Name, "/")
		if len(varParts) == 0 {
			ldSvc.l.Errorf("targets: Invalid variable name for lame-duck targets: %s", v.Name)
			continue
		}
		ldName := varParts[len(varParts)-1]

		// Variable update time is in RFC3339 format
		// https://cloud.google.com/deployment-manager/runtime-configurator/reference/rest/v1beta1/projects.configs.variables
		updateTime, err := time.Parse(time.RFC3339Nano, v.UpdateTime)
		if err != nil {
			ldSvc.l.Errorf("targets: Could not parse variable(%s) update time (%s): %v", v.Name, v.UpdateTime, err)
			continue
		}
		if time.Since(updateTime) < time.Duration(ldSvc.opts.GetExpirationSec())*time.Second {
			ldSvc.l.Infof("targets: Marking target \"%s\" as lame duck.", ldName)
			result = append(result, ldName)
		} else {
			ldSvc.l.Infof("targets: Ignoring the stale (%s) lame duck (%s) entry", time.Since(updateTime), ldName)
		}
	}
	return result
}

// Lameduck puts the target in lameduck mode.
func (ldSvc *Service) Lameduck(name string) error {
	return ldSvc.rtc.Write(name, []byte{0})
}

// Unlameduck removes the target from lameduck mode.
func (ldSvc *Service) Unlameduck(name string) error {
	err := ldSvc.rtc.Delete(name)
	return err
}

// List returns the targets that are in lameduck mode.
func (ldSvc *Service) List() ([]string, error) {
	ldSvc.mu.RLock()
	defer ldSvc.mu.RUnlock()
	return append([]string{}, ldSvc.names...), nil
}

// NewService creates a new lameduck Service using the provided config options
// and an oauth2 enabled *http.Client; if the client is set to nil, an oauth
// enabled client is created automatically using GCP default credentials.
func NewService(optsProto *configpb.Options, c *http.Client, l *logger.Logger) (*Service, error) {
	if optsProto == nil {
		return nil, fmt.Errorf("lameduck.Init: failed to construct lameduck Service: no lameDuckOptions given")
	}

	proj := optsProto.GetRuntimeconfigProject()
	if proj == "" {
		var err error
		proj, err = metadata.ProjectID()
		if err != nil {
			return nil, fmt.Errorf("lameduck.Init: error while getting project id: %v", err)
		}
	}
	cfg := optsProto.GetRuntimeconfigName()

	rtc, err := rtcservice.New(proj, cfg, c)
	if err != nil {
		return nil, fmt.Errorf("lameduck.Init : rtcconfig service initialization failed : %v", err)
	}

	ldSvc := &Service{
		rtc:  rtc,
		opts: optsProto,
		l:    l,
	}
	ldSvc.expand()

	// Update the lameduck targets every [opts.ReEvalSec] seconds.
	go func() {
		for _ = range time.Tick(time.Duration(ldSvc.opts.GetReEvalSec()) * time.Second) {
			ldSvc.expand()
		}
	}()
	return ldSvc, nil
}

// InitDefaultLister initializes the package using the given arguments. If a
// lister is given in the arguments, global.lister is set to that, otherwise a
// new lameduck service is created using the config options, and global.lister
// is set to that service. Initiating the package from a given lister is useful
// for testing pacakges that depend on this package.
func InitDefaultLister(optsProto *configpb.Options, lister Lister, l *logger.Logger) error {
	global.mu.Lock()
	defer global.mu.Unlock()
	// Make sure we only initialize global.lister once.
	if global.lister != nil {
		return nil
	}

	if lister != nil {
		global.lister = lister
		return nil
	}

	ldSvc, err := NewService(optsProto, nil, l)
	if err != nil {
		return err
	}
	global.lister = ldSvc
	return nil
}

// GetDefaultLister returns the global Lister. If global lister is
// uninitialized, it returns an error.
func GetDefaultLister() (Lister, error) {
	global.mu.RLock()
	defer global.mu.RUnlock()
	if global.lister == nil {
		return nil, errors.New("global lameduck service not initialized")
	}
	return global.lister, nil
}
<commit_msg>Don't log expired lame-duck entries indefinitely.<commit_after>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package lameduck implements a lameducks provider. Lameduck provider fetches
// lameducks from the RTC (Runtime Configurator) service. This functionality
// allows an operator to do hitless VM upgrades. If a target is set to be in
// lameduck by the operator, it is taken out of the targets list.
package lameduck

import (
	"errors"
	"fmt"
	"net/http"
	"strings"
	"sync"
	"time"

	"cloud.google.com/go/compute/metadata"
	"github.com/google/cloudprober/logger"
	configpb "github.com/google/cloudprober/targets/lameduck/proto"
	"github.com/google/cloudprober/targets/rtc/rtcservice"
	runtimeconfig "google.golang.org/api/runtimeconfig/v1beta1"
)

// Lister is an interface for getting current lameducks.
type Lister interface {
	List() ([]string, error)
}

// global.lister is a singleton Lister. It caches data from the upstream config
// service, allowing for multiple consumers to lookup for lameducks without
// increasing load on the upstream service.
var global struct {
	mu     sync.RWMutex
	lister Lister
}

// Service provides methods to do lameduck operations on VMs.
type Service struct {
	rtc            rtcservice.Config
	opts           *configpb.Options
	expirationTime time.Duration
	l              *logger.Logger

	mu    sync.RWMutex
	names []string
}

// Updates the list of lameduck targets' names.
func (ldSvc *Service) expand() {
	resp, err := ldSvc.rtc.List()
	if err != nil {
		ldSvc.l.Errorf("targets: Error while getting the runtime config variables for lame-duck targets: %v", err)
		return
	}
	ldSvc.mu.Lock()
	ldSvc.names = ldSvc.processVars(resp)
	ldSvc.mu.Unlock()
}

// Returns the list of un-expired names of lameduck targets.
func (ldSvc *Service) processVars(vars []*runtimeconfig.Variable) []string {
	var result []string
	expirationTime := time.Duration(ldSvc.opts.GetExpirationSec()) * time.Second
	for _, v := range vars {
		ldSvc.l.Debugf("targets: Processing runtime-config var: %s", v.Name)

		// Variable names include the full path, including the config name.
		varParts := strings.Split(v.Name, "/")
		if len(varParts) == 0 {
			ldSvc.l.Errorf("targets: Invalid variable name for lame-duck targets: %s", v.Name)
			continue
		}
		ldName := varParts[len(varParts)-1]

		// Variable update time is in RFC3339 format
		// https://cloud.google.com/deployment-manager/runtime-configurator/reference/rest/v1beta1/projects.configs.variables
		updateTime, err := time.Parse(time.RFC3339Nano, v.UpdateTime)
		if err != nil {
			ldSvc.l.Errorf("targets: Could not parse variable(%s) update time (%s): %v", v.Name, v.UpdateTime, err)
			continue
		}
		if time.Since(updateTime) < expirationTime {
			ldSvc.l.Infof("targets: Marking target \"%s\" as lame duck.", ldName)
			result = append(result, ldName)
			continue
		}
		// Log only if variable is not older than 10 times of expiration time. This
		// is to avoid keep logging old expired entries.
		if time.Since(updateTime) < 10*expirationTime {
			ldSvc.l.Infof("targets: Ignoring the stale (%s) lame duck (%s) entry", time.Since(updateTime), ldName)
		}
	}
	return result
}

// Lameduck puts the target in lameduck mode.
func (ldSvc *Service) Lameduck(name string) error {
	return ldSvc.rtc.Write(name, []byte{0})
}

// Unlameduck removes the target from lameduck mode.
func (ldSvc *Service) Unlameduck(name string) error {
	err := ldSvc.rtc.Delete(name)
	return err
}

// List returns the targets that are in lameduck mode.
func (ldSvc *Service) List() ([]string, error) {
	ldSvc.mu.RLock()
	defer ldSvc.mu.RUnlock()
	return append([]string{}, ldSvc.names...), nil
}

// NewService creates a new lameduck Service using the provided config options
// and an oauth2 enabled *http.Client; if the client is set to nil, an oauth
// enabled client is created automatically using GCP default credentials.
func NewService(optsProto *configpb.Options, c *http.Client, l *logger.Logger) (*Service, error) {
	if optsProto == nil {
		return nil, fmt.Errorf("lameduck.Init: failed to construct lameduck Service: no lameDuckOptions given")
	}

	proj := optsProto.GetRuntimeconfigProject()
	if proj == "" {
		var err error
		proj, err = metadata.ProjectID()
		if err != nil {
			return nil, fmt.Errorf("lameduck.Init: error while getting project id: %v", err)
		}
	}
	cfg := optsProto.GetRuntimeconfigName()

	rtc, err := rtcservice.New(proj, cfg, c)
	if err != nil {
		return nil, fmt.Errorf("lameduck.Init : rtcconfig service initialization failed : %v", err)
	}

	ldSvc := &Service{
		rtc:  rtc,
		opts: optsProto,
		l:    l,
	}
	ldSvc.expand()

	// Update the lameduck targets every [opts.ReEvalSec] seconds.
	go func() {
		for _ = range time.Tick(time.Duration(ldSvc.opts.GetReEvalSec()) * time.Second) {
			ldSvc.expand()
		}
	}()
	return ldSvc, nil
}

// InitDefaultLister initializes the package using the given arguments. If a
// lister is given in the arguments, global.lister is set to that, otherwise a
// new lameduck service is created using the config options, and global.lister
// is set to that service. Initiating the package from a given lister is useful
// for testing pacakges that depend on this package.
func InitDefaultLister(optsProto *configpb.Options, lister Lister, l *logger.Logger) error {
	global.mu.Lock()
	defer global.mu.Unlock()
	// Make sure we only initialize global.lister once.
	if global.lister != nil {
		return nil
	}

	if lister != nil {
		global.lister = lister
		return nil
	}

	ldSvc, err := NewService(optsProto, nil, l)
	if err != nil {
		return err
	}
	global.lister = ldSvc
	return nil
}

// GetDefaultLister returns the global Lister. If global lister is
// uninitialized, it returns an error.
func GetDefaultLister() (Lister, error) {
	global.mu.RLock()
	defer global.mu.RUnlock()
	if global.lister == nil {
		return nil, errors.New("global lameduck service not initialized")
	}
	return global.lister, nil
}
<|endoftext|>