<commit_before>package gc_test

import (
	"os"
	"time"

	"code.cloudfoundry.org/lager/lagertest"

	sq "github.com/Masterminds/squirrel"
	"github.com/concourse/concourse/atc"
	"github.com/concourse/concourse/atc/db"
	"github.com/concourse/concourse/atc/db/lock"
	"github.com/concourse/concourse/atc/postgresrunner"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/tedsuo/ifrit"

	"testing"
)

func TestGc(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Gc Suite")
}

var psql = sq.StatementBuilder.PlaceholderFormat(sq.Dollar)

var (
	postgresRunner postgresrunner.Runner
	dbProcess      ifrit.Process

	dbConn                            db.Conn
	err                               error
	resourceCacheFactory              db.ResourceCacheFactory
	resourceCacheLifecycle            db.ResourceCacheLifecycle
	resourceConfigFactory             db.ResourceConfigFactory
	resourceConfigCheckSessionFactory db.ResourceConfigCheckSessionFactory
	buildFactory                      db.BuildFactory
	lockFactory                       lock.LockFactory

	teamFactory db.TeamFactory

	defaultTeam     db.Team
	defaultPipeline db.Pipeline
	defaultJob      db.Job
	defaultBuild    db.Build

	usedResource db.Resource
	logger       *lagertest.TestLogger
)

var _ = BeforeSuite(func() {
	postgresRunner = postgresrunner.Runner{
		Port: 5433 + GinkgoParallelNode(),
	}

	dbProcess = ifrit.Invoke(postgresRunner)

	postgresRunner.CreateTestDB()
})

var _ = BeforeEach(func() {
	postgresRunner.Truncate()

	dbConn = postgresRunner.OpenConn()

	lockFactory = lock.NewLockFactory(postgresRunner.OpenSingleton())

	teamFactory = db.NewTeamFactory(dbConn, lockFactory)
	buildFactory = db.NewBuildFactory(dbConn, lockFactory, 0)

	defaultTeam, err = teamFactory.CreateTeam(atc.Team{Name: "default-team"})
	Expect(err).NotTo(HaveOccurred())

	defaultBuild, err = defaultTeam.CreateOneOffBuild()
	Expect(err).NotTo(HaveOccurred())

	atcConfig := atc.Config{
		Resources: atc.ResourceConfigs{
			{
				Name:   "some-resource",
				Type:   "some-base-type",
				Source: atc.Source{"some": "source"},
			},
		},
		Jobs: atc.JobConfigs{
			{
				Name: "some-job",
			},
			{
				Name: "some-other-job",
			},
		},
	}

	defaultPipeline, _, err = defaultTeam.SavePipeline("default-pipeline", atcConfig, db.ConfigVersion(0), db.PipelineUnpaused)
	Expect(err).NotTo(HaveOccurred())

	var found bool
	defaultJob, found, err = defaultPipeline.Job("some-job")
	Expect(err).NotTo(HaveOccurred())
	Expect(found).To(BeTrue())

	usedResource, found, err = defaultPipeline.Resource("some-resource")
	Expect(err).NotTo(HaveOccurred())
	Expect(found).To(BeTrue())

	setupTx, err := dbConn.Begin()
	Expect(err).ToNot(HaveOccurred())

	baseResourceType := db.BaseResourceType{
		Name: "some-base-type",
	}
	_, err = baseResourceType.FindOrCreate(setupTx)
	Expect(err).NotTo(HaveOccurred())

	Expect(setupTx.Commit()).To(Succeed())

	logger = lagertest.NewTestLogger("gc-test")

	resourceCacheLifecycle = db.NewResourceCacheLifecycle(dbConn)
	resourceCacheFactory = db.NewResourceCacheFactory(dbConn, lockFactory)
	resourceConfigFactory = db.NewResourceConfigFactory(dbConn, lockFactory)
	resourceConfigCheckSessionFactory = db.NewResourceConfigCheckSessionFactory(dbConn, lockFactory)
})

var _ = AfterEach(func() {
	Expect(dbConn.Close()).To(Succeed())
})

var _ = AfterSuite(func() {
	dbProcess.Signal(os.Interrupt)
	Eventually(dbProcess.Wait(), 10*time.Second).Should(Receive())
})
<commit_msg>missed a spot in gc tests<commit_after>package gc_test

import (
	"code.cloudfoundry.org/lager"
	"os"
	"time"

	"code.cloudfoundry.org/lager/lagertest"

	sq "github.com/Masterminds/squirrel"
	"github.com/concourse/concourse/atc"
	"github.com/concourse/concourse/atc/db"
	"github.com/concourse/concourse/atc/db/lock"
	"github.com/concourse/concourse/atc/postgresrunner"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/tedsuo/ifrit"

	"testing"
)

func TestGc(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Gc Suite")
}

var psql = sq.StatementBuilder.PlaceholderFormat(sq.Dollar)

var (
	postgresRunner postgresrunner.Runner
	dbProcess      ifrit.Process

	dbConn                            db.Conn
	err                               error
	resourceCacheFactory              db.ResourceCacheFactory
	resourceCacheLifecycle            db.ResourceCacheLifecycle
	resourceConfigFactory             db.ResourceConfigFactory
	resourceConfigCheckSessionFactory db.ResourceConfigCheckSessionFactory
	buildFactory                      db.BuildFactory
	lockFactory                       lock.LockFactory

	teamFactory db.TeamFactory

	defaultTeam     db.Team
	defaultPipeline db.Pipeline
	defaultJob      db.Job
	defaultBuild    db.Build

	usedResource db.Resource
	logger       *lagertest.TestLogger
	fakeLogFunc = func(logger lager.Logger, id lock.LockID){}
)

var _ = BeforeSuite(func() {
	postgresRunner = postgresrunner.Runner{
		Port: 5433 + GinkgoParallelNode(),
	}

	dbProcess = ifrit.Invoke(postgresRunner)

	postgresRunner.CreateTestDB()
})

var _ = BeforeEach(func() {
	postgresRunner.Truncate()

	dbConn = postgresRunner.OpenConn()

	lockFactory = lock.NewLockFactory(postgresRunner.OpenSingleton(), fakeLogFunc, fakeLogFunc)

	teamFactory = db.NewTeamFactory(dbConn, lockFactory)
	buildFactory = db.NewBuildFactory(dbConn, lockFactory, 0)

	defaultTeam, err = teamFactory.CreateTeam(atc.Team{Name: "default-team"})
	Expect(err).NotTo(HaveOccurred())

	defaultBuild, err = defaultTeam.CreateOneOffBuild()
	Expect(err).NotTo(HaveOccurred())

	atcConfig := atc.Config{
		Resources: atc.ResourceConfigs{
			{
				Name:   "some-resource",
				Type:   "some-base-type",
				Source: atc.Source{"some": "source"},
			},
		},
		Jobs: atc.JobConfigs{
			{
				Name: "some-job",
			},
			{
				Name: "some-other-job",
			},
		},
	}

	defaultPipeline, _, err = defaultTeam.SavePipeline("default-pipeline", atcConfig, db.ConfigVersion(0), db.PipelineUnpaused)
	Expect(err).NotTo(HaveOccurred())

	var found bool
	defaultJob, found, err = defaultPipeline.Job("some-job")
	Expect(err).NotTo(HaveOccurred())
	Expect(found).To(BeTrue())

	usedResource, found, err = defaultPipeline.Resource("some-resource")
	Expect(err).NotTo(HaveOccurred())
	Expect(found).To(BeTrue())

	setupTx, err := dbConn.Begin()
	Expect(err).ToNot(HaveOccurred())

	baseResourceType := db.BaseResourceType{
		Name: "some-base-type",
	}
	_, err = baseResourceType.FindOrCreate(setupTx)
	Expect(err).NotTo(HaveOccurred())

	Expect(setupTx.Commit()).To(Succeed())

	logger = lagertest.NewTestLogger("gc-test")

	resourceCacheLifecycle = db.NewResourceCacheLifecycle(dbConn)
	resourceCacheFactory = db.NewResourceCacheFactory(dbConn, lockFactory)
	resourceConfigFactory = db.NewResourceConfigFactory(dbConn, lockFactory)
	resourceConfigCheckSessionFactory = db.NewResourceConfigCheckSessionFactory(dbConn, lockFactory)
})

var _ = AfterEach(func() {
	Expect(dbConn.Close()).To(Succeed())
})

var _ = AfterSuite(func() {
	dbProcess.Signal(os.Interrupt)
	Eventually(dbProcess.Wait(), 10*time.Second).Should(Receive())
})
<|endoftext|>


<commit_before>package application

import (
	"encoding/json"
	"github.com/thoas/gokvstores"
	"github.com/thoas/muxer"
	"github.com/thoas/picfit/extractors"
	"github.com/thoas/picfit/hash"
	"github.com/thoas/picfit/image"
	"github.com/thoas/picfit/util"
	"net/http"
	"net/url"
)

var Extractors = map[string]extractors.Extractor{
	"op":  extractors.Operation,
	"fmt": extractors.Format,
	"url": extractors.URL,
	"q":   extractors.Quality,
}

func NotFoundHandler() http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		http.Error(w, "404 not found", http.StatusNotFound)
	})
}

type Options struct {
	Format  string
	Quality int
}

type Request struct {
	*muxer.Request
	Operation  *image.Operation
	Connection gokvstores.KVStoreConnection
	Key        string
	URL        *url.URL
	Filepath   string
	Options    *Options
}

type Handler func(muxer.Response, *Request)

func (h Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	con := App.KVStore.Connection()
	defer con.Close()

	request := muxer.NewRequest(req)

	for k, v := range request.Params {
		request.QueryString[k] = v
	}

	res := muxer.NewResponse(w)

	extracted := map[string]interface{}{}

	for key, extractor := range Extractors {
		result, err := extractor(key, request)

		if err != nil {
			App.Logger.Info(err)

			res.BadRequest()
			return
		}

		extracted[key] = result
	}

	sorted := util.SortMapString(request.QueryString)

	valid := App.IsValidSign(sorted)

	delete(sorted, "sig")

	serialized := hash.Serialize(sorted)

	key := hash.Tokey(serialized)

	App.Logger.Infof("Generating key %s from request: %s", key, serialized)

	var u *url.URL
	var path string
	var format string
	var quality int

	value, ok := extracted["url"]

	if ok {
		u = value.(*url.URL)
	}

	value, ok = extracted["path"]

	if ok {
		path = string(path)
	}

	if !valid || (u == nil && path == "") {
		res.BadRequest()
		return
	}

	value, ok = extracted["fmt"]

	if ok {
		format = string(path)
	}

	value, ok = extracted["q"]

	if ok && value != nil {
		quality = value.(int)
	}

	options := &Options{Quality: quality, Format: format}

	h(res, &Request{
		request,
		extracted["op"].(*image.Operation),
		con,
		key,
		u,
		path,
		options,
	})
}

var ImageHandler Handler = func(res muxer.Response, req *Request) {
	file, err := App.ImageFileFromRequest(req, true, true)

	util.PanicIf(err)

	content, err := file.ToBytes()

	util.PanicIf(err)

	res.SetHeaders(file.Headers, true)
	res.ResponseWriter.Write(content)
}

var GetHandler Handler = func(res muxer.Response, req *Request) {
	file, err := App.ImageFileFromRequest(req, false, false)

	util.PanicIf(err)

	content, err := json.Marshal(map[string]string{
		"filename": file.Filename(),
		"path":     file.Path(),
		"url":      file.URL(),
	})

	util.PanicIf(err)

	res.ContentType("application/json")
	res.ResponseWriter.Write(content)
}

var RedirectHandler Handler = func(res muxer.Response, req *Request) {
	file, err := App.ImageFileFromRequest(req, false, false)

	util.PanicIf(err)

	res.PermanentRedirect(file.URL())
}
<commit_msg>Add missing path extractors<commit_after>package application

import (
	"encoding/json"
	"github.com/thoas/gokvstores"
	"github.com/thoas/muxer"
	"github.com/thoas/picfit/extractors"
	"github.com/thoas/picfit/hash"
	"github.com/thoas/picfit/image"
	"github.com/thoas/picfit/util"
	"net/http"
	"net/url"
)

var Extractors = map[string]extractors.Extractor{
	"op":   extractors.Operation,
	"fmt":  extractors.Format,
	"url":  extractors.URL,
	"q":    extractors.Quality,
	"path": extractors.Path,
}

func NotFoundHandler() http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		http.Error(w, "404 not found", http.StatusNotFound)
	})
}

type Options struct {
	Format  string
	Quality int
}

type Request struct {
	*muxer.Request
	Operation  *image.Operation
	Connection gokvstores.KVStoreConnection
	Key        string
	URL        *url.URL
	Filepath   string
	Options    *Options
}

type Handler func(muxer.Response, *Request)

func (h Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	con := App.KVStore.Connection()
	defer con.Close()

	request := muxer.NewRequest(req)

	for k, v := range request.Params {
		request.QueryString[k] = v
	}

	res := muxer.NewResponse(w)

	extracted := map[string]interface{}{}

	for key, extractor := range Extractors {
		result, err := extractor(key, request)

		if err != nil {
			App.Logger.Info(err)

			res.BadRequest()
			return
		}

		extracted[key] = result
	}

	sorted := util.SortMapString(request.QueryString)

	valid := App.IsValidSign(sorted)

	delete(sorted, "sig")

	serialized := hash.Serialize(sorted)

	key := hash.Tokey(serialized)

	App.Logger.Infof("Generating key %s from request: %s", key, serialized)

	var u *url.URL
	var path string
	var format string
	var quality int

	value, ok := extracted["url"]

	if ok && value != nil {
		u = value.(*url.URL)
	}

	value, ok = extracted["path"]

	if ok {
		path = value.(string)
	}

	if !valid || (u == nil && path == "") {
		res.BadRequest()
		return
	}

	value, ok = extracted["fmt"]

	if ok {
		format = string(path)
	}

	value, ok = extracted["q"]

	if ok && value != nil {
		quality = value.(int)
	}

	options := &Options{Quality: quality, Format: format}

	h(res, &Request{
		request,
		extracted["op"].(*image.Operation),
		con,
		key,
		u,
		path,
		options,
	})
}

var ImageHandler Handler = func(res muxer.Response, req *Request) {
	file, err := App.ImageFileFromRequest(req, true, true)

	util.PanicIf(err)

	content, err := file.ToBytes()

	util.PanicIf(err)

	res.SetHeaders(file.Headers, true)
	res.ResponseWriter.Write(content)
}

var GetHandler Handler = func(res muxer.Response, req *Request) {
	file, err := App.ImageFileFromRequest(req, false, false)

	util.PanicIf(err)

	content, err := json.Marshal(map[string]string{
		"filename": file.Filename(),
		"path":     file.Path(),
		"url":      file.URL(),
	})

	util.PanicIf(err)

	res.ContentType("application/json")
	res.ResponseWriter.Write(content)
}

var RedirectHandler Handler = func(res muxer.Response, req *Request) {
	file, err := App.ImageFileFromRequest(req, false, false)

	util.PanicIf(err)

	res.PermanentRedirect(file.URL())
}
<|endoftext|>


<commit_before>// Copyright (c) 2016, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package sh

import (
	"fmt"
	"io"
	"strings"
)

func Fprint(w io.Writer, n Node) error {
	p := printer{
		w:       w,
		curLine: 1,
	}
	if f, ok := n.(File); ok {
		p.comments = f.Comments
	}
	p.node(n)
	return p.err
}

type printer struct {
	w   io.Writer
	err error

	contiguous bool

	curLine int
	level   int

	comments []Comment

	compactArithm bool
}

var (
	contiguousRight = map[Token]bool{
		DOLLPR:  true,
		LPAREN:  true,
		DLPAREN: true,
		BQUOTE:  true,
		CMDIN:   true,
		DOLLDP:  true,
	}
	contiguousLeft = map[Token]bool{
		SEMICOLON:  true,
		DSEMICOLON: true,
		RPAREN:     true,
		DRPAREN:    true,
		COMMA:      true,
	}
)

func (p *printer) space(b byte) {
	if p.err != nil {
		return
	}
	_, p.err = p.w.Write([]byte{b})
	p.contiguous = false
}

func (p *printer) nonSpaced(a ...interface{}) {
	for _, v := range a {
		if p.err != nil {
			break
		}
		switch x := v.(type) {
		case string:
			if len(x) > 0 {
				last := x[len(x)-1]
				p.contiguous = !space[last]
			}
			_, p.err = io.WriteString(p.w, x)
			p.curLine += strings.Count(x, "\n")
		case Comment:
			p.contiguous = true
			_, p.err = fmt.Fprint(p.w, HASH, x.Text)
		case Token:
			p.contiguous = !contiguousRight[x]
			_, p.err = fmt.Fprint(p.w, x)
		case Node:
			p.node(x)
		}
	}
}

func (p *printer) spaced(a ...interface{}) {
	for _, v := range a {
		if v == nil {
			continue
		}
		if t, ok := v.(Token); ok && contiguousLeft[t] {
		} else if p.contiguous {
			p.space(' ')
		}
		p.nonSpaced(v)
	}
}

func (p *printer) indent() {
	for i := 0; i < p.level; i++ {
		p.space('\t')
	}
}

func (p *printer) separate(pos Pos, fallback, allowTwo bool) {
	p.commentsUpTo(pos.Line)
	if pos.Line > p.curLine {
		p.space('\n')
		if allowTwo && pos.Line > p.curLine+1 {
			// preserve single empty lines
			p.space('\n')
		}
		p.indent()
		p.curLine = pos.Line
	} else if fallback {
		p.nonSpaced(SEMICOLON)
	}
}

func (p *printer) sepSemicolon(v interface{}, pos Pos) {
	p.separate(pos, true, false)
	p.spaced(v)
}

func (p *printer) sepNewline(v interface{}, pos Pos) {
	p.separate(pos, false, true)
	p.spaced(v)
}

func (p *printer) commentsUpTo(line int) {
	if len(p.comments) < 1 {
		return
	}
	c := p.comments[0]
	if line > 0 && c.Hash.Line >= line {
		return
	}
	p.sepNewline(c, c.Hash)
	p.comments = p.comments[1:]
	p.commentsUpTo(line)
}

func (p *printer) node(n Node) {
	switch x := n.(type) {
	case File:
		p.progStmts(x.Stmts)
		p.commentsUpTo(0)
		p.space('\n')
	case Stmt:
		if x.Negated {
			p.spaced(NOT)
		}
		for _, a := range x.Assigns {
			p.spaced(a)
		}
		p.spaced(x.Node)
		for _, r := range x.Redirs {
			p.spaced(r.N)
			p.nonSpaced(r.Op, r.Word)
		}
		if x.Background {
			p.spaced(AND)
		}
	case Assign:
		if x.Name != nil {
			p.spaced(x.Name)
			if x.Append {
				p.nonSpaced(ADD_ASSIGN)
			} else {
				p.nonSpaced(ASSIGN)
			}
		}
		p.nonSpaced(x.Value)
	case Command:
		p.wordJoin(x.Args, true)
	case Subshell:
		p.spaced(LPAREN)
		if len(x.Stmts) == 0 {
			// avoid conflict with ()
			p.space(' ')
		}
		p.stmtJoin(x.Stmts)
		p.sepNewline(RPAREN, x.Rparen)
	case Block:
		p.spaced(LBRACE)
		p.stmtJoin(x.Stmts)
		p.sepSemicolon(RBRACE, x.Rbrace)
	case IfStmt:
		p.spaced(IF, x.Cond, SEMICOLON, THEN)
		p.stmtJoin(x.ThenStmts)
		for _, el := range x.Elifs {
			p.sepSemicolon(ELIF, el.Elif)
			p.spaced(el.Cond, SEMICOLON, THEN)
			p.stmtJoin(el.ThenStmts)
		}
		if len(x.ElseStmts) > 0 {
			p.sepSemicolon(ELSE, x.Else)
			p.stmtJoin(x.ElseStmts)
		}
		p.sepSemicolon(FI, x.Fi)
	case StmtCond:
		p.stmtJoin(x.Stmts)
	case CStyleCond:
		p.spaced(DLPAREN, x.Cond, DRPAREN)
	case WhileStmt:
		p.spaced(WHILE, x.Cond, SEMICOLON, DO)
		p.stmtJoin(x.DoStmts)
		p.sepSemicolon(DONE, x.Done)
	case UntilStmt:
		p.spaced(UNTIL, x.Cond, SEMICOLON, DO)
		p.stmtJoin(x.DoStmts)
		p.sepSemicolon(DONE, x.Done)
	case ForStmt:
		p.spaced(FOR, x.Cond, SEMICOLON, DO)
		p.stmtJoin(x.DoStmts)
		p.sepSemicolon(DONE, x.Done)
	case WordIter:
		p.spaced(x.Name)
		if len(x.List) > 0 {
			p.spaced(IN)
			p.wordJoin(x.List, false)
		}
	case CStyleLoop:
		p.spaced(DLPAREN, x.Init, SEMICOLON, x.Cond,
			SEMICOLON, x.Post, DRPAREN)
	case UnaryExpr:
		if x.Post {
			p.nonSpaced(x.X, x.Op)
		} else {
			p.nonSpaced(x.Op)
			p.contiguous = false
			p.nonSpaced(x.X)
		}
	case BinaryExpr:
		if p.compactArithm {
			p.nonSpaced(x.X, x.Op, x.Y)
		} else {
			p.spaced(x.X, x.Op, x.Y)
		}
	case FuncDecl:
		if x.BashStyle {
			p.spaced(FUNCTION)
		}
		p.spaced(x.Name)
		if !x.BashStyle {
			p.nonSpaced(LPAREN, RPAREN)
		}
		p.spaced(x.Body)
	case Word:
		for _, n := range x.Parts {
			p.nonSpaced(n)
		}
	case Lit:
		p.nonSpaced(x.Value)
	case SglQuoted:
		p.nonSpaced(SQUOTE, x.Value, SQUOTE)
	case Quoted:
		p.nonSpaced(x.Quote)
		for _, n := range x.Parts {
			p.nonSpaced(n)
		}
		p.nonSpaced(quotedStop(x.Quote))
	case CmdSubst:
		if x.Backquotes {
			p.nonSpaced(BQUOTE)
		} else {
			p.nonSpaced(DOLLPR)
		}
		p.stmtJoin(x.Stmts)
		if x.Backquotes {
			p.nonSpaced(BQUOTE)
		} else {
			p.nonSpaced(RPAREN)
		}
	case ParamExp:
		if x.Short {
			p.nonSpaced(DOLLAR, x.Param)
			return
		}
		p.nonSpaced(DOLLBR)
		if x.Length {
			p.nonSpaced(HASH)
		}
		p.nonSpaced(x.Param)
		if x.Ind != nil {
			p.nonSpaced(LBRACK, x.Ind.Word, RBRACK)
		}
		if x.Repl != nil {
			if x.Repl.All {
				p.nonSpaced(QUO)
			}
			p.nonSpaced(QUO, x.Repl.Orig, QUO, x.Repl.With)
		} else if x.Exp != nil {
			p.nonSpaced(x.Exp.Op, x.Exp.Word)
		}
		p.nonSpaced(RBRACE)
	case ArithmExpr:
		p.nonSpaced(DOLLDP, x.X, DRPAREN)
	case ParenExpr:
		p.nonSpaced(LPAREN)
		oldCompact := p.compactArithm
		p.compactArithm = false
		p.nonSpaced(x.X)
		p.compactArithm = oldCompact
		p.nonSpaced(RPAREN)
	case CaseStmt:
		p.spaced(CASE, x.Word, IN)
		for _, pl := range x.List {
			p.separate(wordFirstPos(pl.Patterns), false, true)
			for i, w := range pl.Patterns {
				if i > 0 {
					p.spaced(OR)
				}
				p.spaced(w)
			}
			p.nonSpaced(RPAREN)
			p.stmtJoin(pl.Stmts)
			p.level++
			p.sepNewline(DSEMICOLON, pl.Dsemi)
			p.level--
		}
		if len(x.List) == 0 {
			p.sepSemicolon(ESAC, x.Esac)
		} else {
			p.sepNewline(ESAC, x.Esac)
		}
	case DeclStmt:
		if x.Local {
			p.spaced(LOCAL)
		} else {
			p.spaced(DECLARE)
		}
		for _, w := range x.Opts {
			p.spaced(w)
		}
		for _, a := range x.Assigns {
			p.spaced(a)
		}
	case ArrayExpr:
		p.nonSpaced(LPAREN)
		p.wordJoin(x.List, false)
		p.nonSpaced(RPAREN)
	case CmdInput:
		// avoid conflict with <<
		p.spaced(CMDIN)
		p.stmtJoin(x.Stmts)
		p.nonSpaced(RPAREN)
	case EvalStmt:
		p.spaced(EVAL, x.Stmt)
	case LetStmt:
		p.spaced(LET)
		p.compactArithm = true
		for _, n := range x.Exprs {
			p.spaced(n)
		}
		p.compactArithm = false
	}
}

func (p *printer) wordJoin(ws []Word, keepNewlines bool) {
	anyNewline := false
	for _, w := range ws {
		if keepNewlines && w.Pos().Line > p.curLine {
			p.spaced("\\\n")
			if !anyNewline {
				p.level++
				anyNewline = true
			}
			p.indent()
		}
		p.spaced(w)
	}
	if anyNewline {
		p.level--
	}
}

func (p *printer) progStmts(stmts []Stmt) {
	for i, s := range stmts {
		p.separate(s.Pos(), i > 0, true)
		p.node(s)
	}
}

func (p *printer) stmtJoin(stmts []Stmt) {
	p.level++
	for i, s := range stmts {
		p.separate(s.Pos(), i > 0, i > 0)
		p.node(s)
	}
	p.level--
}
<commit_msg>print: properly indent trailing comments<commit_after>// Copyright (c) 2016, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package sh

import (
	"fmt"
	"io"
	"strings"
)

func Fprint(w io.Writer, n Node) error {
	p := printer{
		w:       w,
		curLine: 1,
	}
	if f, ok := n.(File); ok {
		p.comments = f.Comments
	}
	p.node(n)
	return p.err
}

type printer struct {
	w   io.Writer
	err error

	contiguous bool

	curLine int
	level   int

	comments []Comment

	compactArithm bool
}

var (
	contiguousRight = map[Token]bool{
		DOLLPR:  true,
		LPAREN:  true,
		DLPAREN: true,
		BQUOTE:  true,
		CMDIN:   true,
		DOLLDP:  true,
	}
	contiguousLeft = map[Token]bool{
		SEMICOLON:  true,
		DSEMICOLON: true,
		RPAREN:     true,
		DRPAREN:    true,
		COMMA:      true,
	}
)

func (p *printer) space(b byte) {
	if p.err != nil {
		return
	}
	_, p.err = p.w.Write([]byte{b})
	p.contiguous = false
}

func (p *printer) nonSpaced(a ...interface{}) {
	for _, v := range a {
		if p.err != nil {
			break
		}
		switch x := v.(type) {
		case string:
			if len(x) > 0 {
				last := x[len(x)-1]
				p.contiguous = !space[last]
			}
			_, p.err = io.WriteString(p.w, x)
			p.curLine += strings.Count(x, "\n")
		case Comment:
			p.contiguous = true
			_, p.err = fmt.Fprint(p.w, HASH, x.Text)
		case Token:
			p.contiguous = !contiguousRight[x]
			_, p.err = fmt.Fprint(p.w, x)
		case Node:
			p.node(x)
		}
	}
}

func (p *printer) spaced(a ...interface{}) {
	for _, v := range a {
		if v == nil {
			continue
		}
		if t, ok := v.(Token); ok && contiguousLeft[t] {
		} else if p.contiguous {
			p.space(' ')
		}
		p.nonSpaced(v)
	}
}

func (p *printer) indent() {
	for i := 0; i < p.level; i++ {
		p.space('\t')
	}
}

func (p *printer) separate(pos Pos, fallback, allowTwo bool) {
	p.commentsUpTo(pos.Line)
	if pos.Line > p.curLine {
		p.space('\n')
		if allowTwo && pos.Line > p.curLine+1 {
			// preserve single empty lines
			p.space('\n')
		}
		p.indent()
		p.curLine = pos.Line
	} else if fallback {
		p.nonSpaced(SEMICOLON)
	}
}

func (p *printer) sepSemicolon(v interface{}, pos Pos) {
	p.level++
	p.commentsUpTo(pos.Line)
	p.level--
	p.separate(pos, true, false)
	p.spaced(v)
}

func (p *printer) sepNewline(v interface{}, pos Pos) {
	p.separate(pos, false, true)
	p.spaced(v)
}

func (p *printer) commentsUpTo(line int) {
	if len(p.comments) < 1 {
		return
	}
	c := p.comments[0]
	if line > 0 && c.Hash.Line >= line {
		return
	}
	p.sepNewline(c, c.Hash)
	p.comments = p.comments[1:]
	p.commentsUpTo(line)
}

func (p *printer) node(n Node) {
	switch x := n.(type) {
	case File:
		p.progStmts(x.Stmts)
		p.commentsUpTo(0)
		p.space('\n')
	case Stmt:
		if x.Negated {
			p.spaced(NOT)
		}
		for _, a := range x.Assigns {
			p.spaced(a)
		}
		p.spaced(x.Node)
		for _, r := range x.Redirs {
			p.spaced(r.N)
			p.nonSpaced(r.Op, r.Word)
		}
		if x.Background {
			p.spaced(AND)
		}
	case Assign:
		if x.Name != nil {
			p.spaced(x.Name)
			if x.Append {
				p.nonSpaced(ADD_ASSIGN)
			} else {
				p.nonSpaced(ASSIGN)
			}
		}
		p.nonSpaced(x.Value)
	case Command:
		p.wordJoin(x.Args, true)
	case Subshell:
		p.spaced(LPAREN)
		if len(x.Stmts) == 0 {
			// avoid conflict with ()
			p.space(' ')
		}
		p.stmtJoin(x.Stmts)
		p.sepNewline(RPAREN, x.Rparen)
	case Block:
		p.spaced(LBRACE)
		p.stmtJoin(x.Stmts)
		p.sepSemicolon(RBRACE, x.Rbrace)
	case IfStmt:
		p.spaced(IF, x.Cond, SEMICOLON, THEN)
		p.stmtJoin(x.ThenStmts)
		for _, el := range x.Elifs {
			p.sepSemicolon(ELIF, el.Elif)
			p.spaced(el.Cond, SEMICOLON, THEN)
			p.stmtJoin(el.ThenStmts)
		}
		if len(x.ElseStmts) > 0 {
			p.sepSemicolon(ELSE, x.Else)
			p.stmtJoin(x.ElseStmts)
		}
		p.sepSemicolon(FI, x.Fi)
	case StmtCond:
		p.stmtJoin(x.Stmts)
	case CStyleCond:
		p.spaced(DLPAREN, x.Cond, DRPAREN)
	case WhileStmt:
		p.spaced(WHILE, x.Cond, SEMICOLON, DO)
		p.stmtJoin(x.DoStmts)
		p.sepSemicolon(DONE, x.Done)
	case UntilStmt:
		p.spaced(UNTIL, x.Cond, SEMICOLON, DO)
		p.stmtJoin(x.DoStmts)
		p.sepSemicolon(DONE, x.Done)
	case ForStmt:
		p.spaced(FOR, x.Cond, SEMICOLON, DO)
		p.stmtJoin(x.DoStmts)
		p.sepSemicolon(DONE, x.Done)
	case WordIter:
		p.spaced(x.Name)
		if len(x.List) > 0 {
			p.spaced(IN)
			p.wordJoin(x.List, false)
		}
	case CStyleLoop:
		p.spaced(DLPAREN, x.Init, SEMICOLON, x.Cond,
			SEMICOLON, x.Post, DRPAREN)
	case UnaryExpr:
		if x.Post {
			p.nonSpaced(x.X, x.Op)
		} else {
			p.nonSpaced(x.Op)
			p.contiguous = false
			p.nonSpaced(x.X)
		}
	case BinaryExpr:
		if p.compactArithm {
			p.nonSpaced(x.X, x.Op, x.Y)
		} else {
			p.spaced(x.X, x.Op, x.Y)
		}
	case FuncDecl:
		if x.BashStyle {
			p.spaced(FUNCTION)
		}
		p.spaced(x.Name)
		if !x.BashStyle {
			p.nonSpaced(LPAREN, RPAREN)
		}
		p.spaced(x.Body)
	case Word:
		for _, n := range x.Parts {
			p.nonSpaced(n)
		}
	case Lit:
		p.nonSpaced(x.Value)
	case SglQuoted:
		p.nonSpaced(SQUOTE, x.Value, SQUOTE)
	case Quoted:
		p.nonSpaced(x.Quote)
		for _, n := range x.Parts {
			p.nonSpaced(n)
		}
		p.nonSpaced(quotedStop(x.Quote))
	case CmdSubst:
		if x.Backquotes {
			p.nonSpaced(BQUOTE)
		} else {
			p.nonSpaced(DOLLPR)
		}
		p.stmtJoin(x.Stmts)
		if x.Backquotes {
			p.nonSpaced(BQUOTE)
		} else {
			p.nonSpaced(RPAREN)
		}
	case ParamExp:
		if x.Short {
			p.nonSpaced(DOLLAR, x.Param)
			return
		}
		p.nonSpaced(DOLLBR)
		if x.Length {
			p.nonSpaced(HASH)
		}
		p.nonSpaced(x.Param)
		if x.Ind != nil {
			p.nonSpaced(LBRACK, x.Ind.Word, RBRACK)
		}
		if x.Repl != nil {
			if x.Repl.All {
				p.nonSpaced(QUO)
			}
			p.nonSpaced(QUO, x.Repl.Orig, QUO, x.Repl.With)
		} else if x.Exp != nil {
			p.nonSpaced(x.Exp.Op, x.Exp.Word)
		}
		p.nonSpaced(RBRACE)
	case ArithmExpr:
		p.nonSpaced(DOLLDP, x.X, DRPAREN)
	case ParenExpr:
		p.nonSpaced(LPAREN)
		oldCompact := p.compactArithm
		p.compactArithm = false
		p.nonSpaced(x.X)
		p.compactArithm = oldCompact
		p.nonSpaced(RPAREN)
	case CaseStmt:
		p.spaced(CASE, x.Word, IN)
		for _, pl := range x.List {
			p.separate(wordFirstPos(pl.Patterns), false, true)
			for i, w := range pl.Patterns {
				if i > 0 {
					p.spaced(OR)
				}
				p.spaced(w)
			}
			p.nonSpaced(RPAREN)
			p.stmtJoin(pl.Stmts)
			p.level++
			p.sepNewline(DSEMICOLON, pl.Dsemi)
			p.level--
		}
		if len(x.List) == 0 {
			p.sepSemicolon(ESAC, x.Esac)
		} else {
			p.sepNewline(ESAC, x.Esac)
		}
	case DeclStmt:
		if x.Local {
			p.spaced(LOCAL)
		} else {
			p.spaced(DECLARE)
		}
		for _, w := range x.Opts {
			p.spaced(w)
		}
		for _, a := range x.Assigns {
			p.spaced(a)
		}
	case ArrayExpr:
		p.nonSpaced(LPAREN)
		p.wordJoin(x.List, false)
		p.nonSpaced(RPAREN)
	case CmdInput:
		// avoid conflict with <<
		p.spaced(CMDIN)
		p.stmtJoin(x.Stmts)
		p.nonSpaced(RPAREN)
	case EvalStmt:
		p.spaced(EVAL, x.Stmt)
	case LetStmt:
		p.spaced(LET)
		p.compactArithm = true
		for _, n := range x.Exprs {
			p.spaced(n)
		}
		p.compactArithm = false
	}
}

func (p *printer) wordJoin(ws []Word, keepNewlines bool) {
	anyNewline := false
	for _, w := range ws {
		if keepNewlines && w.Pos().Line > p.curLine {
			p.spaced("\\\n")
			if !anyNewline {
				p.level++
				anyNewline = true
			}
			p.indent()
		}
		p.spaced(w)
	}
	if anyNewline {
		p.level--
	}
}

func (p *printer) progStmts(stmts []Stmt) {
	for i, s := range stmts {
		p.separate(s.Pos(), i > 0, true)
		p.node(s)
	}
}

func (p *printer) stmtJoin(stmts []Stmt) {
	p.level++
	for i, s := range stmts {
		p.separate(s.Pos(), i > 0, i > 0)
		p.node(s)
	}
	p.level--
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"

)

func main() {

jason :=[]byte (`
 {
    "streams": [
        {
            "index": 0,
            "codec_name": "h264",
            "codec_long_name": "H.264 / AVC / MPEG-4 AVC / MPEG-4 part 10",
            "profile": "Main",
            "codec_type": "video",
            "codec_time_base": "1001/60000",
            "codec_tag_string": "[27][0][0][0]",
            "codec_tag": "0x001b",
            "width": 1280,
            "height": 720,
            "coded_width": 1280,
            "coded_height": 720,
            "has_b_frames": 1,
            "sample_aspect_ratio": "1:1",
            "display_aspect_ratio": "16:9",
            "pix_fmt": "yuv420p",
            "level": 31,
            "chroma_location": "left",
            "field_order": "progressive",
            "refs": 1,
            "is_avc": "false",
            "nal_length_size": "0",
            "id": "0x1e1",
            "r_frame_rate": "30000/1001",
            "avg_frame_rate": "30000/1001",
            "time_base": "1/90000",
            "start_pts": 183003,
            "start_time": "2.033367",
            "bits_per_raw_sample": "8",
            "disposition": {
                "default": 0,
                "dub": 0,
                "original": 0,
                "comment": 0,
                "lyrics": 0,
                "karaoke": 0,
                "forced": 0,
                "hearing_impaired": 0,
                "visual_impaired": 0,
                "clean_effects": 0,
                "attached_pic": 0,
                "timed_thumbnails": 0
            }
        },
        {
            "index": 1,
            "codec_name": "aac",
            "codec_long_name": "AAC (Advanced Audio Coding)",
            "profile": "LC",
            "codec_type": "audio",
            "codec_time_base": "1/48000",
            "codec_tag_string": "[15][0][0][0]",
            "codec_tag": "0x000f",
            "sample_fmt": "fltp",
            "sample_rate": "48000",
            "channels": 2,
            "channel_layout": "stereo",
            "bits_per_sample": 0,
            "id": "0x1e2",
            "r_frame_rate": "0/0",
            "avg_frame_rate": "0/0",
            "time_base": "1/90000",
            "start_pts": 181083,
            "start_time": "2.012033",
            "duration_ts": 154512000,
            "duration": "1716.800000",
            "bit_rate": "256875",
            "disposition": {
                "default": 0,
                "dub": 0,
                "original": 0,
                "comment": 0,
                "lyrics": 0,
                "karaoke": 0,
                "forced": 0,
                "hearing_impaired": 0,
                "visual_impaired": 0,
                "clean_effects": 0,
                "attached_pic": 0,
                "timed_thumbnails": 0
            },
            "tags": {
                "language": "eng"
            }
        },
        {
            "index": 2,
            "codec_name": "scte_35",
            "codec_long_name": "SCTE 35 Message Queue",
            "codec_type": "data",
            "codec_tag_string": "[0][0][0][0]",
            "codec_tag": "0x0000",
            "id": "0x1f4",
            "r_frame_rate": "0/0",
            "avg_frame_rate": "0/0",
            "time_base": "1/90000",
            "start_pts": 181083,
            "start_time": "2.012033",
            "duration_ts": 154512000,
            "duration": "1716.800000",
            "disposition": {
                "default": 0,
                "dub": 0,
                "original": 0,
                "comment": 0,
                "lyrics": 0,
                "karaoke": 0,
                "forced": 0,
                "hearing_impaired": 0,
                "visual_impaired": 0,
                "clean_effects": 0,
                "attached_pic": 0,
                "timed_thumbnails": 0
            }
        },
        {
            "index": 3,
            "codec_name": "timed_id3",
            "codec_long_name": "timed ID3 metadata",
            "codec_type": "data",
            "codec_tag_string": "ID3 ",
            "codec_tag": "0x20334449",
            "id": "0x1f6",
            "r_frame_rate": "0/0",
            "avg_frame_rate": "0/0",
            "time_base": "1/90000",
            "start_pts": 183003,
            "start_time": "2.033367",
            "disposition": {
                "default": 0,
                "dub": 0,
                "original": 0,
                "comment": 0,
                "lyrics": 0,
                "karaoke": 0,
                "forced": 0,
                "hearing_impaired": 0,
                "visual_impaired": 0,
                "clean_effects": 0,
                "attached_pic": 0,
                "timed_thumbnails": 0
            }
        }
    ],
    "format": {
        "filename": "/home/a/bitstream/examples/scte35.ts",
        "nb_streams": 4,
        "nb_programs": 1,
        "format_name": "mpegts",
        "format_long_name": "MPEG-TS (MPEG-2 Transport Stream)",
        "start_time": "2.012033",
        "duration": "1716.800000",
        "size": "940732448",
        "bit_rate": "4383655",
        "probe_score": 50
    }
}
`)

type Format struct{
	FormatName 	string	`json:"format_name"`
	Duration	string	`json:"duration"`
	BitRate		string	`json:"bit_rate"`
}

type Stream struct {
CodecType 	string 	`json:"codec_type"`
CodecName	string 	`json:"codec_name"`
Profile 	string	`json:"profile"`
Level		float64	`json:"level"`
Width		float64	`json:"width"`
Height		float64	`json:"height"`

}

type Container struct {
Streams	[]Stream	`json:"streams"`
Format	Format		`json:"format"`
}

var f Container
json.Unmarshal(jason, &f)

fmt.Println(f.Format)
for _,i := range f.Streams{
	if i.CodecType=="video" {
	fmt.Println(i)

	fmt.Printf("%x",int(i.Level))
	}
}
}

<commit_msg>stanza struct<commit_after>package main

import (
	"encoding/json"
	"fmt"

)

func main() {

jason :=[]byte (`
 {
    "streams": [
        {
            "index": 0,
            "codec_name": "h264",
            "codec_long_name": "H.264 / AVC / MPEG-4 AVC / MPEG-4 part 10",
            "profile": "Main",
            "codec_type": "video",
            "codec_time_base": "1001/60000",
            "codec_tag_string": "[27][0][0][0]",
            "codec_tag": "0x001b",
            "width": 1280,
            "height": 720,
            "coded_width": 1280,
            "coded_height": 720,
            "has_b_frames": 1,
            "sample_aspect_ratio": "1:1",
            "display_aspect_ratio": "16:9",
            "pix_fmt": "yuv420p",
            "level": 31,
            "chroma_location": "left",
            "field_order": "progressive",
            "refs": 1,
            "is_avc": "false",
            "nal_length_size": "0",
            "id": "0x1e1",
            "r_frame_rate": "30000/1001",
            "avg_frame_rate": "30000/1001",
            "time_base": "1/90000",
            "start_pts": 183003,
            "start_time": "2.033367",
            "bits_per_raw_sample": "8",
            "disposition": {
                "default": 0,
                "dub": 0,
                "original": 0,
                "comment": 0,
                "lyrics": 0,
                "karaoke": 0,
                "forced": 0,
                "hearing_impaired": 0,
                "visual_impaired": 0,
                "clean_effects": 0,
                "attached_pic": 0,
                "timed_thumbnails": 0
            }
        },
        {
            "index": 1,
            "codec_name": "aac",
            "codec_long_name": "AAC (Advanced Audio Coding)",
            "profile": "LC",
            "codec_type": "audio",
            "codec_time_base": "1/48000",
            "codec_tag_string": "[15][0][0][0]",
            "codec_tag": "0x000f",
            "sample_fmt": "fltp",
            "sample_rate": "48000",
            "channels": 2,
            "channel_layout": "stereo",
            "bits_per_sample": 0,
            "id": "0x1e2",
            "r_frame_rate": "0/0",
            "avg_frame_rate": "0/0",
            "time_base": "1/90000",
            "start_pts": 181083,
            "start_time": "2.012033",
            "duration_ts": 154512000,
            "duration": "1716.800000",
            "bit_rate": "256875",
            "disposition": {
                "default": 0,
                "dub": 0,
                "original": 0,
                "comment": 0,
                "lyrics": 0,
                "karaoke": 0,
                "forced": 0,
                "hearing_impaired": 0,
                "visual_impaired": 0,
                "clean_effects": 0,
                "attached_pic": 0,
                "timed_thumbnails": 0
            },
            "tags": {
                "language": "eng"
            }
        },
        {
            "index": 2,
            "codec_name": "scte_35",
            "codec_long_name": "SCTE 35 Message Queue",
            "codec_type": "data",
            "codec_tag_string": "[0][0][0][0]",
            "codec_tag": "0x0000",
            "id": "0x1f4",
            "r_frame_rate": "0/0",
            "avg_frame_rate": "0/0",
            "time_base": "1/90000",
            "start_pts": 181083,
            "start_time": "2.012033",
            "duration_ts": 154512000,
            "duration": "1716.800000",
            "disposition": {
                "default": 0,
                "dub": 0,
                "original": 0,
                "comment": 0,
                "lyrics": 0,
                "karaoke": 0,
                "forced": 0,
                "hearing_impaired": 0,
                "visual_impaired": 0,
                "clean_effects": 0,
                "attached_pic": 0,
                "timed_thumbnails": 0
            }
        },
        {
            "index": 3,
            "codec_name": "timed_id3",
            "codec_long_name": "timed ID3 metadata",
            "codec_type": "data",
            "codec_tag_string": "ID3 ",
            "codec_tag": "0x20334449",
            "id": "0x1f6",
            "r_frame_rate": "0/0",
            "avg_frame_rate": "0/0",
            "time_base": "1/90000",
            "start_pts": 183003,
            "start_time": "2.033367",
            "disposition": {
                "default": 0,
                "dub": 0,
                "original": 0,
                "comment": 0,
                "lyrics": 0,
                "karaoke": 0,
                "forced": 0,
                "hearing_impaired": 0,
                "visual_impaired": 0,
                "clean_effects": 0,
                "attached_pic": 0,
                "timed_thumbnails": 0
            }
        }
    ],
    "format": {
        "filename": "/home/a/bitstream/examples/scte35.ts",
        "nb_streams": 4,
        "nb_programs": 1,
        "format_name": "mpegts",
        "format_long_name": "MPEG-TS (MPEG-2 Transport Stream)",
        "start_time": "2.012033",
        "duration": "1716.800000",
        "size": "940732448",
        "bit_rate": "4383655",
        "probe_score": 50
    }
}
`)

type Format struct{
	FormatName 	string	`json:"format_name"`
	Duration	string	`json:"duration"`
	BitRate		string	`json:"bit_rate"`
}

type Stream struct {
CodecType 	string 	`json:"codec_type"`
CodecName	string 	`json:"codec_name"`
Profile 	string	`json:"profile"`
Level		float64	`json:"level"`
Width		float64	`json:"width"`
Height		float64	`json:"height"`

}

type Container struct {
Streams	[]Stream	`json:"streams"`
Format	Format		`json:"format"`
}

type Stanza struct {
Bandwidth	string
Resolution	string
Level		float64
Profile		string
Acodec		string
}

var st Stanza
var f Container
json.Unmarshal(jason, &f)

st.Bandwidth=f.Format.BitRate
for _,i := range f.Streams{

	if i.CodecType=="video" {
	st.Resolution= fmt.Sprintf("=%vx%v",i.Width,i.Height)
	if i.Profile=="High"{
		st.Profile="64"
	}
	if i.Profile =="Main"{
		st.Profile="4d"
	}
	if i.Profile =="Baseline"{
		st.Profile="42"
	}
	st.Level=i.Level

	}

}
fmt.Printf("#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=%v,RESOLUTION=%s,CODECS=\"avc1.%v00%x,mp4a.40.2\"\n",st.Bandwidth,st.Resolution,st.Profile,int(st.Level))

}

<|endoftext|>


<commit_before>// Copyright © 2017 Pennock Tech, LLC.
// All rights reserved, except as granted under license.
// Licensed per file LICENSE.txt

// +build go1.8

package main

import (
	"crypto/tls"
	"fmt"
	"net"
	"net/smtp"
	"net/textproto"
	"strconv"
	"strings"
	"time"

	"golang.org/x/crypto/ocsp"

	"go.pennock.tech/smtpdane/internal/errorlist"
)

type validationContext struct {
	tlsaSet  *TLSAset
	hostname string
	altNames []string
	ip       net.IP
	port     int
	status   *programStatus
	time     time.Time
}

func (vc *validationContext) Messagef(spec string, params ...interface{}) {
	vc.status.Message(fmt.Sprintf("[%s %v] ", vc.hostname, vc.ip) + fmt.Sprintf(spec, params...))
}

func (vc *validationContext) Errorf(spec string, params ...interface{}) {
	vc.Messagef(ColorRed(spec), params...)
	vc.status.AddErr()
}

func (vc *validationContext) Successf(spec string, params ...interface{}) {
	vc.Messagef(ColorGreen(spec), params...)
}

// probeHost is the top-level function of a go-routine and is responsible for
// probing one remote SMTP connection.
//
// Messages should be reported via the Output function of the status; newlines
// are appended and each string is guaranteed to be emitted with no
// interweaving of other results within the string.
func probeHost(hostSpec string, status *programStatus, otherValidNames ...string) {
	defer status.probing.Done()

	hostname, port, err := HostnamePortFrom(hostSpec)
	if err != nil {
		status.Errorf("error parsing %q: %s", hostSpec, err)
		return
	}

	ipList, resolvedHostname, err := ResolveAddrSecure(hostname)
	if err != nil {
		switch e := err.(type) {
		case *errorlist.List:
			status.Errorf("error resolving %q:\n%s", hostname, e.FmtIndented())
		default:
			status.Errorf("error resolving %q: %s", hostname, err)
		}
		return
	}

	if resolvedHostname == hostname {
		status.Messagef("found %d addresses for %q: %v", len(ipList), hostname, ipList)
	} else {
		if opts.mxLookup {
			// Being generous by not just deeming this an error; still, mark it red
			status.Messagef(ColorRed("VIOLATION: MX hostname is a CNAME: %q -> %q"), hostname, resolvedHostname)
		}
		status.Messagef("found %d addresses for %q at %q: %v", len(ipList), hostname, resolvedHostname, ipList)
	}

	// RFC 7671 section 7: chase CNAMEs (as long as secure) of Base Domain and
	// try for TLSA there first, but then fall back to the original name if not
	// found.  Only the final name and original name should be tried, not any
	// intermediate CNAMEs if they were chained.
	//
	// MX hostnames are not supposed to be CNAMEs so this _shouldn't_ crop up.
	// But if it does, handle it.

	tlsaSet, err := ResolveTLSA(resolvedHostname, port)
	if err != nil {
		switch e := err.(type) {
		case *errorlist.List:
			status.Errorf("error resolving TLSA for %q port %d:\n%s", resolvedHostname, port, e.FmtIndented())
		default:
			status.Errorf("error resolving TLSA for %q port %d: %v", resolvedHostname, port, err)
		}

		tlsaSet, err = ResolveTLSA(hostname, port)
		if err != nil {
			switch e := err.(type) {
			case *errorlist.List:
				status.Errorf("error resolving TLSA for %q port %d:\n%s", hostname, port, e.FmtIndented())
			default:
				status.Errorf("error resolving TLSA for %q port %d: %v", hostname, port, err)
			}
			return
		}
	}

	tlsaLines := make([]string, 1+len(tlsaSet.RRs))
	if tlsaSet.name == tlsaSet.foundName {
		tlsaLines[0] = fmt.Sprintf("found %d TLSA records for %q", len(tlsaSet.RRs), tlsaSet.name)
	} else {
		tlsaLines[0] = fmt.Sprintf("found %d TLSA records for %q at %q", len(tlsaSet.RRs), tlsaSet.name, tlsaSet.foundName)
	}
	// sort, or leave as-is showing round-robin results order?
	for i := range tlsaSet.RRs {
		name, ok := KnownCAs.NameForTLSA(tlsaSet.RRs[i])
		if ok {
			tlsaLines[i+1] = TLSAMediumString(tlsaSet.RRs[i]) + " ; " + name
		} else {
			tlsaLines[i+1] = TLSAMediumString(tlsaSet.RRs[i])
		}
	}
	status.Message(strings.Join(tlsaLines, "\n  "))

	var altNames []string = nil
	if len(otherValidNames) > 0 || len(opts.akaNames) > 0 {
		altNames = make([]string, 0, len(otherValidNames)+len(opts.akaNames))
		altNames = append(altNames, otherValidNames...)
		altNames = append(altNames, opts.akaNames...)
	}

	for _, ip := range ipList {
		if opts.onlyIPv4 && ip.To4() == nil {
			continue
		}
		if opts.onlyIPv6 && ip.To4() != nil {
			continue
		}
		status.probing.Add(1)
		go (&validationContext{
			tlsaSet:  tlsaSet,
			hostname: hostname,
			altNames: altNames,
			ip:       ip,
			port:     port,
			status:   status,
			time:     time.Now(),
		}).probeAddr()
	}
}

func (vc *validationContext) probeAddr() {
	defer vc.status.probing.Done()

	// DialTCP takes the vc.ip/vc.port sensibly, but the moment we want timeout
	// control, we need to go through a function which wants us to join them
	// back into a string first (and so risks the library using DNS).
	//
	// If we think there's a serious risk of that, when given input which looks
	// like IPs, we can now provide a Resolver which fails for hostnames.
	// Alternatively, we could use our own timeout logic, but doing that cleanly
	// requires providing the cancel channel, which is now a deprecated interface.
	// So we can do things "simple but deprecated" or "jumping through many hoops"
	// because the sane way is being hidden away behind too much abstraction.
	raddr := net.JoinHostPort(vc.ip.String(), strconv.Itoa(vc.port))

	conn, err := net.DialTimeout("tcp", raddr, opts.connectTimeout)
	if err != nil {
		vc.status.Errorf("dial failed: %s", err)
		return
	}

	// split out into a separate function which can be invoked by testing
	// utilities on a pre-established connection.
	vc.probeConnectedAddr(conn)
}

func (vc *validationContext) probeConnectedAddr(conn net.Conn) {
	verifier, chCertDetails := peerCertificateVerifierFor(vc)
	tlsConfig := &tls.Config{
		ServerName:            vc.hostname,
		InsecureSkipVerify:    true, // we verify ourselves in the VerifyPeerCertificate
		VerifyPeerCertificate: verifier,
	}

	if opts.tlsOnConnect {
		vc.tryTLSOnConn(conn, tlsConfig, chCertDetails)
		return
	}

	s, err := smtp.NewClient(conn, vc.hostname)
	if err != nil {
		vc.Errorf("failed to establish SMTP client on connection: %s", err)
		_ = conn.Close()
		return
	}

	// TODO: figure out a sane timeout mechanism (which also handles pre-banner
	// delays) or some other mechanism to handle Golang net/smtp just hanging
	// when given a TLS-on-connect server (which is reasonable, since for TLS,
	// client-speaks-first and the SMTP code is just waiting for the server to
	// speak).
	err = s.Hello(opts.heloName)
	if err != nil {
		vc.Errorf("EHLO failed: %s", err)
		s.Close()
		return
	}

	ok, _ := s.Extension("STARTTLS")
	if !ok {
		vc.Errorf("server does not advertise STARTTLS")
		s.Close()
		return
	}

	vc.Messagef("issuing STARTTLS")
	err = s.StartTLS(tlsConfig)
	if err != nil {
		vc.Errorf("STARTTLS failed: %s", err)
	}
	if tlsState, ok := s.TLSConnectionState(); ok {
		vc.checkCertInfo(tlsState, chCertDetails)
	}
	err = s.Quit()
	if err != nil {
		vc.Errorf("QUIT failed: %s", err)
	}
	return
}

func (vc *validationContext) tryTLSOnConn(conn net.Conn, tlsConfig *tls.Config, chCertDetails <-chan certDetails) {
	vc.Messagef("starting TLS immediately")
	c := tls.Client(conn, tlsConfig)
	t := textproto.NewConn(c)

	_, _, err := t.ReadResponse(220)
	if err != nil {
		t.Close()
		vc.Errorf("banner read failed: %s", err)
		return
	}

	vc.checkCertInfo(c.ConnectionState(), chCertDetails)

	id, err := t.Cmd("EHLO %s", vc.hostname)
	t.StartResponse(id)
	_, _, err = t.ReadResponse(250)
	t.EndResponse(id)
	if err != nil {
		vc.Errorf("EHLO failed: %s", err)
	}

	id, err = t.Cmd("QUIT")
	t.StartResponse(id)
	_, _, err = t.ReadResponse(221)
	t.EndResponse(id)
	if err != nil {
		vc.Errorf("QUIT failed: %s", err)
	}

	// When speaking to OpenSSL servers, we shut down cleanly without grabbing
	// the EOF first, but when speaking to Golang TLS, that fails us.
	_, err = t.ReadLine()

	t.Close()
}

func (vc *validationContext) checkCertInfo(cs tls.ConnectionState, chCertDetails <-chan certDetails) {
	if !opts.showCertInfo && !opts.expectOCSP {
		return
	}
	haveOCSP := cs.OCSPResponse != nil && len(cs.OCSPResponse) > 0

	if opts.showCertInfo {
		vc.Messagef("TLS session: version=%04x ciphersuite=%04x ocsp=%v", cs.Version, cs.CipherSuite, haveOCSP)
	}

	if !haveOCSP {
		if opts.expectOCSP {
			vc.Errorf("missing OCSP response")
		}
		return
	}
	count := 0
	for cd := range chCertDetails {
		count += 1
		if cd.validChain == nil || len(cd.validChain) < 1 {
			vc.Messagef("  OCSP: not validating for chainless %s", strconv.QuoteToGraphic(cd.eeCert.Subject.CommonName))
			continue
		}
		r, err := ocsp.ParseResponseForCert(cs.OCSPResponse, cd.eeCert, cd.validChain[0])
		if err != nil {
			vc.Errorf("  OCSP: response invalid for %s from %s:\n        %s",
				cd.eeCert.Subject.CommonName,
				cd.validChain[0].Subject.CommonName,
				err)
			continue
		}

		switch st := ocsp.ResponseStatus(r.Status); st {
		case ocsp.Success:
			tmpl := "OCSP: status=%s sn=%v producedAt=(%s) thisUpdate=(%s) nextUpdate=(%s)"
			if opts.showCertInfo {
				tmpl = "  " + tmpl
			}
			if opts.expectOCSP {
				tmpl = ColorGreen(tmpl)
			}
			vc.Messagef(tmpl, st, r.SerialNumber, r.ProducedAt, r.ThisUpdate, r.NextUpdate)
		case ocsp.TryLater:
			vc.Messagef("  OCSP: status=%s", st)
		default:
			vc.Errorf("  OCSP: status=%s RevokedAt=(%s)", st, r.RevokedAt)
		}
	}
	if count == 0 {
		vc.Errorf("Saw OCSP response but got no chain information out of validation")
	}
}
<commit_msg>Pull in more hash algorithms for TLS (SHA2/512)<commit_after>// Copyright © 2017 Pennock Tech, LLC.
// All rights reserved, except as granted under license.
// Licensed per file LICENSE.txt

// +build go1.8

package main

import (
	"crypto/tls"
	"fmt"
	"net"
	"net/smtp"
	"net/textproto"
	"strconv"
	"strings"
	"time"

	// Hash algorithms to be available for validation; any not in stdlib
	// should be optional and not here, but in a build-tag-constrainted file
	// which just does the import so that crypto.RegisterHash() is called.
	// Those needed for TLS:
	//   <https://www.iana.org/assignments/tls-parameters/tls-parameters.xhtml#tls-parameters-18>
	// (so no others pulled in yet)
	_ "crypto/sha256"
	_ "crypto/sha512"

	"golang.org/x/crypto/ocsp"

	"go.pennock.tech/smtpdane/internal/errorlist"
)

type validationContext struct {
	tlsaSet  *TLSAset
	hostname string
	altNames []string
	ip       net.IP
	port     int
	status   *programStatus
	time     time.Time
}

func (vc *validationContext) Messagef(spec string, params ...interface{}) {
	vc.status.Message(fmt.Sprintf("[%s %v] ", vc.hostname, vc.ip) + fmt.Sprintf(spec, params...))
}

func (vc *validationContext) Errorf(spec string, params ...interface{}) {
	vc.Messagef(ColorRed(spec), params...)
	vc.status.AddErr()
}

func (vc *validationContext) Successf(spec string, params ...interface{}) {
	vc.Messagef(ColorGreen(spec), params...)
}

// probeHost is the top-level function of a go-routine and is responsible for
// probing one remote SMTP connection.
//
// Messages should be reported via the Output function of the status; newlines
// are appended and each string is guaranteed to be emitted with no
// interweaving of other results within the string.
func probeHost(hostSpec string, status *programStatus, otherValidNames ...string) {
	defer status.probing.Done()

	hostname, port, err := HostnamePortFrom(hostSpec)
	if err != nil {
		status.Errorf("error parsing %q: %s", hostSpec, err)
		return
	}

	ipList, resolvedHostname, err := ResolveAddrSecure(hostname)
	if err != nil {
		switch e := err.(type) {
		case *errorlist.List:
			status.Errorf("error resolving %q:\n%s", hostname, e.FmtIndented())
		default:
			status.Errorf("error resolving %q: %s", hostname, err)
		}
		return
	}

	if resolvedHostname == hostname {
		status.Messagef("found %d addresses for %q: %v", len(ipList), hostname, ipList)
	} else {
		if opts.mxLookup {
			// Being generous by not just deeming this an error; still, mark it red
			status.Messagef(ColorRed("VIOLATION: MX hostname is a CNAME: %q -> %q"), hostname, resolvedHostname)
		}
		status.Messagef("found %d addresses for %q at %q: %v", len(ipList), hostname, resolvedHostname, ipList)
	}

	// RFC 7671 section 7: chase CNAMEs (as long as secure) of Base Domain and
	// try for TLSA there first, but then fall back to the original name if not
	// found.  Only the final name and original name should be tried, not any
	// intermediate CNAMEs if they were chained.
	//
	// MX hostnames are not supposed to be CNAMEs so this _shouldn't_ crop up.
	// But if it does, handle it.

	tlsaSet, err := ResolveTLSA(resolvedHostname, port)
	if err != nil {
		switch e := err.(type) {
		case *errorlist.List:
			status.Errorf("error resolving TLSA for %q port %d:\n%s", resolvedHostname, port, e.FmtIndented())
		default:
			status.Errorf("error resolving TLSA for %q port %d: %v", resolvedHostname, port, err)
		}

		tlsaSet, err = ResolveTLSA(hostname, port)
		if err != nil {
			switch e := err.(type) {
			case *errorlist.List:
				status.Errorf("error resolving TLSA for %q port %d:\n%s", hostname, port, e.FmtIndented())
			default:
				status.Errorf("error resolving TLSA for %q port %d: %v", hostname, port, err)
			}
			return
		}
	}

	tlsaLines := make([]string, 1+len(tlsaSet.RRs))
	if tlsaSet.name == tlsaSet.foundName {
		tlsaLines[0] = fmt.Sprintf("found %d TLSA records for %q", len(tlsaSet.RRs), tlsaSet.name)
	} else {
		tlsaLines[0] = fmt.Sprintf("found %d TLSA records for %q at %q", len(tlsaSet.RRs), tlsaSet.name, tlsaSet.foundName)
	}
	// sort, or leave as-is showing round-robin results order?
	for i := range tlsaSet.RRs {
		name, ok := KnownCAs.NameForTLSA(tlsaSet.RRs[i])
		if ok {
			tlsaLines[i+1] = TLSAMediumString(tlsaSet.RRs[i]) + " ; " + name
		} else {
			tlsaLines[i+1] = TLSAMediumString(tlsaSet.RRs[i])
		}
	}
	status.Message(strings.Join(tlsaLines, "\n  "))

	var altNames []string = nil
	if len(otherValidNames) > 0 || len(opts.akaNames) > 0 {
		altNames = make([]string, 0, len(otherValidNames)+len(opts.akaNames))
		altNames = append(altNames, otherValidNames...)
		altNames = append(altNames, opts.akaNames...)
	}

	for _, ip := range ipList {
		if opts.onlyIPv4 && ip.To4() == nil {
			continue
		}
		if opts.onlyIPv6 && ip.To4() != nil {
			continue
		}
		status.probing.Add(1)
		go (&validationContext{
			tlsaSet:  tlsaSet,
			hostname: hostname,
			altNames: altNames,
			ip:       ip,
			port:     port,
			status:   status,
			time:     time.Now(),
		}).probeAddr()
	}
}

func (vc *validationContext) probeAddr() {
	defer vc.status.probing.Done()

	// DialTCP takes the vc.ip/vc.port sensibly, but the moment we want timeout
	// control, we need to go through a function which wants us to join them
	// back into a string first (and so risks the library using DNS).
	//
	// If we think there's a serious risk of that, when given input which looks
	// like IPs, we can now provide a Resolver which fails for hostnames.
	// Alternatively, we could use our own timeout logic, but doing that cleanly
	// requires providing the cancel channel, which is now a deprecated interface.
	// So we can do things "simple but deprecated" or "jumping through many hoops"
	// because the sane way is being hidden away behind too much abstraction.
	raddr := net.JoinHostPort(vc.ip.String(), strconv.Itoa(vc.port))

	conn, err := net.DialTimeout("tcp", raddr, opts.connectTimeout)
	if err != nil {
		vc.status.Errorf("dial failed: %s", err)
		return
	}

	// split out into a separate function which can be invoked by testing
	// utilities on a pre-established connection.
	vc.probeConnectedAddr(conn)
}

func (vc *validationContext) probeConnectedAddr(conn net.Conn) {
	verifier, chCertDetails := peerCertificateVerifierFor(vc)
	tlsConfig := &tls.Config{
		ServerName:            vc.hostname,
		InsecureSkipVerify:    true, // we verify ourselves in the VerifyPeerCertificate
		VerifyPeerCertificate: verifier,
	}

	if opts.tlsOnConnect {
		vc.tryTLSOnConn(conn, tlsConfig, chCertDetails)
		return
	}

	s, err := smtp.NewClient(conn, vc.hostname)
	if err != nil {
		vc.Errorf("failed to establish SMTP client on connection: %s", err)
		_ = conn.Close()
		return
	}

	// TODO: figure out a sane timeout mechanism (which also handles pre-banner
	// delays) or some other mechanism to handle Golang net/smtp just hanging
	// when given a TLS-on-connect server (which is reasonable, since for TLS,
	// client-speaks-first and the SMTP code is just waiting for the server to
	// speak).
	err = s.Hello(opts.heloName)
	if err != nil {
		vc.Errorf("EHLO failed: %s", err)
		s.Close()
		return
	}

	ok, _ := s.Extension("STARTTLS")
	if !ok {
		vc.Errorf("server does not advertise STARTTLS")
		s.Close()
		return
	}

	vc.Messagef("issuing STARTTLS")
	err = s.StartTLS(tlsConfig)
	if err != nil {
		vc.Errorf("STARTTLS failed: %s", err)
	}
	if tlsState, ok := s.TLSConnectionState(); ok {
		vc.checkCertInfo(tlsState, chCertDetails)
	}
	err = s.Quit()
	if err != nil {
		vc.Errorf("QUIT failed: %s", err)
	}
	return
}

func (vc *validationContext) tryTLSOnConn(conn net.Conn, tlsConfig *tls.Config, chCertDetails <-chan certDetails) {
	vc.Messagef("starting TLS immediately")
	c := tls.Client(conn, tlsConfig)
	t := textproto.NewConn(c)

	_, _, err := t.ReadResponse(220)
	if err != nil {
		t.Close()
		vc.Errorf("banner read failed: %s", err)
		return
	}

	vc.checkCertInfo(c.ConnectionState(), chCertDetails)

	id, err := t.Cmd("EHLO %s", vc.hostname)
	t.StartResponse(id)
	_, _, err = t.ReadResponse(250)
	t.EndResponse(id)
	if err != nil {
		vc.Errorf("EHLO failed: %s", err)
	}

	id, err = t.Cmd("QUIT")
	t.StartResponse(id)
	_, _, err = t.ReadResponse(221)
	t.EndResponse(id)
	if err != nil {
		vc.Errorf("QUIT failed: %s", err)
	}

	// When speaking to OpenSSL servers, we shut down cleanly without grabbing
	// the EOF first, but when speaking to Golang TLS, that fails us.
	_, err = t.ReadLine()

	t.Close()
}

func (vc *validationContext) checkCertInfo(cs tls.ConnectionState, chCertDetails <-chan certDetails) {
	if !opts.showCertInfo && !opts.expectOCSP {
		return
	}
	haveOCSP := cs.OCSPResponse != nil && len(cs.OCSPResponse) > 0

	if opts.showCertInfo {
		vc.Messagef("TLS session: version=%04x ciphersuite=%04x ocsp=%v", cs.Version, cs.CipherSuite, haveOCSP)
	}

	if !haveOCSP {
		if opts.expectOCSP {
			vc.Errorf("missing OCSP response")
		}
		return
	}
	count := 0
	for cd := range chCertDetails {
		count += 1
		if cd.validChain == nil || len(cd.validChain) < 1 {
			vc.Messagef("  OCSP: not validating for chainless %s", strconv.QuoteToGraphic(cd.eeCert.Subject.CommonName))
			continue
		}
		r, err := ocsp.ParseResponseForCert(cs.OCSPResponse, cd.eeCert, cd.validChain[0])
		if err != nil {
			vc.Errorf("  OCSP: response invalid for %s from %s:\n        %s",
				cd.eeCert.Subject.CommonName,
				cd.validChain[0].Subject.CommonName,
				err)
			continue
		}

		switch st := ocsp.ResponseStatus(r.Status); st {
		case ocsp.Success:
			tmpl := "OCSP: status=%s sn=%v producedAt=(%s) thisUpdate=(%s) nextUpdate=(%s)"
			if opts.showCertInfo {
				tmpl = "  " + tmpl
			}
			if opts.expectOCSP {
				tmpl = ColorGreen(tmpl)
			}
			vc.Messagef(tmpl, st, r.SerialNumber, r.ProducedAt, r.ThisUpdate, r.NextUpdate)
		case ocsp.TryLater:
			vc.Messagef("  OCSP: status=%s", st)
		default:
			vc.Errorf("  OCSP: status=%s RevokedAt=(%s)", st, r.RevokedAt)
		}
	}
	if count == 0 {
		vc.Errorf("Saw OCSP response but got no chain information out of validation")
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cli

import (
	"fmt"

	"github.com/golang/protobuf/proto"
	"github.com/maruel/subcommands"

	"go.chromium.org/luci/common/cli"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/flag"

	"go.chromium.org/luci/machine-db/api/crimson/v1"
)

// GetDatacentersCmd is the command to get datacenters.
type GetDatacentersCmd struct {
	subcommands.CommandRunBase
	req crimson.ListDatacentersRequest
}

// Run runs the command to get datacenters.
func (c *GetDatacentersCmd) Run(app subcommands.Application, args []string, env subcommands.Env) int {
	ctx := cli.GetContext(app, c, env)
	client := getClient(ctx)
	resp, err := client.ListDatacenters(ctx, &c.req)
	if err != nil {
		errors.Log(ctx, err)
		return 1
	}
	// TODO(smut): Format this response.
	fmt.Print(proto.MarshalTextString(resp))
	return 0
}

// getDatacentersCmd returns a command to get datacenters.
func getDatacentersCmd() *subcommands.Command {
	return &subcommands.Command{
		UsageLine: "get-dcs [-name <name>]...",
		ShortDesc: "retrieves datacenters",
		LongDesc:  "Retrieves datacenters matching the given names, or all datacenters if names are omitted.",
		CommandRun: func() subcommands.CommandRun {
			cmd := &GetDatacentersCmd{}
			cmd.Flags.Var(flag.StringSlice(&cmd.req.Names), "name", "Name of a datacenter to filter by. Can be specified multiple times.")
			return cmd
		},
	}
}
<commit_msg>[Machine Database] Format get-dcs output into tab-separated columns<commit_after>// Copyright 2017 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cli

import (
	"encoding/csv"
	"os"

	"github.com/maruel/subcommands"

	"go.chromium.org/luci/common/cli"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/flag"

	"go.chromium.org/luci/machine-db/api/crimson/v1"
)

// GetDatacentersCmd is the command to get datacenters.
type GetDatacentersCmd struct {
	subcommands.CommandRunBase
	req     crimson.ListDatacentersRequest
	headers bool
}

// Run runs the command to get datacenters.
func (c *GetDatacentersCmd) Run(app subcommands.Application, args []string, env subcommands.Env) int {
	ctx := cli.GetContext(app, c, env)
	client := getClient(ctx)
	resp, err := client.ListDatacenters(ctx, &c.req)
	if err != nil {
		errors.Log(ctx, err)
		return 1
	}
	if len(resp.Datacenters) > 0 {
		w := csv.NewWriter(os.Stdout)
		w.Comma = '\t'
		defer w.Flush()
		if c.headers {
			w.Write([]string{"Name", "Description"})
		}
		for _, dc := range resp.Datacenters {
			w.Write([]string{dc.Name, dc.Description})
		}
	}
	return 0
}

// getDatacentersCmd returns a command to get datacenters.
func getDatacentersCmd() *subcommands.Command {
	return &subcommands.Command{
		UsageLine: "get-dcs [-name <name>]...",
		ShortDesc: "retrieves datacenters",
		LongDesc:  "Retrieves datacenters matching the given names, or all datacenters if names are omitted.",
		CommandRun: func() subcommands.CommandRun {
			cmd := &GetDatacentersCmd{}
			cmd.Flags.Var(flag.StringSlice(&cmd.req.Names), "name", "Name of a datacenter to filter by. Can be specified multiple times.")
			cmd.Flags.BoolVar(&cmd.headers, "headers", false, "Show column headers.")
			return cmd
		},
	}
}
<|endoftext|>


<commit_before>package mailserver

import (
	"database/sql"
	"fmt"
	"time"

	// Import postgres driver
	_ "github.com/lib/pq"
	"github.com/status-im/migrate/v4"
	"github.com/status-im/migrate/v4/database/postgres"
	bindata "github.com/status-im/migrate/v4/source/go_bindata"

	"github.com/status-im/status-go/mailserver/migrations"

	"github.com/ethereum/go-ethereum/log"
	"github.com/ethereum/go-ethereum/rlp"

	"github.com/status-im/status-go/eth-node/types"
	"github.com/status-im/status-go/whisper/v6"
)

func NewPostgresDB(uri string) (*PostgresDB, error) {
	db, err := sql.Open("postgres", uri)
	if err != nil {
		return nil, err
	}

	instance := &PostgresDB{db: db}
	if err := instance.setup(); err != nil {
		return nil, err
	}

	return instance, nil
}

type PostgresDB struct {
	db *sql.DB
}

type postgresIterator struct {
	*sql.Rows
}

func (i *postgresIterator) DBKey() (*DBKey, error) {
	var value []byte
	var id []byte
	if err := i.Scan(&id, &value); err != nil {
		return nil, err
	}
	return &DBKey{raw: id}, nil
}

func (i *postgresIterator) Error() error {
	return nil
}

func (i *postgresIterator) Release() {
	i.Close()
}

func (i *postgresIterator) GetEnvelope(bloom []byte) ([]byte, error) {
	var value []byte
	var id []byte
	if err := i.Scan(&id, &value); err != nil {
		return nil, err
	}

	return value, nil
}

func (i *PostgresDB) BuildIterator(query CursorQuery) (Iterator, error) {
	var upperLimit []byte
	var stmtString string
	if len(query.cursor) > 0 {
		// If we have a cursor, we don't want to include that envelope in the result set
		upperLimit = query.cursor

		// We disable security checks as we need to use string interpolation
		// for this, but it's converted to 0s and 1s so no injection should be possible
		/* #nosec */
		stmtString = fmt.Sprintf("SELECT id, data FROM envelopes where id >= $1 AND id < $2 AND bloom & b'%s'::bit(512) = bloom ORDER BY ID DESC LIMIT $3", toBitString(query.bloom))
	} else {
		upperLimit = query.end
		// We disable security checks as we need to use string interpolation
		// for this, but it's converted to 0s and 1s so no injection should be possible
		/* #nosec */
		stmtString = fmt.Sprintf("SELECT id, data FROM envelopes where id >= $1 AND id <= $2 AND bloom & b'%s'::bit(512) = bloom ORDER BY ID DESC LIMIT $3", toBitString(query.bloom))
	}

	stmt, err := i.db.Prepare(stmtString)
	if err != nil {
		return nil, err
	}

	rows, err := stmt.Query(query.start, upperLimit, query.limit)

	if err != nil {
		return nil, err
	}

	return &postgresIterator{rows}, nil
}

func (i *PostgresDB) setup() error {
	resources := bindata.Resource(
		migrations.AssetNames(),
		func(name string) ([]byte, error) {
			return migrations.Asset(name)
		},
	)

	source, err := bindata.WithInstance(resources)
	if err != nil {
		return err
	}

	driver, err := postgres.WithInstance(i.db, &postgres.Config{})
	if err != nil {
		return err
	}

	m, err := migrate.NewWithInstance(
		"go-bindata",
		source,
		"postgres",
		driver)
	if err != nil {
		return err
	}

	if err = m.Up(); err != migrate.ErrNoChange {
		return err
	}

	return nil
}

func (i *PostgresDB) Close() error {
	return i.db.Close()
}

func (i *PostgresDB) GetEnvelope(key *DBKey) ([]byte, error) {
	statement := `SELECT data FROM envelopes WHERE id = $1`

	stmt, err := i.db.Prepare(statement)
	if err != nil {
		return nil, err
	}
	defer stmt.Close()

	var envelope []byte

	if err = stmt.QueryRow(key.Bytes()).Scan(&envelope); err != nil {
		return nil, err
	}

	return envelope, nil
}

func (i *PostgresDB) Prune(t time.Time, batch int) (int, error) {
	var zero types.Hash
	var emptyTopic types.TopicType
	kl := NewDBKey(0, emptyTopic, zero)
	ku := NewDBKey(uint32(t.Unix()), emptyTopic, zero)
	statement := "DELETE FROM envelopes WHERE id BETWEEN $1 AND $2"

	stmt, err := i.db.Prepare(statement)
	if err != nil {
		return 0, err
	}
	defer stmt.Close()

	if _, err = stmt.Exec(kl.Bytes(), ku.Bytes()); err != nil {
		return 0, err
	}

	return 0, nil
}

func (i *PostgresDB) SaveEnvelope(env types.Envelope) error {
	topic := env.Topic()
	key := NewDBKey(env.Expiry()-env.TTL(), topic, env.Hash())
	rawEnvelope, err := rlp.EncodeToBytes(env)
	if err != nil {
		log.Error(fmt.Sprintf("rlp.EncodeToBytes failed: %s", err))
		archivedErrorsCounter.Inc()
		return err
	}

	statement := "INSERT INTO envelopes (id, data, topic, bloom) VALUES ($1, $2, $3, B'"
	statement += toBitString(env.Bloom())
	statement += "'::bit(512)) ON CONFLICT (id) DO NOTHING;"
	stmt, err := i.db.Prepare(statement)
	if err != nil {
		return err
	}
	defer stmt.Close()

	_, err = stmt.Exec(
		key.Bytes(),
		rawEnvelope,
		topicToByte(topic),
	)

	if err != nil {
		archivedErrorsCounter.Inc()
		return err
	}

	archivedEnvelopesCounter.Inc()
	archivedEnvelopeSizeMeter.Observe(float64(whisper.EnvelopeHeaderLength + env.Size()))

	return nil
}

func topicToByte(t types.TopicType) []byte {
	return []byte{t[0], t[1], t[2], t[3]}
}

func toBitString(bloom []byte) string {
	val := ""
	for _, n := range bloom {
		val += fmt.Sprintf("%08b", n)
	}
	return val
}
<commit_msg>Report correct number of pruned rows in Postgres mailserver (#1789)<commit_after>package mailserver

import (
	"database/sql"
	"fmt"
	"time"

	// Import postgres driver
	_ "github.com/lib/pq"
	"github.com/status-im/migrate/v4"
	"github.com/status-im/migrate/v4/database/postgres"
	bindata "github.com/status-im/migrate/v4/source/go_bindata"

	"github.com/status-im/status-go/mailserver/migrations"

	"github.com/ethereum/go-ethereum/log"
	"github.com/ethereum/go-ethereum/rlp"

	"github.com/status-im/status-go/eth-node/types"
	"github.com/status-im/status-go/whisper/v6"
)

func NewPostgresDB(uri string) (*PostgresDB, error) {
	db, err := sql.Open("postgres", uri)
	if err != nil {
		return nil, err
	}

	instance := &PostgresDB{db: db}
	if err := instance.setup(); err != nil {
		return nil, err
	}

	return instance, nil
}

type PostgresDB struct {
	db *sql.DB
}

type postgresIterator struct {
	*sql.Rows
}

func (i *postgresIterator) DBKey() (*DBKey, error) {
	var value []byte
	var id []byte
	if err := i.Scan(&id, &value); err != nil {
		return nil, err
	}
	return &DBKey{raw: id}, nil
}

func (i *postgresIterator) Error() error {
	return nil
}

func (i *postgresIterator) Release() {
	i.Close()
}

func (i *postgresIterator) GetEnvelope(bloom []byte) ([]byte, error) {
	var value []byte
	var id []byte
	if err := i.Scan(&id, &value); err != nil {
		return nil, err
	}

	return value, nil
}

func (i *PostgresDB) BuildIterator(query CursorQuery) (Iterator, error) {
	var upperLimit []byte
	var stmtString string
	if len(query.cursor) > 0 {
		// If we have a cursor, we don't want to include that envelope in the result set
		upperLimit = query.cursor

		// We disable security checks as we need to use string interpolation
		// for this, but it's converted to 0s and 1s so no injection should be possible
		/* #nosec */
		stmtString = fmt.Sprintf("SELECT id, data FROM envelopes where id >= $1 AND id < $2 AND bloom & b'%s'::bit(512) = bloom ORDER BY ID DESC LIMIT $3", toBitString(query.bloom))
	} else {
		upperLimit = query.end
		// We disable security checks as we need to use string interpolation
		// for this, but it's converted to 0s and 1s so no injection should be possible
		/* #nosec */
		stmtString = fmt.Sprintf("SELECT id, data FROM envelopes where id >= $1 AND id <= $2 AND bloom & b'%s'::bit(512) = bloom ORDER BY ID DESC LIMIT $3", toBitString(query.bloom))
	}

	stmt, err := i.db.Prepare(stmtString)
	if err != nil {
		return nil, err
	}

	rows, err := stmt.Query(query.start, upperLimit, query.limit)

	if err != nil {
		return nil, err
	}

	return &postgresIterator{rows}, nil
}

func (i *PostgresDB) setup() error {
	resources := bindata.Resource(
		migrations.AssetNames(),
		func(name string) ([]byte, error) {
			return migrations.Asset(name)
		},
	)

	source, err := bindata.WithInstance(resources)
	if err != nil {
		return err
	}

	driver, err := postgres.WithInstance(i.db, &postgres.Config{})
	if err != nil {
		return err
	}

	m, err := migrate.NewWithInstance(
		"go-bindata",
		source,
		"postgres",
		driver)
	if err != nil {
		return err
	}

	if err = m.Up(); err != migrate.ErrNoChange {
		return err
	}

	return nil
}

func (i *PostgresDB) Close() error {
	return i.db.Close()
}

func (i *PostgresDB) GetEnvelope(key *DBKey) ([]byte, error) {
	statement := `SELECT data FROM envelopes WHERE id = $1`

	stmt, err := i.db.Prepare(statement)
	if err != nil {
		return nil, err
	}
	defer stmt.Close()

	var envelope []byte

	if err = stmt.QueryRow(key.Bytes()).Scan(&envelope); err != nil {
		return nil, err
	}

	return envelope, nil
}

func (i *PostgresDB) Prune(t time.Time, batch int) (int, error) {
	var zero types.Hash
	var emptyTopic types.TopicType
	kl := NewDBKey(0, emptyTopic, zero)
	ku := NewDBKey(uint32(t.Unix()), emptyTopic, zero)
	statement := "DELETE FROM envelopes WHERE id BETWEEN $1 AND $2"

	stmt, err := i.db.Prepare(statement)
	if err != nil {
		return 0, err
	}
	defer stmt.Close()

	result, err := stmt.Exec(kl.Bytes(), ku.Bytes())
	if err != nil {
		return 0, err
	}
	rows, err := result.RowsAffected()
	if err != nil {
		return 0, err
	}
	return int(rows), nil
}

func (i *PostgresDB) SaveEnvelope(env types.Envelope) error {
	topic := env.Topic()
	key := NewDBKey(env.Expiry()-env.TTL(), topic, env.Hash())
	rawEnvelope, err := rlp.EncodeToBytes(env)
	if err != nil {
		log.Error(fmt.Sprintf("rlp.EncodeToBytes failed: %s", err))
		archivedErrorsCounter.Inc()
		return err
	}

	statement := "INSERT INTO envelopes (id, data, topic, bloom) VALUES ($1, $2, $3, B'"
	statement += toBitString(env.Bloom())
	statement += "'::bit(512)) ON CONFLICT (id) DO NOTHING;"
	stmt, err := i.db.Prepare(statement)
	if err != nil {
		return err
	}
	defer stmt.Close()

	_, err = stmt.Exec(
		key.Bytes(),
		rawEnvelope,
		topicToByte(topic),
	)

	if err != nil {
		archivedErrorsCounter.Inc()
		return err
	}

	archivedEnvelopesCounter.Inc()
	archivedEnvelopeSizeMeter.Observe(float64(whisper.EnvelopeHeaderLength + env.Size()))

	return nil
}

func topicToByte(t types.TopicType) []byte {
	return []byte{t[0], t[1], t[2], t[3]}
}

func toBitString(bloom []byte) string {
	val := ""
	for _, n := range bloom {
		val += fmt.Sprintf("%08b", n)
	}
	return val
}
<|endoftext|>


<commit_before>package bbs

import "fmt"

//This has all the structs for various commands

type BBSCommand struct {
	Command string `json:"cmd"`
}

type UserCommand struct {
	Command string `json:"cmd"`
	Session string `json:"session"`
}

//From start to end inclusive, starting from 1.
type Range struct {
	Start int `json:"start"`
	End   int `json:"end"`
}

func (r *Range) String() string {
	return fmt.Sprintf("%d-%d", r.Start, r.End)
}

func (r *Range) Validate() bool {
	if r.Start > r.End {
		return false
	}
	return true
}

// "hello" message (server -> client)
type HelloMessage struct {
	Command         string     `json:"cmd"`
	Name            string     `json:"name"`
	ProtocolVersion int        `json:"version"`
	Description     string     `json:"desc"`
	SecureURL       string     `json:"secure,omitempty"` //https URL, if any
	Options         []string   `json:"options,omitempty"`
	Access          AccessInfo `json:"access"`
	Formats         []string   `json:"format"` //formats the server accepts, the first one should be the primary one
	ServerVersion   string     `json:"server"`
}

// guest commands are commands you can use without logging on (e.g. "list", "get")
// user commands require being logged in first (usually "post" and "reply")
type AccessInfo struct {
	GuestCommands []string `json:"guest,omitempty"`
	UserCommands  []string `json:"user,omitempty"`
}

// "error" message (server -> client)
type ErrorMessage struct {
	Command string `json:"cmd"`
	ReplyTo string `json:"wrt"`
	Error   string `json:"error"`
}

// session expired or invalid? use this
var SessionErrorMessage *ErrorMessage = &ErrorMessage{"error", "session", "Invalid session."}

// "ok" message (server -> client)
type OKMessage struct {
	Command string `json:"cmd"`
	ReplyTo string `json:"wrt"`
	Result  string `json:"result,omitempty"`
}

// "login" command (client -> server)
type LoginCommand struct {
	Command         string `json:"cmd"`
	Username        string `json:"username"`
	Password        string `json:"password"`
	ProtocolVersion int    `json:"version"`
}

// "welcome" message (server -> client)
type WelcomeMessage struct {
	Command  string `json:"cmd"`
	Username string `json:"username,omitempty"` //omit for option 'anon'
	Session  string `json:"session"`
}

// "logout" command (client -> server)
type LogoutCommand struct {
	Command string `json:"cmd"`
	Session string `json:"session"`
}

// "get" command (client -> server)
type GetCommand struct {
	Command  string `json:"cmd"`
	Session  string `json:"session,omitempty"`
	ThreadID string `json:"id"`
	Range    *Range `json:"range"`            //option: "range"
	Filter   string `json:"filter,omitempty"` //option: "filter"
	Format   string `json:"format,omitempty"`
}

// "list" command (client -> server)
type ListCommand struct {
	Command string `json:"cmd"`
	Session string `json:"session,omitempty"`
	Type    string `json:"type"`
	Query   string `json:"query"` //board for "boards", tag expression for "tags" (like "Dogs+Pizza-Anime")
}

// "reply" command (client -> server)
type ReplyCommand struct {
	Command string `json:"cmd"`
	Session string `json:"session,omitempty"`
	To      string `json:"to"`
	Text    string `json:"body"`
	Format  string `json:"format,omitempty"`
}

// "post" command (client -> server)
type PostCommand struct {
	Command string   `json:"cmd"`
	Session string   `json:"session,omitempty"`
	Title   string   `json:"title"`
	Text    string   `json:"body"`
	Format  string   `json:"format,omitempty"`
	Board   string   `json:"board,omitempty"` //option: "boards"
	Tags    []string `json:"tags,omitempty"`  //option: "tags"
}

// "msg" message (server -> client) [response to "get"]
type ThreadMessage struct {
	Command  string     `json:"cmd"`
	ID       string     `json:"id"`
	Title    string     `json:"title,omitempty"`
	Range    *Range     `json:"range,omitempty"`
	Closed   bool       `json:"closed,omitempty"`
	Filter   string     `json:"filter,omitempty"` //option: "filter"
	Board    string     `json:"board,omitempty"`  //option: "boards"
	Tags     []string   `json:"tags,omitempty"`   //option: "tags"
	Format   string     `json:"format,omitempty"`
	Messages []*Message `json:"messages"`
	More     bool       `json:"more,omitempty"`
}

func (t *ThreadMessage) Size() int {
	if t.Messages != nil {
		return len(t.Messages)
	}
	return 0
}

// format for posts used in "msg"
type Message struct {
	ID                 string `json:"id"`
	Author             string `json:"user"`
	AuthorID           string `json:"user_id,omitempty"`
	Date               string `json:"date,omitempty"`
	Text               string `json:"body"`
	Signature          string `json:"sig,omitempty"`
	AuthorTitle        string `json:"user_title,omitempty"`   //option: "usertitles"
	AvatarURL          string `json:"avatar,omitempty"`       //option: "avatars"
	AvatarThumbnailURL string `json:"avatar_thumb,omitempty"` //option: "avatars"
	PictureURL         string `json:"img,omitempty"`          //option: "imageboard"
	ThumbnailURL       string `json:"thumb,omitempty"`        //option: "imageboard"
}

type TypedMessage struct {
	Command string `json:"cmd"`
	Type    string `json:"type"`
}

// "list" message where type = "thread" (server -> client)
type ListMessage struct {
	Command string           `json:"cmd"`
	Type    string           `json:"type"`
	Query   string           `json:"query,omitempty"`
	Threads []*ThreadListing `json:"threads"`
}

// "list" message where type = "board" (server -> client)
type BoardListMessage struct {
	Command string          `json:"cmd"`
	Type    string          `json:"type"`
	Query   string          `json:"query,omitempty"`
	Boards  []*BoardListing `json:"boards"`
}

// format for threads in "list"
type ThreadListing struct {
	ID           string   `json:"id"`
	Title        string   `json:"title"`
	Author       string   `json:"user,omitempty"`
	AuthorID     string   `json:"user_id,omitempty"`
	Date         string   `json:"date,omitempty"`
	PostCount    int      `json:"posts,omitempty"`
	UnreadPosts  int      `json:"unread_posts,omitempty"`
	Sticky       bool     `json:"sticky,omitempty"` //a sticky (aka pinned) topic
	Closed       bool     `json:"closed,omitempty"` //a closed (aka locked) topic
	Tags         []string `json:"tags,omitempty"`   //option: "tags"
	PictureURL   string   `json:"img,omitempty"`    //option: "imageboard"
	ThumbnailURL string   `json:"thumb,omitempty"`  //option: "imageboard"
}

// format for boards in "list"
type BoardListing struct {
	ID          string `json:"id"`
	Name        string `json:"name,omitempty"`
	Description string `json:"desc,omitempty"`
	ThreadCount int    `json:"threads,omitempty"`
	PostCount   int    `json:"posts,omitempty"`
	Date        string `json:"date"`
}
<commit_msg>"lists" param in "hello" cmd<commit_after>package bbs

import "fmt"

//This has all the structs for various commands

type BBSCommand struct {
	Command string `json:"cmd"`
}

type UserCommand struct {
	Command string `json:"cmd"`
	Session string `json:"session"`
}

//From start to end inclusive, starting from 1.
type Range struct {
	Start int `json:"start"`
	End   int `json:"end"`
}

func (r *Range) String() string {
	return fmt.Sprintf("%d-%d", r.Start, r.End)
}

func (r *Range) Validate() bool {
	if r.Start > r.End {
		return false
	}
	return true
}

// "hello" message (server -> client)
type HelloMessage struct {
	Command         string     `json:"cmd"`
	Name            string     `json:"name"`
	ProtocolVersion int        `json:"version"`
	Description     string     `json:"desc"`
	SecureURL       string     `json:"secure,omitempty"` //https URL, if any
	Options         []string   `json:"options,omitempty"`
	Access          AccessInfo `json:"access"`
	Formats         []string   `json:"format"` //formats the server accepts, the first one should be the primary one
	Lists           []string   `json:"lists"`
	ServerVersion   string     `json:"server"`
}

// guest commands are commands you can use without logging on (e.g. "list", "get")
// user commands require being logged in first (usually "post" and "reply")
type AccessInfo struct {
	GuestCommands []string `json:"guest,omitempty"`
	UserCommands  []string `json:"user,omitempty"`
}

// "error" message (server -> client)
type ErrorMessage struct {
	Command string `json:"cmd"`
	ReplyTo string `json:"wrt"`
	Error   string `json:"error"`
}

// session expired or invalid? use this
var SessionErrorMessage *ErrorMessage = &ErrorMessage{"error", "session", "Invalid session."}

// "ok" message (server -> client)
type OKMessage struct {
	Command string `json:"cmd"`
	ReplyTo string `json:"wrt"`
	Result  string `json:"result,omitempty"`
}

// "login" command (client -> server)
type LoginCommand struct {
	Command         string `json:"cmd"`
	Username        string `json:"username"`
	Password        string `json:"password"`
	ProtocolVersion int    `json:"version"`
}

// "welcome" message (server -> client)
type WelcomeMessage struct {
	Command  string `json:"cmd"`
	Username string `json:"username,omitempty"` //omit for option 'anon'
	Session  string `json:"session"`
}

// "logout" command (client -> server)
type LogoutCommand struct {
	Command string `json:"cmd"`
	Session string `json:"session"`
}

// "get" command (client -> server)
type GetCommand struct {
	Command  string `json:"cmd"`
	Session  string `json:"session,omitempty"`
	ThreadID string `json:"id"`
	Range    *Range `json:"range"`            //option: "range"
	Filter   string `json:"filter,omitempty"` //option: "filter"
	Format   string `json:"format,omitempty"`
}

// "list" command (client -> server)
type ListCommand struct {
	Command string `json:"cmd"`
	Session string `json:"session,omitempty"`
	Type    string `json:"type"`
	Query   string `json:"query"` //board for "boards", tag expression for "tags" (like "Dogs+Pizza-Anime")
}

// "reply" command (client -> server)
type ReplyCommand struct {
	Command string `json:"cmd"`
	Session string `json:"session,omitempty"`
	To      string `json:"to"`
	Text    string `json:"body"`
	Format  string `json:"format,omitempty"`
}

// "post" command (client -> server)
type PostCommand struct {
	Command string   `json:"cmd"`
	Session string   `json:"session,omitempty"`
	Title   string   `json:"title"`
	Text    string   `json:"body"`
	Format  string   `json:"format,omitempty"`
	Board   string   `json:"board,omitempty"` //option: "boards"
	Tags    []string `json:"tags,omitempty"`  //option: "tags"
}

// "msg" message (server -> client) [response to "get"]
type ThreadMessage struct {
	Command  string     `json:"cmd"`
	ID       string     `json:"id"`
	Title    string     `json:"title,omitempty"`
	Range    *Range     `json:"range,omitempty"`
	Closed   bool       `json:"closed,omitempty"`
	Filter   string     `json:"filter,omitempty"` //option: "filter"
	Board    string     `json:"board,omitempty"`  //option: "boards"
	Tags     []string   `json:"tags,omitempty"`   //option: "tags"
	Format   string     `json:"format,omitempty"`
	Messages []*Message `json:"messages"`
	More     bool       `json:"more,omitempty"`
}

func (t *ThreadMessage) Size() int {
	if t.Messages != nil {
		return len(t.Messages)
	}
	return 0
}

// format for posts used in "msg"
type Message struct {
	ID                 string `json:"id"`
	Author             string `json:"user"`
	AuthorID           string `json:"user_id,omitempty"`
	Date               string `json:"date,omitempty"`
	Text               string `json:"body"`
	Signature          string `json:"sig,omitempty"`
	AuthorTitle        string `json:"user_title,omitempty"`   //option: "usertitles"
	AvatarURL          string `json:"avatar,omitempty"`       //option: "avatars"
	AvatarThumbnailURL string `json:"avatar_thumb,omitempty"` //option: "avatars"
	PictureURL         string `json:"img,omitempty"`          //option: "imageboard"
	ThumbnailURL       string `json:"thumb,omitempty"`        //option: "imageboard"
}

type TypedMessage struct {
	Command string `json:"cmd"`
	Type    string `json:"type"`
}

// "list" message where type = "thread" (server -> client)
type ListMessage struct {
	Command string           `json:"cmd"`
	Type    string           `json:"type"`
	Query   string           `json:"query,omitempty"`
	Threads []*ThreadListing `json:"threads"`
}

// "list" message where type = "board" (server -> client)
type BoardListMessage struct {
	Command string          `json:"cmd"`
	Type    string          `json:"type"`
	Query   string          `json:"query,omitempty"`
	Boards  []*BoardListing `json:"boards"`
}

// format for threads in "list"
type ThreadListing struct {
	ID           string   `json:"id"`
	Title        string   `json:"title"`
	Author       string   `json:"user,omitempty"`
	AuthorID     string   `json:"user_id,omitempty"`
	Date         string   `json:"date,omitempty"`
	PostCount    int      `json:"posts,omitempty"`
	UnreadPosts  int      `json:"unread_posts,omitempty"`
	Sticky       bool     `json:"sticky,omitempty"` //a sticky (aka pinned) topic
	Closed       bool     `json:"closed,omitempty"` //a closed (aka locked) topic
	Tags         []string `json:"tags,omitempty"`   //option: "tags"
	PictureURL   string   `json:"img,omitempty"`    //option: "imageboard"
	ThumbnailURL string   `json:"thumb,omitempty"`  //option: "imageboard"
}

// format for boards in "list"
type BoardListing struct {
	ID          string `json:"id"`
	Name        string `json:"name,omitempty"`
	Description string `json:"desc,omitempty"`
	ThreadCount int    `json:"threads,omitempty"`
	PostCount   int    `json:"posts,omitempty"`
	Date        string `json:"date"`
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"net/http"
	"net/http/httputil"
	"net/url"
	"sync"
	"net"
	"time"

	"github.com/hagen1778/chproxy/config"
	"github.com/hagen1778/chproxy/log"
	"github.com/prometheus/client_golang/prometheus"
)

// Creates new reverseProxy with provided config
func NewReverseProxy(cfg *config.Config) (*reverseProxy, error) {
	rp := &reverseProxy{}
	rp.ReverseProxy = &httputil.ReverseProxy{
		Director: func(*http.Request) {},
		ErrorLog: log.ErrorLogger,
		Transport: &observableTransport{
			http.Transport{
				Proxy: http.ProxyFromEnvironment,
				DialContext: (&net.Dialer{
					KeepAlive: 30 * time.Second,
					DualStack: true,
				}).DialContext,
				MaxIdleConns:          100,
				IdleConnTimeout:       90 * time.Second,
				TLSHandshakeTimeout:   10 * time.Second,
				ExpectContinueTimeout: 1 * time.Second,
			},
		},
	}
	if err := rp.ApplyConfig(cfg); err != nil {
		return nil, err
	}

	return rp, nil
}

// Serves incoming requests according to config
func (rp *reverseProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	log.Debugf("Accepting request: %v", req.Header)
	scope, err := rp.getRequestScope(req)
	if err != nil {
		respondWIthErr(rw, err)
		return
	}
	log.Debugf("Request scope is: %s", scope)

	if err = scope.inc(); err != nil {
		respondWIthErr(rw, err)
		return
	}

	ctx := context.Background()
	if scope.user.maxExecutionTime != 0 {
		var cancel context.CancelFunc
		ctx, cancel = context.WithTimeout(ctx, scope.user.maxExecutionTime)
		defer cancel()
	}
	req = req.WithContext(ctx)

	label := prometheus.Labels{"user": scope.user.name, "target": scope.target.addr.Host}
	requestSum.With(label).Inc()

	rp.ReverseProxy.ServeHTTP(rw, req)

	if req.Context().Err() != nil {
		timeouts.With(label).Inc()
		rp.killQueries(scope.user.name, scope.user.maxExecutionTime.Seconds())
		message := fmt.Sprintf("timeout for user %q exceeded: %v", scope.user.name, scope.user.maxExecutionTime)
		fmt.Fprint(rw, message)
	} else {
		requestSuccess.With(label).Inc()
	}

	scope.dec()
	log.Debugf("Request for scope %s successfully proxied", scope)
}

// Reloads configuration from passed file
// return error if configuration is invalid
func (rp *reverseProxy) ReloadConfig(file string) error {
	cfg, err := config.LoadFile(file)
	if err != nil {
		return fmt.Errorf("can't load config %q: %s", file, err)
	}

	return rp.ApplyConfig(cfg)
}

// Applies provided config to reverseProxy
// New config will be applied only if non-nil error returned
func (rp *reverseProxy) ApplyConfig(cfg *config.Config) error {
	if err := cfg.Validate(); err != nil {
		return err
	}

	rp.Lock()
	defer rp.Unlock()

	targets := make([]*target, len(cfg.Cluster.Shards))
	for i, t := range cfg.Cluster.Shards {
		addr, err := url.Parse(fmt.Sprintf("%s://%s", cfg.Cluster.Scheme, t))
		if err != nil {
			return err
		}

		targets[i] = &target{
			addr: addr,
		}
	}

	users := make(map[string]*user, len(cfg.Users))
	for _, u := range cfg.Users {
		users[u.Name] = &user{
			name:                 u.Name,
			maxConcurrentQueries: u.MaxConcurrentQueries,
			maxExecutionTime:     u.MaxExecutionTime,
		}
	}

	rp.targets = targets
	rp.users = users
	return nil
}

type reverseProxy struct {
	*httputil.ReverseProxy

	sync.Mutex
	users   map[string]*user
	targets []*target
}

func (rp *reverseProxy) getRequestScope(req *http.Request) (*scope, error) {
	user, err := rp.getUser(req)
	if err != nil {
		return nil, err
	}

	target := rp.getTarget()
	req.URL.Scheme = target.addr.Scheme
	req.URL.Host = target.addr.Host

	return &scope{
		user:   user,
		target: target,
	}, nil
}

func (rp *reverseProxy) getUser(req *http.Request) (*user, error) {
	name := extractUserFromRequest(req)

	rp.Lock()
	defer rp.Unlock()

	user, ok := rp.users[name]
	if !ok {
		return nil, fmt.Errorf("unknown username %q", name)
	}

	return user, nil
}

func (rp *reverseProxy) getTarget() *target {
	rp.Lock()
	defer rp.Unlock()

	var idle *target
	for _, t := range rp.targets {
		t.Lock()
		if t.runningQueries == 0 {
			t.Unlock()
			return t
		}

		if idle == nil || idle.runningQueries > t.runningQueries {
			idle = t
		}
		t.Unlock()
	}

	return idle
}

// We don't use query_id because of distributed processing, the query ID is not passed to remote servers
func (rp *reverseProxy) killQueries(user string, elapsed float64) {
	rp.Lock()
	addrs := make([]string, len(rp.targets))
	for i, target := range rp.targets {
		addrs[i] = target.addr.String()
	}
	rp.Unlock()

	q := fmt.Sprintf("KILL QUERY WHERE initial_user = '%s' AND elapsed >= %d", user, int(elapsed))
	for _, addr := range addrs {
		if err := doQuery(q, addr); err != nil {
			log.Errorf("error while killing queries older %.2fs than for user %q: %s", elapsed, user, err)
		}
	}
}

type observableTransport struct {
	http.Transport
}

func (pt *observableTransport) RoundTrip(r *http.Request) (*http.Response, error) {
	response, err := pt.Transport.RoundTrip(r)

	if response != nil {
		statusCodes.With(
			prometheus.Labels{"target": r.Host, "code": response.Status},
		).Inc()
	}

	if err != nil {
		errors.With(
			prometheus.Labels{"target": r.Host, "message": err.Error()},
		).Inc()
	}

	return response, err
}<commit_msg>fix host in metrics<commit_after>package main

import (
	"context"
	"fmt"
	"net/http"
	"net/http/httputil"
	"net/url"
	"sync"
	"net"
	"time"

	"github.com/hagen1778/chproxy/config"
	"github.com/hagen1778/chproxy/log"
	"github.com/prometheus/client_golang/prometheus"
)

// Creates new reverseProxy with provided config
func NewReverseProxy(cfg *config.Config) (*reverseProxy, error) {
	rp := &reverseProxy{}
	rp.ReverseProxy = &httputil.ReverseProxy{
		Director: func(*http.Request) {},
		ErrorLog: log.ErrorLogger,
		Transport: &observableTransport{
			http.Transport{
				Proxy: http.ProxyFromEnvironment,
				DialContext: (&net.Dialer{
					KeepAlive: 30 * time.Second,
					DualStack: true,
				}).DialContext,
				MaxIdleConns:          100,
				IdleConnTimeout:       90 * time.Second,
				TLSHandshakeTimeout:   10 * time.Second,
				ExpectContinueTimeout: 1 * time.Second,
			},
		},
	}
	if err := rp.ApplyConfig(cfg); err != nil {
		return nil, err
	}

	return rp, nil
}

// Serves incoming requests according to config
func (rp *reverseProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	log.Debugf("Accepting request: %v", req.Header)
	scope, err := rp.getRequestScope(req)
	if err != nil {
		respondWIthErr(rw, err)
		return
	}
	log.Debugf("Request scope is: %s", scope)

	if err = scope.inc(); err != nil {
		respondWIthErr(rw, err)
		return
	}

	ctx := context.Background()
	if scope.user.maxExecutionTime != 0 {
		var cancel context.CancelFunc
		ctx, cancel = context.WithTimeout(ctx, scope.user.maxExecutionTime)
		defer cancel()
	}
	req = req.WithContext(ctx)

	label := prometheus.Labels{"user": scope.user.name, "target": scope.target.addr.Host}
	requestSum.With(label).Inc()

	rp.ReverseProxy.ServeHTTP(rw, req)

	if req.Context().Err() != nil {
		timeouts.With(label).Inc()
		rp.killQueries(scope.user.name, scope.user.maxExecutionTime.Seconds())
		message := fmt.Sprintf("timeout for user %q exceeded: %v", scope.user.name, scope.user.maxExecutionTime)
		fmt.Fprint(rw, message)
	} else {
		requestSuccess.With(label).Inc()
	}

	scope.dec()
	log.Debugf("Request for scope %s successfully proxied", scope)
}

// Reloads configuration from passed file
// return error if configuration is invalid
func (rp *reverseProxy) ReloadConfig(file string) error {
	cfg, err := config.LoadFile(file)
	if err != nil {
		return fmt.Errorf("can't load config %q: %s", file, err)
	}

	return rp.ApplyConfig(cfg)
}

// Applies provided config to reverseProxy
// New config will be applied only if non-nil error returned
func (rp *reverseProxy) ApplyConfig(cfg *config.Config) error {
	if err := cfg.Validate(); err != nil {
		return err
	}

	rp.Lock()
	defer rp.Unlock()

	targets := make([]*target, len(cfg.Cluster.Shards))
	for i, t := range cfg.Cluster.Shards {
		addr, err := url.Parse(fmt.Sprintf("%s://%s", cfg.Cluster.Scheme, t))
		if err != nil {
			return err
		}

		targets[i] = &target{
			addr: addr,
		}
	}

	users := make(map[string]*user, len(cfg.Users))
	for _, u := range cfg.Users {
		users[u.Name] = &user{
			name:                 u.Name,
			maxConcurrentQueries: u.MaxConcurrentQueries,
			maxExecutionTime:     u.MaxExecutionTime,
		}
	}

	rp.targets = targets
	rp.users = users
	return nil
}

type reverseProxy struct {
	*httputil.ReverseProxy

	sync.Mutex
	users   map[string]*user
	targets []*target
}

func (rp *reverseProxy) getRequestScope(req *http.Request) (*scope, error) {
	user, err := rp.getUser(req)
	if err != nil {
		return nil, err
	}

	target := rp.getTarget()
	req.URL.Scheme = target.addr.Scheme
	req.URL.Host = target.addr.Host

	return &scope{
		user:   user,
		target: target,
	}, nil
}

func (rp *reverseProxy) getUser(req *http.Request) (*user, error) {
	name := extractUserFromRequest(req)

	rp.Lock()
	defer rp.Unlock()

	user, ok := rp.users[name]
	if !ok {
		return nil, fmt.Errorf("unknown username %q", name)
	}

	return user, nil
}

func (rp *reverseProxy) getTarget() *target {
	rp.Lock()
	defer rp.Unlock()

	var idle *target
	for _, t := range rp.targets {
		t.Lock()
		if t.runningQueries == 0 {
			t.Unlock()
			return t
		}

		if idle == nil || idle.runningQueries > t.runningQueries {
			idle = t
		}
		t.Unlock()
	}

	return idle
}

// We don't use query_id because of distributed processing, the query ID is not passed to remote servers
func (rp *reverseProxy) killQueries(user string, elapsed float64) {
	rp.Lock()
	addrs := make([]string, len(rp.targets))
	for i, target := range rp.targets {
		addrs[i] = target.addr.String()
	}
	rp.Unlock()

	q := fmt.Sprintf("KILL QUERY WHERE initial_user = '%s' AND elapsed >= %d", user, int(elapsed))
	for _, addr := range addrs {
		if err := doQuery(q, addr); err != nil {
			log.Errorf("error while killing queries older %.2fs than for user %q: %s", elapsed, user, err)
		}
	}
}

type observableTransport struct {
	http.Transport
}

func (pt *observableTransport) RoundTrip(r *http.Request) (*http.Response, error) {
	response, err := pt.Transport.RoundTrip(r)
	if response != nil {
		statusCodes.With(
			prometheus.Labels{"target": r.URL.Host, "code": response.Status},
		).Inc()
	}

	if err != nil {
		errors.With(
			prometheus.Labels{"target": r.URL.Host, "message": err.Error()},
		).Inc()
	}

	return response, err
}<|endoftext|>