<commit_before>// A static blog generator, because everyone needs to write one. With
// categories, markdown, atom feeds.
//
// To get started, copy example/exampleconf.go and customize it for
// your setup. Run something like example/build.sh to build your site.
//
// You need to provide your own templates.
//
// Thomas Kappler <http://www.thomaskappler.net/>
//
// This code is under BSD license. See license-bsd.txt.
package blog11

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"time"
)

type Site struct {
	articles    articles
	conf        *SiteConf
	renderCache map[string]string
}

type SiteConf struct {
	Author, AuthorUri string
	BaseUrl           string
	SiteTitle         string

	TemplateDir string

	WritingDir                 string
	WritingFileExtension       string
	WritingFileDateStampFormat string

	OutDir           string
	CategoriesOutDir string
	ImgOutDir        string

	MaxArticlesOnIndex           int
	NumFreqCategories            int
	MinArticlesForFreqCategories int
	MaxAgeForFreqCategories      time.Duration
}

type category string

func (c category) String() string { return string(c) }

func (c category) Id() string { return strings.Replace(c.String(), " ", "_", -1) }

type categoryArticles struct {
	Category category
	Articles articles
}

func (c categoryArticles) EarliestDateFormatted() string {
	return formatDateShort(c.Articles.earliestDate())
}

func (c categoryArticles) LatestDateFormatted() string {
	return formatDateShort(c.Articles.latestDate())
}

type articlesByCategory []categoryArticles

// Order by number of articles per category, then by newest article.
func (s articlesByCategory) Len() int      { return len(s) }
func (s articlesByCategory) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s articlesByCategory) Less(i, j int) bool {
	li, lj := len(s[i].Articles), len(s[j].Articles)
	if li > lj {
		return true
	} else if lj > li {
		return false
	}

	latestDate1 := s[i].Articles.latestDate()
	latestDate2 := s[j].Articles.latestDate()
	return latestDate1.After(latestDate2)
}

func (ac *articlesByCategory) addArticle(c category, a *article) {
	for i, cat := range *ac {
		if cat.Category == c {
			cat.Articles = append(cat.Articles, a)
			(*ac)[i] = cat
			return
		}
	}

	newCA := categoryArticles{c, make([]*article, 1, 10)}
	newCA.Articles[0] = a
	*ac = append(*ac, newCA)
}

func (ac articlesByCategory) String() string {
	b := new(bytes.Buffer)
	for _, c := range ac {
		b.WriteString(c.Category.String())
		b.WriteString(": ")
		for i, a := range c.Articles {
			if i > 0 {
				b.WriteString(", ")
			}
			b.WriteString(a.Title)
		}
		b.WriteString("\n")
	}
	return b.String()
}

// Return the most frequent n categories.
func (ac articlesByCategory) frequentCategories(n, minArticles int) []category {
	frequent := make([]category, 0, n)
	for i, c := range ac {
		if i == n || len(c.Articles) < minArticles {
			break
		}
		frequent = append(frequent, c.Category)
	}

	return frequent
}

func findArticleFiles(dir, fileExtension string) ([]string, error) {
	files := make([]string, 0, 100)

	myWalkFunc := func(path string, info os.FileInfo, err error) error {
		if err != nil {
			fmt.Printf("Error at %v: %v\n", path, err)
			return nil
		}

		if !info.IsDir() && strings.HasSuffix(path, fileExtension) {
			files = append(files, path)
		}
		return nil
	}

	err := filepath.Walk(dir, myWalkFunc)
	return files, err
}

func readArticleFromFile(path, dateStampFormat string) (*article, error) {
	fileBaseName := filepath.Base(path)
	fileBaseName = fileBaseName[:len(fileBaseName)-len(filepath.Ext(fileBaseName))]

	fileContent, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}

	firstEmptyLine := bytes.Index(fileContent, []byte("\n\n"))
	if firstEmptyLine == -1 {
		return nil, fmt.Errorf("Weird article %v: no empty line.", path)
	}

	a := &article{
		Id:         fileBaseName,
		Body:       fileContent[firstEmptyLine+2:],
		Categories: make([]category, 0, 5),
	}

	headerLines := bytes.Split(fileContent[:firstEmptyLine], []byte("\n"))
	for _, l := range headerLines {
		if colon := bytes.Index(l, []byte(":")); colon != -1 {
			key, val := l[:colon], bytes.TrimSpace(l[colon+1:])
			switch string(key) {
			case "title":
				a.Title = string(val)
			case "blurb":
				a.Blurb = string(val)
			case "categories":
				for _, c := range bytes.Split(val, []byte(",")) {
					a.Categories = append(a.Categories, category(string(bytes.TrimSpace(c))))
				}
			case "static":
				a.Static = true
			default:
				fmt.Printf("  Skipping unknown header field %s in article %v\n", key, fileBaseName)
			}
		} else {
			return nil, fmt.Errorf("Invalid header line in article %v: %s", path, l)
		}
	}

	// Extract the date from the filename, if it's not a static page.
	if !a.Static {
		if len(fileBaseName) < len(dateStampFormat)+1 {
			return nil, fmt.Errorf("Skipping %v, name too short.", fileBaseName)
		}

		dateStr := fileBaseName[:len(dateStampFormat)]
		date, err := time.Parse(dateStampFormat, dateStr)
		if err != nil {
			return nil, fmt.Errorf("Invalid date stamp in %v", dateStampFormat)
		}
		a.Date = date
	}

	return a, nil
}

func renderArticleListToFile(articles []*article, path string, tp templateParam, showTopicsLink bool, engine templateEngine) error {
	outFile, err := os.Create(path)
	if err != nil {
		return err
	}
	defer outFile.Close()

	return engine.renderArticleList(tp, articles, showTopicsLink, outFile)
}

func ReadSite(conf *SiteConf) (*Site, error) {
	files, err := findArticleFiles(conf.WritingDir, conf.WritingFileExtension)
	if err != nil {
		return nil, err
	}

	thisSite := Site{
		articles:    make(articles, 0, 100),
		conf:        conf,
		renderCache: make(map[string]string),
	}

	for _, f := range files {
		a, err := readArticleFromFile(f, conf.WritingFileDateStampFormat)
		if err != nil {
			return nil, err
		}
		thisSite.articles = append(thisSite.articles, a)
	}

	// Order articles by date.
	sort.Sort(thisSite.articles)

	return &thisSite, nil
}

func (s *Site) RenderHtml() error {
	engine := newTemplateEngine(newMarkdownRenderer(), s.conf.TemplateDir)

	// Create a global template parameter holder. We'll re-use for all
	// pages, overwriting the title.
	maxAgeForFreqCategories := s.conf.MaxAgeForFreqCategories
	if maxAgeForFreqCategories == 0 {
		maxAgeForFreqCategories = time.Hour * 24 * 365 * 2
	}

	frequentCategoriesFrom := time.Now().Add(-1 * maxAgeForFreqCategories)
	globalTP := templateParam{
		FrequentCategories: s.articles.byCategoryFrom(frequentCategoriesFrom).frequentCategories(
			s.conf.NumFreqCategories,
			s.conf.MinArticlesForFreqCategories),
	}

	// Render the articles.
	for _, a := range s.articles {
		outHtmlName := filepath.Join(s.conf.OutDir, a.Id+".html")
		var b bytes.Buffer
		globalTP.PageTitle = a.Title
		globalTP.FeedId = "index"
		globalTP.FileId = a.Id
		err, renderedBody := engine.renderArticle(globalTP, a, &b)
		if err != nil {
			return err
		}
		ioutil.WriteFile(outHtmlName, b.Bytes(), os.FileMode(0664))

		s.renderCache[a.Id] = renderedBody
	}

	// Render the category pages.
	byCat := s.articles.byCategory()

	catDir := filepath.Join(s.conf.OutDir, s.conf.CategoriesOutDir)
	if _, err := os.Stat(catDir); os.IsNotExist(err) {
		err2 := os.Mkdir(catDir, os.FileMode(0775))
		if err2 != nil {
			log.Printf("Error creating directory %v: %v", catDir, err2)
		}
	}

	for _, c := range byCat {
		catId := c.Category.Id()
		outHtmlName := filepath.Join(catDir, catId+".html")
		globalTP.PageTitle = c.Category.String()
		globalTP.FeedId = catId
		globalTP.FileId = catId
		err := renderArticleListToFile(c.Articles, outHtmlName, globalTP, false, engine)
		if err != nil {
			return err
		}
	}

	// Render the topics/categories overview page.
	var b bytes.Buffer
	globalTP.PageTitle = "Topics"
	globalTP.FeedId = "index"
	globalTP.FileId = "topics"
	err := engine.renderTopics(globalTP, byCat, &b)
	if err != nil {
		return err
	}
	outHtmlName := filepath.Join(s.conf.OutDir, globalTP.FileId+".html")
	ioutil.WriteFile(outHtmlName, b.Bytes(), os.FileMode(0664))

	// Render index.html with the last MaxArticlesOnIndex articles.
	articlesForIndex := s.articles
	haveMoreArticles := len(s.articles) > s.conf.MaxArticlesOnIndex
	if haveMoreArticles {
		articlesForIndex = articlesForIndex[:s.conf.MaxArticlesOnIndex]
	}
	globalTP.PageTitle = s.conf.SiteTitle
	globalTP.FeedId = "index"
	globalTP.FileId = "index"
	outHtmlName = filepath.Join(s.conf.OutDir, globalTP.FileId+".html")
	return renderArticleListToFile(articlesForIndex, outHtmlName, globalTP, haveMoreArticles, engine)
}

func (s *Site) RenderAll() error {
	err := s.RenderHtml()
	if err != nil {
		return err
	}
	return s.RenderAtom()
}
<commit_msg>Handle Windows line endings when detecting header.<commit_after>// A static blog generator, because everyone needs to write one. With
// categories, markdown, atom feeds.
//
// To get started, copy example/exampleconf.go and customize it for
// your setup. Run something like example/build.sh to build your site.
//
// You need to provide your own templates.
//
// Thomas Kappler <http://www.thomaskappler.net/>
//
// This code is under BSD license. See license-bsd.txt.
package blog11

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"time"
)

type Site struct {
	articles    articles
	conf        *SiteConf
	renderCache map[string]string
}

type SiteConf struct {
	Author, AuthorUri string
	BaseUrl           string
	SiteTitle         string

	TemplateDir string

	WritingDir                 string
	WritingFileExtension       string
	WritingFileDateStampFormat string

	OutDir           string
	CategoriesOutDir string
	ImgOutDir        string

	MaxArticlesOnIndex           int
	NumFreqCategories            int
	MinArticlesForFreqCategories int
	MaxAgeForFreqCategories      time.Duration
}

type category string

func (c category) String() string { return string(c) }

func (c category) Id() string { return strings.Replace(c.String(), " ", "_", -1) }

type categoryArticles struct {
	Category category
	Articles articles
}

func (c categoryArticles) EarliestDateFormatted() string {
	return formatDateShort(c.Articles.earliestDate())
}

func (c categoryArticles) LatestDateFormatted() string {
	return formatDateShort(c.Articles.latestDate())
}

type articlesByCategory []categoryArticles

// Order by number of articles per category, then by newest article.
func (s articlesByCategory) Len() int      { return len(s) }
func (s articlesByCategory) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s articlesByCategory) Less(i, j int) bool {
	li, lj := len(s[i].Articles), len(s[j].Articles)
	if li > lj {
		return true
	} else if lj > li {
		return false
	}

	latestDate1 := s[i].Articles.latestDate()
	latestDate2 := s[j].Articles.latestDate()
	return latestDate1.After(latestDate2)
}

func (ac *articlesByCategory) addArticle(c category, a *article) {
	for i, cat := range *ac {
		if cat.Category == c {
			cat.Articles = append(cat.Articles, a)
			(*ac)[i] = cat
			return
		}
	}

	newCA := categoryArticles{c, make([]*article, 1, 10)}
	newCA.Articles[0] = a
	*ac = append(*ac, newCA)
}

func (ac articlesByCategory) String() string {
	b := new(bytes.Buffer)
	for _, c := range ac {
		b.WriteString(c.Category.String())
		b.WriteString(": ")
		for i, a := range c.Articles {
			if i > 0 {
				b.WriteString(", ")
			}
			b.WriteString(a.Title)
		}
		b.WriteString("\n")
	}
	return b.String()
}

// Return the most frequent n categories.
func (ac articlesByCategory) frequentCategories(n, minArticles int) []category {
	frequent := make([]category, 0, n)
	for i, c := range ac {
		if i == n || len(c.Articles) < minArticles {
			break
		}
		frequent = append(frequent, c.Category)
	}

	return frequent
}

func findArticleFiles(dir, fileExtension string) ([]string, error) {
	files := make([]string, 0, 100)

	myWalkFunc := func(path string, info os.FileInfo, err error) error {
		if err != nil {
			fmt.Printf("Error at %v: %v\n", path, err)
			return nil
		}

		if !info.IsDir() && strings.HasSuffix(path, fileExtension) {
			files = append(files, path)
		}
		return nil
	}

	err := filepath.Walk(dir, myWalkFunc)
	return files, err
}

func readArticleFromFile(path, dateStampFormat string) (*article, error) {
	fileBaseName := filepath.Base(path)
	fileBaseName = fileBaseName[:len(fileBaseName)-len(filepath.Ext(fileBaseName))]

	fileContent, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}

	firstEmptyLine := bytes.Index(fileContent, []byte("\n\n"))
	if firstEmptyLine == -1 {
		firstEmptyLine = bytes.Index(fileContent, []byte("\r\n\r\n"))

		if firstEmptyLine == -1 {
			return nil, fmt.Errorf("weird article %v: no empty line", path)
		}
	}

	a := &article{
		Id:         fileBaseName,
		Body:       fileContent[firstEmptyLine+2:],
		Categories: make([]category, 0, 5),
	}

	headerLines := bytes.Split(fileContent[:firstEmptyLine], []byte("\n"))
	for _, l := range headerLines {
		if colon := bytes.Index(l, []byte(":")); colon != -1 {
			key, val := l[:colon], bytes.TrimSpace(l[colon+1:])
			switch string(key) {
			case "title":
				a.Title = string(val)
			case "blurb":
				a.Blurb = string(val)
			case "categories":
				for _, c := range bytes.Split(val, []byte(",")) {
					a.Categories = append(a.Categories, category(string(bytes.TrimSpace(c))))
				}
			case "static":
				a.Static = true
			default:
				fmt.Printf("  Skipping unknown header field %s in article %v\n", key, fileBaseName)
			}
		} else {
			return nil, fmt.Errorf("Invalid header line in article %v: %s", path, l)
		}
	}

	// Extract the date from the filename, if it's not a static page.
	if !a.Static {
		if len(fileBaseName) < len(dateStampFormat)+1 {
			return nil, fmt.Errorf("Skipping %v, name too short.", fileBaseName)
		}

		dateStr := fileBaseName[:len(dateStampFormat)]
		date, err := time.Parse(dateStampFormat, dateStr)
		if err != nil {
			return nil, fmt.Errorf("Invalid date stamp in %v", dateStampFormat)
		}
		a.Date = date
	}

	return a, nil
}

func renderArticleListToFile(articles []*article, path string, tp templateParam, showTopicsLink bool, engine templateEngine) error {
	outFile, err := os.Create(path)
	if err != nil {
		return err
	}
	defer outFile.Close()

	return engine.renderArticleList(tp, articles, showTopicsLink, outFile)
}

func ReadSite(conf *SiteConf) (*Site, error) {
	files, err := findArticleFiles(conf.WritingDir, conf.WritingFileExtension)
	if err != nil {
		return nil, err
	}

	thisSite := Site{
		articles:    make(articles, 0, 100),
		conf:        conf,
		renderCache: make(map[string]string),
	}

	for _, f := range files {
		a, err := readArticleFromFile(f, conf.WritingFileDateStampFormat)
		if err != nil {
			return nil, err
		}
		thisSite.articles = append(thisSite.articles, a)
	}

	// Order articles by date.
	sort.Sort(thisSite.articles)

	return &thisSite, nil
}

func (s *Site) RenderHtml() error {
	engine := newTemplateEngine(newMarkdownRenderer(), s.conf.TemplateDir)

	// Create a global template parameter holder. We'll re-use for all
	// pages, overwriting the title.
	maxAgeForFreqCategories := s.conf.MaxAgeForFreqCategories
	if maxAgeForFreqCategories == 0 {
		maxAgeForFreqCategories = time.Hour * 24 * 365 * 2
	}

	frequentCategoriesFrom := time.Now().Add(-1 * maxAgeForFreqCategories)
	globalTP := templateParam{
		FrequentCategories: s.articles.byCategoryFrom(frequentCategoriesFrom).frequentCategories(
			s.conf.NumFreqCategories,
			s.conf.MinArticlesForFreqCategories),
	}

	// Render the articles.
	for _, a := range s.articles {
		outHtmlName := filepath.Join(s.conf.OutDir, a.Id+".html")
		var b bytes.Buffer
		globalTP.PageTitle = a.Title
		globalTP.FeedId = "index"
		globalTP.FileId = a.Id
		err, renderedBody := engine.renderArticle(globalTP, a, &b)
		if err != nil {
			return err
		}
		ioutil.WriteFile(outHtmlName, b.Bytes(), os.FileMode(0664))

		s.renderCache[a.Id] = renderedBody
	}

	// Render the category pages.
	byCat := s.articles.byCategory()

	catDir := filepath.Join(s.conf.OutDir, s.conf.CategoriesOutDir)
	if _, err := os.Stat(catDir); os.IsNotExist(err) {
		err2 := os.Mkdir(catDir, os.FileMode(0775))
		if err2 != nil {
			log.Printf("Error creating directory %v: %v", catDir, err2)
		}
	}

	for _, c := range byCat {
		catId := c.Category.Id()
		outHtmlName := filepath.Join(catDir, catId+".html")
		globalTP.PageTitle = c.Category.String()
		globalTP.FeedId = catId
		globalTP.FileId = catId
		err := renderArticleListToFile(c.Articles, outHtmlName, globalTP, false, engine)
		if err != nil {
			return err
		}
	}

	// Render the topics/categories overview page.
	var b bytes.Buffer
	globalTP.PageTitle = "Topics"
	globalTP.FeedId = "index"
	globalTP.FileId = "topics"
	err := engine.renderTopics(globalTP, byCat, &b)
	if err != nil {
		return err
	}
	outHtmlName := filepath.Join(s.conf.OutDir, globalTP.FileId+".html")
	ioutil.WriteFile(outHtmlName, b.Bytes(), os.FileMode(0664))

	// Render index.html with the last MaxArticlesOnIndex articles.
	articlesForIndex := s.articles
	haveMoreArticles := len(s.articles) > s.conf.MaxArticlesOnIndex
	if haveMoreArticles {
		articlesForIndex = articlesForIndex[:s.conf.MaxArticlesOnIndex]
	}
	globalTP.PageTitle = s.conf.SiteTitle
	globalTP.FeedId = "index"
	globalTP.FileId = "index"
	outHtmlName = filepath.Join(s.conf.OutDir, globalTP.FileId+".html")
	return renderArticleListToFile(articlesForIndex, outHtmlName, globalTP, haveMoreArticles, engine)
}

func (s *Site) RenderAll() error {
	err := s.RenderHtml()
	if err != nil {
		return err
	}
	return s.RenderAtom()
}
<|endoftext|>


<commit_before>// Copyright 2013-2016 Aerospike, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package aerospike

import (
	// . "github.com/aerospike/aerospike-client-go/logger"

	"io"
	"strconv"

	. "github.com/aerospike/aerospike-client-go/types"
)

var aeroerr error = NewAerospikeError(PARSE_ERROR, "Error parsing peers list.")

func parsePeers(cluster *Cluster, node *Node) (*peerListParser, error) {
	var cmd string
	if cluster.clientPolicy.TlsConfig != nil {
		if cluster.clientPolicy.UseServicesAlternate {
			cmd = "peers-tls-alt"
		} else {
			cmd = "peers-tls-std"
		}
	} else {
		if cluster.clientPolicy.UseServicesAlternate {
			cmd = "peers-clear-alt"
		} else {
			cmd = "peers-clear-std"
		}
	}

	info, err := node.RequestInfo(cmd)
	if err != nil {
		return nil, err
	}

	peersStr, exists := info[cmd]
	if !exists {
		return nil, NewAerospikeError(PARSE_ERROR, "Info Command response was empty.")
	}

	p := peerListParser{buf: []byte(peersStr)}
	if err := p.Parse(); err != nil {
		return nil, err
	}

	return &p, nil
}

type peerListParser struct {
	buf []byte
	pos int

	defPort *int64
	gen     *int64
	peers   []*peer
}

func (p *peerListParser) generation() int64 {
	if p.gen != nil {
		return *p.gen
	}
	return 0
}

func (p *peerListParser) Expect(ch byte) bool {
	if p.pos == len(p.buf) {
		return false
	}

	if p.buf[p.pos] == ch {
		p.pos++
		return true
	}
	return false
}

func (p *peerListParser) ReadByte() *byte {
	if p.pos == len(p.buf) {
		return nil
	}

	ch := p.buf[p.pos]
	p.pos++
	return &ch
}

func (p *peerListParser) PeekByte() *byte {
	if p.pos == len(p.buf) {
		return nil
	}

	ch := p.buf[p.pos]
	return &ch
}

func (p *peerListParser) ReadInt64() (*int64, error) {
	if p.pos == len(p.buf) {
		return nil, io.EOF
	}

	if p.buf[p.pos] == ',' {
		return nil, nil
	}

	begin := p.pos
	for p.pos < len(p.buf) {
		ch := p.buf[p.pos]
		if ch == ',' {
			break
		}
		p.pos++
	}

	num, err := strconv.ParseInt(string(p.buf[begin:p.pos]), 10, 64)
	if err != nil {
		return nil, err
	}
	return &num, nil
}

func (p *peerListParser) ReadString() (string, error) {
	if p.pos == len(p.buf) {
		return "", io.EOF
	}

	if p.buf[p.pos] == ',' {
		return "", nil
	}

	begin := p.pos
	bracket := p.buf[p.pos] == '['
	for p.pos < len(p.buf) {
		ch := p.buf[p.pos]
		if ch == ',' {
			break
		}

		if ch == ']' {
			if !bracket {
				break
			}
			bracket = false
		}
		p.pos++
	}

	return string(p.buf[begin:p.pos]), nil
}

func (p *peerListParser) ParseHost(host string) (*Host, error) {
	ppos := -1
	bpos := -1
	for i := 0; i < len(host); i++ {
		switch host[i] {
		case ':':
			ppos = i
		case ']':
			ppos = -1
			bpos = i
		}
	}

	port := 0
	if p.defPort != nil {
		port = int(*p.defPort)
	}
	var err error
	if ppos >= 0 {
		portStr := host[ppos+1:]
		port, err = strconv.Atoi(portStr)
		if err != nil {
			return nil, err
		}
	}

	var addr string
	if bpos >= 0 {
		addr = host[1:bpos]
	} else {
		if ppos >= 0 {
			addr = host[:ppos]
		} else {
			addr = host
		}
	}

	return NewHost(addr, port), nil
}

func (p *peerListParser) ReadHosts() ([]*Host, error) {
	if !p.Expect('[') {
		return nil, aeroerr
	}

	hostList := []*Host{}
	for {
		hostStr, err := p.ReadString()
		if err != nil {
			return nil, err
		}

		if hostStr == "" {
			break
		}

		host, err := p.ParseHost(hostStr)
		if err != nil {
			return nil, aeroerr
		}

		hostList = append(hostList, host)

		if !p.Expect(',') {
			break
		}
	}

	if !p.Expect(']') {
		return nil, aeroerr
	}

	return hostList, nil
}

func (p *peerListParser) ReadPeer() (*peer, error) {
	if !p.Expect('[') {
		return nil, nil
	}

	nodeName, err := p.ReadString()
	if err != nil {
		return nil, err
	}

	if !p.Expect(',') {
		return nil, aeroerr
	}
	tlsName, err := p.ReadString()
	if err != nil {
		return nil, err
	}

	if !p.Expect(',') {
		return nil, aeroerr
	}

	hostList, err := p.ReadHosts()
	if err != nil {
		return nil, err
	}

	if !p.Expect(']') {
		return nil, aeroerr
	}

	nodeData := &peer{nodeName: nodeName, tlsName: tlsName, hosts: hostList}
	return nodeData, nil
}

func (p *peerListParser) ReadNodeList() ([]*peer, error) {
	ch := p.ReadByte()
	if ch == nil {
		return nil, nil
	}

	if *ch != '[' {
		return nil, aeroerr
	}

	nodeList := []*peer{}
	for {
		node, err := p.ReadPeer()
		if err != nil {
			return nil, err
		}

		if node == nil {
			break
		}

		nodeList = append(nodeList, node)

		if !p.Expect(',') {
			break
		}
	}

	if !p.Expect(']') {
		return nil, aeroerr
	}

	return nodeList, nil
}

func (p *peerListParser) Parse() error {
	var err error
	p.gen, err = p.ReadInt64()
	if err != nil {
		return err
	}

	if !p.Expect(',') {
		return aeroerr
	}

	p.defPort, err = p.ReadInt64()
	if err != nil {
		return err
	}

	if !p.Expect(',') {
		return aeroerr
	}

	p.peers, err = p.ReadNodeList()
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Add tlsName to the host name in the peers parser<commit_after>// Copyright 2013-2016 Aerospike, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package aerospike

import (
	// . "github.com/aerospike/aerospike-client-go/logger"

	"io"
	"strconv"

	. "github.com/aerospike/aerospike-client-go/types"
)

var aeroerr error = NewAerospikeError(PARSE_ERROR, "Error parsing peers list.")

func parsePeers(cluster *Cluster, node *Node) (*peerListParser, error) {
	var cmd string
	if cluster.clientPolicy.TlsConfig != nil {
		if cluster.clientPolicy.UseServicesAlternate {
			cmd = "peers-tls-alt"
		} else {
			cmd = "peers-tls-std"
		}
	} else {
		if cluster.clientPolicy.UseServicesAlternate {
			cmd = "peers-clear-alt"
		} else {
			cmd = "peers-clear-std"
		}
	}

	info, err := node.RequestInfo(cmd)
	if err != nil {
		return nil, err
	}

	peersStr, exists := info[cmd]
	if !exists {
		return nil, NewAerospikeError(PARSE_ERROR, "Info Command response was empty.")
	}

	p := peerListParser{buf: []byte(peersStr)}
	if err := p.Parse(); err != nil {
		return nil, err
	}

	return &p, nil
}

type peerListParser struct {
	buf []byte
	pos int

	defPort *int64
	gen     *int64
	peers   []*peer
}

func (p *peerListParser) generation() int64 {
	if p.gen != nil {
		return *p.gen
	}
	return 0
}

func (p *peerListParser) Expect(ch byte) bool {
	if p.pos == len(p.buf) {
		return false
	}

	if p.buf[p.pos] == ch {
		p.pos++
		return true
	}
	return false
}

func (p *peerListParser) ReadByte() *byte {
	if p.pos == len(p.buf) {
		return nil
	}

	ch := p.buf[p.pos]
	p.pos++
	return &ch
}

func (p *peerListParser) PeekByte() *byte {
	if p.pos == len(p.buf) {
		return nil
	}

	ch := p.buf[p.pos]
	return &ch
}

func (p *peerListParser) ReadInt64() (*int64, error) {
	if p.pos == len(p.buf) {
		return nil, io.EOF
	}

	if p.buf[p.pos] == ',' {
		return nil, nil
	}

	begin := p.pos
	for p.pos < len(p.buf) {
		ch := p.buf[p.pos]
		if ch == ',' {
			break
		}
		p.pos++
	}

	num, err := strconv.ParseInt(string(p.buf[begin:p.pos]), 10, 64)
	if err != nil {
		return nil, err
	}
	return &num, nil
}

func (p *peerListParser) ReadString() (string, error) {
	if p.pos == len(p.buf) {
		return "", io.EOF
	}

	if p.buf[p.pos] == ',' {
		return "", nil
	}

	begin := p.pos
	bracket := p.buf[p.pos] == '['
	for p.pos < len(p.buf) {
		ch := p.buf[p.pos]
		if ch == ',' {
			break
		}

		if ch == ']' {
			if !bracket {
				break
			}
			bracket = false
		}
		p.pos++
	}

	return string(p.buf[begin:p.pos]), nil
}

func (p *peerListParser) ParseHost(host string) (*Host, error) {
	ppos := -1
	bpos := -1
	for i := 0; i < len(host); i++ {
		switch host[i] {
		case ':':
			ppos = i
		case ']':
			ppos = -1
			bpos = i
		}
	}

	port := 0
	if p.defPort != nil {
		port = int(*p.defPort)
	}
	var err error
	if ppos >= 0 {
		portStr := host[ppos+1:]
		port, err = strconv.Atoi(portStr)
		if err != nil {
			return nil, err
		}
	}

	var addr string
	if bpos >= 0 {
		addr = host[1:bpos]
	} else {
		if ppos >= 0 {
			addr = host[:ppos]
		} else {
			addr = host
		}
	}

	return NewHost(addr, port), nil
}

func (p *peerListParser) ReadHosts(tlsName string) ([]*Host, error) {
	if !p.Expect('[') {
		return nil, aeroerr
	}

	hostList := []*Host{}
	for {
		hostStr, err := p.ReadString()
		if err != nil {
			return nil, err
		}

		if hostStr == "" {
			break
		}

		host, err := p.ParseHost(hostStr)
		if err != nil {
			return nil, aeroerr
		}

		host.TLSName = tlsName
		hostList = append(hostList, host)

		if !p.Expect(',') {
			break
		}
	}

	if !p.Expect(']') {
		return nil, aeroerr
	}

	return hostList, nil
}

func (p *peerListParser) ReadPeer() (*peer, error) {
	if !p.Expect('[') {
		return nil, nil
	}

	nodeName, err := p.ReadString()
	if err != nil {
		return nil, err
	}

	if !p.Expect(',') {
		return nil, aeroerr
	}
	tlsName, err := p.ReadString()
	if err != nil {
		return nil, err
	}

	if !p.Expect(',') {
		return nil, aeroerr
	}

	hostList, err := p.ReadHosts(tlsName)
	if err != nil {
		return nil, err
	}

	if !p.Expect(']') {
		return nil, aeroerr
	}

	nodeData := &peer{nodeName: nodeName, tlsName: tlsName, hosts: hostList}
	return nodeData, nil
}

func (p *peerListParser) ReadNodeList() ([]*peer, error) {
	ch := p.ReadByte()
	if ch == nil {
		return nil, nil
	}

	if *ch != '[' {
		return nil, aeroerr
	}

	nodeList := []*peer{}
	for {
		node, err := p.ReadPeer()
		if err != nil {
			return nil, err
		}

		if node == nil {
			break
		}

		nodeList = append(nodeList, node)

		if !p.Expect(',') {
			break
		}
	}

	if !p.Expect(']') {
		return nil, aeroerr
	}

	return nodeList, nil
}

func (p *peerListParser) Parse() error {
	var err error
	p.gen, err = p.ReadInt64()
	if err != nil {
		return err
	}

	if !p.Expect(',') {
		return aeroerr
	}

	p.defPort, err = p.ReadInt64()
	if err != nil {
		return err
	}

	if !p.Expect(',') {
		return aeroerr
	}

	p.peers, err = p.ReadNodeList()
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2003-2005 Maxim Sobolev. All rights reserved.
// Copyright (c) 2006-2015 Sippy Software, Inc. All rights reserved.
// Copyright (c) 2015 Andrii Pylypenko. All rights reserved.
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation and/or
// other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
// ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package sippy_header

import (
    "strconv"

    "sippy/net"
    "sippy/utils"
)

type SipCSeqBody struct {
    CSeq    int
    Method  string
}

type SipCSeq struct {
    normalName
    string_body string
    body        *SipCSeqBody
}

func NewSipCSeq(cseq int, method string) *SipCSeq {
    return &SipCSeq{
        normalName  : _sip_cseq_name,
        body        : newSipCSeqBody(cseq, method),
    }
}

func newSipCSeqBody(cseq int, method string) *SipCSeqBody {
    return &SipCSeqBody{
        CSeq        : cseq,
        Method      : method,
    }
}

var _sip_cseq_name normalName = newNormalName("CSeq")

func CreateSipCSeq(body string) []SipHeader {
    return []SipHeader{
        &SipCSeq{
            normalName  : _sip_cseq_name,
            string_body : body,
        },
    }
}

func (self *SipCSeq) parse() error {
    arr := sippy_utils.FieldsN(self.string_body, 2)
    cseq, err := strconv.Atoi(arr[0])
    if err != nil {
        return err
    }
    body := &SipCSeqBody{
        CSeq        : cseq,
    }
    if len(arr) == 2 {
        body.Method = arr[1]
    }
    self.body = body
    return nil
}

func (self *SipCSeq) GetCopyAsIface() SipHeader {
    return self.GetCopy()
}

func (self *SipCSeq) GetBody() (*SipCSeqBody, error) {
    if self.body == nil {
        if err := self.parse(); err != nil {
            return nil, err
        }
    }
    return self.body, nil
}

func (self *SipCSeq) GetCopy() *SipCSeq {
    tmp := *self
    return &tmp
}

func (self *SipCSeq) LocalStr(*sippy_net.HostPort, bool) string {
    return self.String()
}

func (self *SipCSeq) String() string {
    return self.Name() + ": " + self.StringBody()
}

func (self *SipCSeq) StringBody() string {
    if self.body != nil {
        return self.body.String()
    }
    return self.string_body
}

func (self *SipCSeqBody) String() string {
    return strconv.Itoa(self.CSeq) + " " + self.Method
}
<commit_msg>Fix the CSeq::GetCopy().<commit_after>// Copyright (c) 2003-2005 Maxim Sobolev. All rights reserved.
// Copyright (c) 2006-2015 Sippy Software, Inc. All rights reserved.
// Copyright (c) 2015 Andrii Pylypenko. All rights reserved.
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation and/or
// other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
// ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package sippy_header

import (
    "strconv"

    "sippy/net"
    "sippy/utils"
)

type SipCSeqBody struct {
    CSeq    int
    Method  string
}

type SipCSeq struct {
    normalName
    string_body string
    body        *SipCSeqBody
}

func NewSipCSeq(cseq int, method string) *SipCSeq {
    return &SipCSeq{
        normalName  : _sip_cseq_name,
        body        : newSipCSeqBody(cseq, method),
    }
}

func newSipCSeqBody(cseq int, method string) *SipCSeqBody {
    return &SipCSeqBody{
        CSeq        : cseq,
        Method      : method,
    }
}

var _sip_cseq_name normalName = newNormalName("CSeq")

func CreateSipCSeq(body string) []SipHeader {
    return []SipHeader{
        &SipCSeq{
            normalName  : _sip_cseq_name,
            string_body : body,
        },
    }
}

func (self *SipCSeq) parse() error {
    arr := sippy_utils.FieldsN(self.string_body, 2)
    cseq, err := strconv.Atoi(arr[0])
    if err != nil {
        return err
    }
    body := &SipCSeqBody{
        CSeq        : cseq,
    }
    if len(arr) == 2 {
        body.Method = arr[1]
    }
    self.body = body
    return nil
}

func (self *SipCSeq) GetCopyAsIface() SipHeader {
    return self.GetCopy()
}

func (self *SipCSeq) GetBody() (*SipCSeqBody, error) {
    if self.body == nil {
        if err := self.parse(); err != nil {
            return nil, err
        }
    }
    return self.body, nil
}

func (self *SipCSeq) GetCopy() *SipCSeq {
    tmp := *self
    if self.body != nil {
        body := *self.body
        tmp.body = &body
    }
    return &tmp
}

func (self *SipCSeq) LocalStr(*sippy_net.HostPort, bool) string {
    return self.String()
}

func (self *SipCSeq) String() string {
    return self.Name() + ": " + self.StringBody()
}

func (self *SipCSeq) StringBody() string {
    if self.body != nil {
        return self.body.String()
    }
    return self.string_body
}

func (self *SipCSeqBody) String() string {
    return strconv.Itoa(self.CSeq) + " " + self.Method
}
<|endoftext|>


<commit_before>package storyactions

import (
	"fmt"
	"strings"
	"time"

	"github.com/fragmenta/query"
	"github.com/fragmenta/server/schedule"

	"github.com/kennygrant/gohackernews/src/lib/facebook"
	"github.com/kennygrant/gohackernews/src/lib/twitter"
	"github.com/kennygrant/gohackernews/src/stories"
)

// TweetTopStory tweets the top story
func TweetTopStory(context schedule.Context) {
	context.Log("Sending top story tweet")

	// Get the top story which has not been tweeted yet, newer than 1 day (we don't look at older stories)
	q := stories.Popular().Limit(1).Order("rank desc, points desc, id desc")

	// Don't fetch old stories - at some point soon this can come down to 1 day
	// as all older stories will have been tweeted
	q.Where("created_at > current_timestamp - interval '60 days'")

	// Don't fetch stories that have already been tweeted
	q.Where("tweeted_at IS NULL")

	// Fetch the stories
	results, err := stories.FindAll(q)
	if err != nil {
		context.Logf("#error getting top story tweet %s", err)
		return
	}

	if len(results) > 0 {
		story := results[0]

		// Link to the primary url for this type of story
		url := story.PrimaryURL()

		if strings.HasPrefix(url, "/") {
			url = "https://golangnews.com" + url
		}

		tweet := fmt.Sprintf("%s #golang %s", story.Name, url)

		context.Logf("#info sending tweet:%s", tweet)

		_, err := twitter.Tweet(tweet)
		if err != nil {
			context.Logf("#error tweeting top story %s", err)
			return
		}

		// Record that this story has been tweeted in db
		params := map[string]string{"tweeted_at": query.TimeString(time.Now().UTC())}
		err = story.Update(params)
		if err != nil {
			context.Logf("#error updating top story tweet %s", err)
			return
		}
	} else {
		context.Logf("#warn no top story found for tweet")
	}

}

// FacebookPostTopStory facebook posts the top story
func FacebookPostTopStory(context schedule.Context) {
	context.Log("#info posting top story facebook")

	// Get the top story
	q := stories.Popular().Limit(1).Order("rank desc, points desc, id desc")

	// Don't fetch old stories
	q.Where("created_at > current_timestamp - interval '6 hours'")

	// Fetch the story
	results, err := stories.FindAll(q)
	if err != nil {
		context.Logf("#error getting top story for fb %s", err)
		return
	}

	if len(results) > 0 {
		story := results[0]
		context.Logf("#info facebook posting %s", story.Name)
		err := facebook.Post(story.Name, story.Url)
		if err != nil {
			context.Logf("#error facebook post top story %s", err)
			return
		}
		// Do not record fb posts - this could lead to duplicates...
		// we should perhaps have a join table for social media posts, rather than dates on stories?
	} else {
		context.Logf("#warn no top story found for fb")
	}
}
<commit_msg>Tweet older stories too<commit_after>package storyactions

import (
	"fmt"
	"strings"
	"time"

	"github.com/fragmenta/query"
	"github.com/fragmenta/server/schedule"

	"github.com/kennygrant/gohackernews/src/lib/facebook"
	"github.com/kennygrant/gohackernews/src/lib/twitter"
	"github.com/kennygrant/gohackernews/src/stories"
)

// TweetTopStory tweets the top story
func TweetTopStory(context schedule.Context) {
	context.Log("Sending top story tweet")

	// Get the top story which has not been tweeted yet, newer than 1 day (we don't look at older stories)
	q := stories.Popular().Limit(1).Order("rank desc, points desc, id desc")

	// Don't fetch old stories - at some point soon this can come down to 1 day
	// as all older stories will have been tweeted
	// For now omit this as we have a backlog of old unposted stories
	// q.Where("created_at > current_timestamp - interval '60 days'")

	// Don't fetch stories that have already been tweeted
	q.Where("tweeted_at IS NULL")

	// Fetch the stories
	results, err := stories.FindAll(q)
	if err != nil {
		context.Logf("#error getting top story tweet %s", err)
		return
	}

	if len(results) > 0 {
		story := results[0]

		// Link to the primary url for this type of story
		url := story.PrimaryURL()

		if strings.HasPrefix(url, "/") {
			url = "https://golangnews.com" + url
		}

		tweet := fmt.Sprintf("%s #golang %s", story.Name, url)

		context.Logf("#info sending tweet:%s", tweet)

		_, err := twitter.Tweet(tweet)
		if err != nil {
			context.Logf("#error tweeting top story %s", err)
			return
		}

		// Record that this story has been tweeted in db
		params := map[string]string{"tweeted_at": query.TimeString(time.Now().UTC())}
		err = story.Update(params)
		if err != nil {
			context.Logf("#error updating top story tweet %s", err)
			return
		}
	} else {
		context.Logf("#warn no top story found for tweet")
	}

}

// FacebookPostTopStory facebook posts the top story
func FacebookPostTopStory(context schedule.Context) {
	context.Log("#info posting top story facebook")

	// Get the top story
	q := stories.Popular().Limit(1).Order("rank desc, points desc, id desc")

	// Don't fetch old stories
	q.Where("created_at > current_timestamp - interval '6 hours'")

	// Fetch the story
	results, err := stories.FindAll(q)
	if err != nil {
		context.Logf("#error getting top story for fb %s", err)
		return
	}

	if len(results) > 0 {
		story := results[0]
		context.Logf("#info facebook posting %s", story.Name)
		err := facebook.Post(story.Name, story.Url)
		if err != nil {
			context.Logf("#error facebook post top story %s", err)
			return
		}
		// Do not record fb posts - this could lead to duplicates...
		// we should perhaps have a join table for social media posts, rather than dates on stories?
	} else {
		context.Logf("#warn no top story found for fb")
	}
}
<|endoftext|>


<commit_before>package pin

import (
	ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
	key "github.com/ipfs/go-ipfs/blocks/key"
	"github.com/ipfs/go-ipfs/blocks/set"
)

type indirectPin struct {
	blockset  set.BlockSet
	refCounts map[key.Key]int
}

func newIndirectPin() *indirectPin {
	return &indirectPin{
		blockset:  set.NewSimpleBlockSet(),
		refCounts: make(map[key.Key]int),
	}
}

func loadIndirPin(d ds.Datastore, k ds.Key) (*indirectPin, error) {
	var rcStore map[string]int
	err := loadSet(d, k, &rcStore)
	if err != nil {
		return nil, err
	}

	refcnt := make(map[key.Key]int)
	var keys []key.Key
	for encK, v := range rcStore {
		if v > 0 {
			k := key.B58KeyDecode(encK)
			keys = append(keys, k)
			refcnt[k] = v
		}
	}
	// log.Debugf("indirPin keys: %#v", keys)

	return &indirectPin{blockset: set.SimpleSetFromKeys(keys), refCounts: refcnt}, nil
}

func storeIndirPin(d ds.Datastore, k ds.Key, p *indirectPin) error {

	rcStore := map[string]int{}
	for k, v := range p.refCounts {
		rcStore[key.B58KeyEncode(k)] = v
	}
	return storeSet(d, k, rcStore)
}

func (i *indirectPin) Increment(k key.Key) {
	c := i.refCounts[k]
	i.refCounts[k] = c + 1
	if c <= 0 {
		i.blockset.AddBlock(k)
	}
}

func (i *indirectPin) Decrement(k key.Key) {
	if i.refCounts[k] == 0 {
		log.Warningf("pinning: bad call: asked to unpin nonexistent indirect key: %v", k)
		return
	}
	c := i.refCounts[k] - 1
	i.refCounts[k] = c
	if c <= 0 {
		i.blockset.RemoveBlock(k)
		delete(i.refCounts, k)
	}
}

func (i *indirectPin) HasKey(k key.Key) bool {
	return i.blockset.HasKey(k)
}

func (i *indirectPin) Set() set.BlockSet {
	return i.blockset
}

func (i *indirectPin) GetRefs() map[key.Key]int {
	return i.refCounts
}
<commit_msg>pin: Remove dead code<commit_after>package pin

import (
	ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
	key "github.com/ipfs/go-ipfs/blocks/key"
	"github.com/ipfs/go-ipfs/blocks/set"
)

type indirectPin struct {
	blockset  set.BlockSet
	refCounts map[key.Key]int
}

func newIndirectPin() *indirectPin {
	return &indirectPin{
		blockset:  set.NewSimpleBlockSet(),
		refCounts: make(map[key.Key]int),
	}
}

func loadIndirPin(d ds.Datastore, k ds.Key) (*indirectPin, error) {
	var rcStore map[string]int
	err := loadSet(d, k, &rcStore)
	if err != nil {
		return nil, err
	}

	refcnt := make(map[key.Key]int)
	var keys []key.Key
	for encK, v := range rcStore {
		if v > 0 {
			k := key.B58KeyDecode(encK)
			keys = append(keys, k)
			refcnt[k] = v
		}
	}
	// log.Debugf("indirPin keys: %#v", keys)

	return &indirectPin{blockset: set.SimpleSetFromKeys(keys), refCounts: refcnt}, nil
}

func storeIndirPin(d ds.Datastore, k ds.Key, p *indirectPin) error {

	rcStore := map[string]int{}
	for k, v := range p.refCounts {
		rcStore[key.B58KeyEncode(k)] = v
	}
	return storeSet(d, k, rcStore)
}

func (i *indirectPin) Increment(k key.Key) {
	c := i.refCounts[k]
	i.refCounts[k] = c + 1
	if c <= 0 {
		i.blockset.AddBlock(k)
	}
}

func (i *indirectPin) Decrement(k key.Key) {
	if i.refCounts[k] == 0 {
		log.Warningf("pinning: bad call: asked to unpin nonexistent indirect key: %v", k)
		return
	}
	c := i.refCounts[k] - 1
	i.refCounts[k] = c
	if c <= 0 {
		i.blockset.RemoveBlock(k)
		delete(i.refCounts, k)
	}
}

func (i *indirectPin) HasKey(k key.Key) bool {
	return i.blockset.HasKey(k)
}

func (i *indirectPin) GetRefs() map[key.Key]int {
	return i.refCounts
}
<|endoftext|>


<commit_before>package osenv

import (
        "fmt"
        "syscall"
        "unsafe"
		)

const(
	//registry keys to modify for 32 bit and 64 bit
	key = `Software\Microsoft\Windows\CurrentVersion\Internet Settings\`
	key_WOW6432Node = `Software\WOW6432Node\Microsoft\Windows\CurrentVersion\Internet Settings`

	REG_NONE      uint64 = 0 // No value type
	REG_SZ               = 1 // Unicode nul terminated string
	REG_EXPAND_SZ        = 2 // Unicode nul terminated string
	// (with environment variable references)
	REG_BINARY                     = 3 // Free form binary
	REG_DWORD                      = 4 // 32-bit number
	REG_DWORD_LITTLE_ENDIAN        = 4 // 32-bit number (same as REG_DWORD)
	REG_DWORD_BIG_ENDIAN           = 5 // 32-bit number
	REG_LINK                       = 6 // Symbolic Link (unicode)
	REG_MULTI_SZ                   = 7 // Multiple Unicode strings
	REG_RESOURCE_LIST              = 8 // Resource list in the resource map
	REG_FULL_RESOURCE_DESCRIPTOR   = 9 // Resource list in the hardware description
	REG_RESOURCE_REQUIREMENTS_LIST = 10
	REG_QWORD                      = 11 // 64-bit number
	REG_QWORD_LITTLE_ENDIAN        = 11 // 64-bit number (same as REG_QWORD)
)

func getValueFromKey(key, value_name string) (value_data string, err error) {
	var h syscall.Handle
	err = syscall.RegOpenKeyEx(syscall.HKEY_LOCAL_MACHINE, syscall.StringToUTF16Ptr(key), 0, syscall.KEY_READ, &h)
	if err != nil {
		return
	}
	defer syscall.RegCloseKey(h)
	var count uint32
	var buf [124]uint16
	n := uint32(len(buf))
	err = syscall.RegQueryValueEx(h, syscall.StringToUTF16Ptr(value_name), nil, &count, (*byte)(unsafe.Pointer(&buf[0])), &n)
	if err != nil {
		return
	}
	value_data = syscall.UTF16ToString(buf[:])
	return
}

func writeToKey(key, value_name string, size, value_data, value_data_type uintptr) (err error) {
		var (
        advapi32, _ = syscall.LoadLibrary("advapi32.dll")
        RegSetValueEx, _ = syscall.GetProcAddress(advapi32, "RegSetValueExW")
	)
	var h syscall.Handle
	err = syscall.RegOpenKeyEx(syscall.HKEY_LOCAL_MACHINE, syscall.StringToUTF16Ptr(key), 0, syscall.KEY_ALL_ACCESS, &h)
	if err != nil {
		fmt.Println(err)
		return
	}
	defer syscall.RegCloseKey(h)
	ret, _, Callerr := syscall.Syscall6(RegSetValueEx, 6,
		uintptr(h),
		uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(value_name))),
		0,
		value_data_type,
		value_data,
		size)
	fmt.Println(ret, Callerr)

	defer syscall.FreeLibrary(advapi32)
	return
}

func EnableProxyServer(key, ProxyServer string) (){
	var (enabled 	uint32 = 1
		 disabled	uint32 = 0)
	ProxyOverride := "<local>"
	writeToKey(key, "ProxyServer", uintptr(len(ProxyServer)*2), uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(ProxyServer))),  REG_SZ)
	writeToKey(key, "ProxyHttp1.1", 4, uintptr(unsafe.Pointer(&disabled)),  REG_DWORD)
	writeToKey(key, "ProxyEnable", 4, uintptr(unsafe.Pointer(&enabled)),  REG_DWORD)
	writeToKey(key, "MigrateProxy", 4, uintptr(unsafe.Pointer(&enabled)),  REG_DWORD)
	writeToKey(key, "ProxyOverride", uintptr(len(ProxyOverride)*2), uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(ProxyOverride))), REG_SZ)
}

// SetEnvironmentValues updates the registry keys with the
// proxy values stored in the settings object.  Both the lower-case
// and upper-case variants are set.
//
// http_proxy, HTTP_PROXY
// https_proxy, HTTPS_PROXY
// ftp_proxy, FTP_PROXY
func (s *ProxySettings) SetEnvironmentValues() {
	// this replaces the setenv part
	EnableProxyServer(key, s.Http)
}

// getProxySetting returns the current value of the proxy url, if it exists,
// otherwise, the value returned is "" (aka empty string)
// warning: get proxy setting (old juju version uses a string parameter, the
// key name, we don't need that)

func getProxySetting() string {
	value, err := getProxySetting(key, "ProxyServer");
	if err != nil {
		if value == "" {
			return ""
		}
		else {
			return value
		}
	}
	return value
}<commit_msg>Changes after testing ouside the project<commit_after>package osenv

import (
        "fmt"
        "syscall"
        "unsafe"
		)

const(
	//registry keys to modify for 32 bit and 64 bit
	key = `Software\Microsoft\Windows\CurrentVersion\Internet Settings\`
	key_WOW6432Node = `Software\WOW6432Node\Microsoft\Windows\CurrentVersion\Internet Settings`

	REG_NONE      uint64 = 0 // No value type
	REG_SZ               = 1 // Unicode nul terminated string
	REG_EXPAND_SZ        = 2 // Unicode nul terminated string
	// (with environment variable references)
	REG_BINARY                     = 3 // Free form binary
	REG_DWORD                      = 4 // 32-bit number
	REG_DWORD_LITTLE_ENDIAN        = 4 // 32-bit number (same as REG_DWORD)
	REG_DWORD_BIG_ENDIAN           = 5 // 32-bit number
	REG_LINK                       = 6 // Symbolic Link (unicode)
	REG_MULTI_SZ                   = 7 // Multiple Unicode strings
	REG_RESOURCE_LIST              = 8 // Resource list in the resource map
	REG_FULL_RESOURCE_DESCRIPTOR   = 9 // Resource list in the hardware description
	REG_RESOURCE_REQUIREMENTS_LIST = 10
	REG_QWORD                      = 11 // 64-bit number
	REG_QWORD_LITTLE_ENDIAN        = 11 // 64-bit number (same as REG_QWORD)
)

func getValueFromKey(key, value_name string) (value_data string, err error) {
	var h syscall.Handle
	err = syscall.RegOpenKeyEx(syscall.HKEY_LOCAL_MACHINE, syscall.StringToUTF16Ptr(key), 0, syscall.KEY_READ, &h)
	if err != nil {
		return
	}
	defer syscall.RegCloseKey(h)
	var count uint32
	var buf [124]uint16
	n := uint32(len(buf))
	err = syscall.RegQueryValueEx(h, syscall.StringToUTF16Ptr(value_name), nil, &count, (*byte)(unsafe.Pointer(&buf[0])), &n)
	if err != nil {
		return
	}
	value_data = syscall.UTF16ToString(buf[:])
	return
}

func writeToKey(key, value_name string, size, value_data, value_data_type uintptr) (err error) {
		var (
        advapi32, _ = syscall.LoadLibrary("advapi32.dll")
        RegSetValueEx, _ = syscall.GetProcAddress(advapi32, "RegSetValueExW")
	)
	var h syscall.Handle
	err = syscall.RegOpenKeyEx(syscall.HKEY_LOCAL_MACHINE, syscall.StringToUTF16Ptr(key), 0, syscall.KEY_ALL_ACCESS, &h)
	if err != nil {
		fmt.Println(err)
		return
	}
	defer syscall.RegCloseKey(h)
	ret, _, Callerr := syscall.Syscall6(RegSetValueEx, 6,
		uintptr(h),
		uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(value_name))),
		0,
		value_data_type,
		value_data,
		size)
	fmt.Println(ret, Callerr)

	defer syscall.FreeLibrary(advapi32)
	return
}

func EnableProxyServer(key, ProxyServer string) (){
	var (enabled 	uint32 = 1
		 disabled	uint32 = 0)
	ProxyOverride := "<local>"
	writeToKey(key, "ProxyServer", uintptr(len(ProxyServer)*2), uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(ProxyServer))),  REG_SZ)
	writeToKey(key, "ProxyHttp1.1", 4, uintptr(unsafe.Pointer(&disabled)),  REG_DWORD)
	writeToKey(key, "ProxyEnable", 4, uintptr(unsafe.Pointer(&enabled)),  REG_DWORD)
	writeToKey(key, "MigrateProxy", 4, uintptr(unsafe.Pointer(&enabled)),  REG_DWORD)
	writeToKey(key, "ProxyOverride", uintptr(len(ProxyOverride)*2), uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(ProxyOverride))), REG_SZ)
}

// SetEnvironmentValues updates the registry keys with the
// proxy values stored in the settings object.  Both the lower-case
// and upper-case variants are set.
//
// http_proxy, HTTP_PROXY
// https_proxy, HTTPS_PROXY
// ftp_proxy, FTP_PROXY
func (s *ProxySettings) SetEnvironmentValues() {
    // this replaces the setenv part
    EnableProxyServer(key, s.Http)
}

// getProxySetting returns the current value of the proxy url, if it exists,
// otherwise, the value returned is "" (aka empty string)
// warning: get proxy setting (old juju version uses a string parameter, the
// key name, we don't need that)

func getProxySetting() string {
    value, err := getValueFromKey(key, "ProxyServer")
    if err != nil {
        return value
    }
    return ""
}<|endoftext|>


<commit_before>/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package k8s

import (
	"context"
	"encoding/json"
	"fmt"
	"strings"
	"time"

	"github.com/pkg/errors"
	corev1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/util/wait"
	"knative.dev/pkg/apis"
	duckv1 "knative.dev/pkg/apis/duck/v1"
	kubeclient "knative.dev/pkg/client/injection/kube/client"
	"knative.dev/pkg/injection/clients/dynamicclient"

	"knative.dev/reconciler-test/pkg/environment"
	"knative.dev/reconciler-test/pkg/feature"
)

// PollTimings will find the correct timings based on priority:
// - passed timing slice [interval, timeout].
// - values from from context.
// - defaults.
func PollTimings(ctx context.Context, timings []time.Duration) (time.Duration /*interval*/, time.Duration /*timeout*/) {
	// Use the passed timing first, but it could be nil or a strange length.
	if len(timings) >= 2 {
		return timings[0], timings[1]
	}

	var interval *time.Duration

	// Use the passed timings if only interval is provided.
	if len(timings) == 1 {
		interval = &timings[0]
	}

	di, timeout := environment.PollTimingsFromContext(ctx)
	if interval == nil {
		interval = &di
	}

	return *interval, timeout
}

// WaitForReadyOrDone will wait for a resource to become ready or succeed.
// Timing is optional but if provided is [interval, timeout].
func WaitForReadyOrDone(ctx context.Context, t feature.T, ref corev1.ObjectReference, timing ...time.Duration) error {
	interval, timeout := PollTimings(ctx, timing)

	k := ref.GroupVersionKind()
	gvr, _ := meta.UnsafeGuessKindToResource(k)

	switch gvr.Resource {
	case "jobs":
		err := WaitUntilJobDone(ctx, t, kubeclient.Get(ctx), ref.Namespace, ref.Name, interval, timeout)
		if err != nil {
			return err
		}
		return nil

	default:
		err := WaitForResourceReady(ctx, t, ref.Namespace, ref.Name, gvr, interval, timeout)
		if err != nil {
			return err
		}
	}

	return nil
}

// WaitForResourceReady waits until the specified resource in the given namespace are ready.
// Timing is optional but if provided is [interval, timeout].
func WaitForResourceReady(ctx context.Context, t feature.T, namespace, name string, gvr schema.GroupVersionResource, timing ...time.Duration) error {
	return WaitForResourceCondition(ctx, t, namespace, name, gvr, isReady(t, name), timing...)
}

// WaitForResourceNotReady waits until the specified resource in the given namespace is not ready.
// Only the top level ready condition is considered (internal `happy` condition of knative.dev/pkg).
// Timing is optional but if provided is [interval, timeout].
func WaitForResourceNotReady(ctx context.Context, t feature.T, namespace, name string, gvr schema.GroupVersionResource, timing ...time.Duration) error {
	return WaitForResourceCondition(ctx, t, namespace, name, gvr, isNotReady(t, name), timing...)
}

// ConditionFunc is a function that determines whether a condition on a resource is satisfied.
type ConditionFunc func(resource duckv1.KResource) bool

// WaitForResourceCondition waits until the specified resource in the given namespace satisfies a given condition.
// Timing is optional but if provided is [interval, timeout].
func WaitForResourceCondition(ctx context.Context, t feature.T, namespace, name string, gvr schema.GroupVersionResource, condition ConditionFunc, timing ...time.Duration) error {
	interval, timeout := PollTimings(ctx, timing)

	like := &duckv1.KResource{}
	return wait.PollImmediate(interval, timeout, func() (bool, error) {
		client := dynamicclient.Get(ctx)

		us, err := client.Resource(gvr).Namespace(namespace).Get(ctx, name, metav1.GetOptions{})
		if err != nil {
			if apierrors.IsNotFound(err) {
				t.Log(namespace, name, "not found", err)
				// keep polling
				return false, nil
			}
			return false, err
		}
		obj := like.DeepCopy()
		if err = runtime.DefaultUnstructuredConverter.FromUnstructured(us.Object, obj); err != nil {
			t.Fatalf("Error DefaultUnstructured.Dynamiconverter. %v", err)
		}
		obj.ResourceVersion = gvr.Version
		obj.APIVersion = gvr.GroupVersion().String()

		// First see if the resource has conditions.
		if len(obj.Status.Conditions) == 0 {
			t.Log("Resource has no conditions")
			return false, nil // keep polling
		}

		// Verify condition.
		return condition(*obj), nil
	})
}

func isReady(t feature.T, name string) ConditionFunc {
	lastMsg := ""
	return func(obj duckv1.KResource) bool {
		if obj.Generation != obj.GetStatus().ObservedGeneration {
			return false
		}
		ready := readyCondition(obj)
		if ready != nil {
			if !ready.IsTrue() {
				msg := fmt.Sprintf("%s is not %s\n\nResource: %s\n", name, ready.Type, resource(obj))
				if msg != lastMsg {
					t.Log(msg)
					lastMsg = msg
				}
			}

			t.Logf("%s is %s, %s: %s\n", name, ready.Type, ready.Reason, ready.Message)
			return ready.IsTrue()
		}

		// Last resort, look at all conditions.
		// As a side-effect of this test,
		//   if a resource has no conditions, then it is ready.
		allReady := true
		for _, c := range obj.Status.Conditions {
			if !c.IsTrue() {
				msg := fmt.Sprintf("%s is not %s, %s: %s\n\nResource: %s\n", name, c.Type, c.Reason, c.Message, resource(obj))
				if msg != lastMsg {
					t.Log(msg)
					lastMsg = msg
				}
				allReady = false
				break
			}
		}
		return allReady
	}
}

func isNotReady(t feature.T, name string) ConditionFunc {
	lastMsg := ""
	return func(obj duckv1.KResource) bool {
		ready := readyCondition(obj)
		if ready == nil {
			msg := fmt.Sprintf("%s hasn't any of %s or %s conditions\n\nResource: %s\n", name, apis.ConditionReady, apis.ConditionSucceeded, resource(obj))
			if msg != lastMsg {
				t.Log(msg)
				lastMsg = msg
			}
			return false
		}
		t.Logf("%s is %s, %s: %s\n\nResource: %s\n", name, ready.Type, ready.Reason, ready.Message, resource(obj))
		return ready.IsFalse()
	}
}

func resource(obj duckv1.KResource) string {
	resource, err := json.MarshalIndent(obj, " ", " ")
	if err != nil {
		resource = []byte(err.Error())
	}
	return string(resource)
}

// readyCondition returns Ready or Succeeded condition.
func readyCondition(obj duckv1.KResource) *apis.Condition {
	// Ready type.
	ready := obj.Status.GetCondition(apis.ConditionReady)
	if ready != nil {
		return ready
	}
	// Succeeded type.
	return obj.Status.GetCondition(apis.ConditionSucceeded)
}

// WaitForServiceEndpointsOrFail polls the status of the specified Service
// every interval until number of service endpoints >= numOfEndpoints.
func WaitForServiceEndpointsOrFail(ctx context.Context, t feature.T, svcName string, numberOfExpectedEndpoints int) {
	endpointsService := kubeclient.Get(ctx).CoreV1().Endpoints(environment.FromContext(ctx).Namespace())
	interval, timeout := PollTimings(ctx, nil)
	err := wait.PollImmediate(interval, timeout, func() (bool, error) {
		endpoint, err := endpointsService.Get(ctx, svcName, metav1.GetOptions{})
		if err != nil {
			return false, err
		}

		return countEndpointsNum(endpoint) >= numberOfExpectedEndpoints, nil
	})
	if err != nil {
		t.Fatalf("Failed while waiting for %d endpoints in service %s: %+v", numberOfExpectedEndpoints, svcName, errors.WithStack(err))
	}
}

// WaitForPodRunningOrFail waits for the given pod to be in running state.
func WaitForPodRunningOrFail(ctx context.Context, t feature.T, podName string) {
	ns := environment.FromContext(ctx).Namespace()
	podClient := kubeclient.Get(ctx).CoreV1().Pods(ns)
	p := podClient
	interval, timeout := PollTimings(ctx, nil)
	err := wait.PollImmediate(interval, timeout, func() (bool, error) {
		p, err := p.Get(ctx, podName, metav1.GetOptions{})
		if err != nil {
			return true, err
		}
		isRunning := podRunning(p)

		if !isRunning {
			t.Log("Pod %s/%s is not running, dumping the pod object we got:", ns, podName)
			b, _ := json.MarshalIndent(p, "", " ")
			t.Log(string(b))
		}

		return isRunning, nil
	})
	if err != nil {
		sb := strings.Builder{}
		if p, err := podClient.Get(ctx, podName, metav1.GetOptions{}); err != nil {
			sb.WriteString(err.Error())
			sb.WriteString("\n")
		} else {
			for _, c := range p.Spec.Containers {
				if b, err := PodLogs(ctx, podName, c.Name, environment.FromContext(ctx).Namespace()); err != nil {
					sb.WriteString(err.Error())
				} else {
					sb.Write(b)
				}
				sb.WriteString("\n")
			}
		}
		t.Fatalf("Failed while waiting for pod %s running: %+v", podName, errors.WithStack(err))
	}
}

// PodLogs returns Pod logs for given Pod and Container in the namespace
func PodLogs(ctx context.Context, podName, containerName, namespace string) ([]byte, error) {
	podClient := kubeclient.Get(ctx).CoreV1().Pods(namespace)
	podList, err := podClient.List(ctx, metav1.ListOptions{})
	if err != nil {
		return nil, err
	}
	for i := range podList.Items {
		// Pods are big, so avoid copying.
		pod := &podList.Items[i]
		if strings.Contains(pod.Name, podName) {
			result := podClient.GetLogs(pod.Name, &corev1.PodLogOptions{
				Container: containerName,
			}).Do(ctx)
			return result.Raw()
		}
	}
	return nil, fmt.Errorf("could not find logs for %s/%s:%s", namespace, podName, containerName)
}

// WaitForAddress waits until a resource has an address.
// Timing is optional but if provided is [interval, timeout].
func WaitForAddress(ctx context.Context, gvr schema.GroupVersionResource, name string, timing ...time.Duration) (*apis.URL, error) {
	interval, timeout := PollTimings(ctx, timing)

	var addr *apis.URL
	err := wait.PollImmediate(interval, timeout, func() (bool, error) {
		var err error
		addr, err = Address(ctx, gvr, name)
		if err == nil && addr == nil {
			// keep polling
			return false, nil
		}
		if err != nil {
			if apierrors.IsNotFound(err) {
				// keep polling
				return false, nil
			}
			// seems fatal.
			return false, err
		}
		// success!
		return true, nil
	})
	return addr, err
}

func countEndpointsNum(e *corev1.Endpoints) int {
	if e == nil || e.Subsets == nil {
		return 0
	}
	num := 0
	for _, sub := range e.Subsets {
		num += len(sub.Addresses)
	}
	return num
}

// podRunning will check the status conditions of the pod and return true if it's Running.
func podRunning(pod *corev1.Pod) bool {
	return pod.Status.Phase == corev1.PodRunning || pod.Status.Phase == corev1.PodSucceeded
}
<commit_msg>Actually log pod logs on error waiting to become ready (#283)<commit_after>/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package k8s

import (
	"context"
	"encoding/json"
	"fmt"
	"strings"
	"time"

	"github.com/pkg/errors"
	corev1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/util/wait"
	"knative.dev/pkg/apis"
	duckv1 "knative.dev/pkg/apis/duck/v1"
	kubeclient "knative.dev/pkg/client/injection/kube/client"
	"knative.dev/pkg/injection/clients/dynamicclient"

	"knative.dev/reconciler-test/pkg/environment"
	"knative.dev/reconciler-test/pkg/feature"
)

// PollTimings will find the correct timings based on priority:
// - passed timing slice [interval, timeout].
// - values from from context.
// - defaults.
func PollTimings(ctx context.Context, timings []time.Duration) (time.Duration /*interval*/, time.Duration /*timeout*/) {
	// Use the passed timing first, but it could be nil or a strange length.
	if len(timings) >= 2 {
		return timings[0], timings[1]
	}

	var interval *time.Duration

	// Use the passed timings if only interval is provided.
	if len(timings) == 1 {
		interval = &timings[0]
	}

	di, timeout := environment.PollTimingsFromContext(ctx)
	if interval == nil {
		interval = &di
	}

	return *interval, timeout
}

// WaitForReadyOrDone will wait for a resource to become ready or succeed.
// Timing is optional but if provided is [interval, timeout].
func WaitForReadyOrDone(ctx context.Context, t feature.T, ref corev1.ObjectReference, timing ...time.Duration) error {
	interval, timeout := PollTimings(ctx, timing)

	k := ref.GroupVersionKind()
	gvr, _ := meta.UnsafeGuessKindToResource(k)

	switch gvr.Resource {
	case "jobs":
		err := WaitUntilJobDone(ctx, t, kubeclient.Get(ctx), ref.Namespace, ref.Name, interval, timeout)
		if err != nil {
			return err
		}
		return nil

	default:
		err := WaitForResourceReady(ctx, t, ref.Namespace, ref.Name, gvr, interval, timeout)
		if err != nil {
			return err
		}
	}

	return nil
}

// WaitForResourceReady waits until the specified resource in the given namespace are ready.
// Timing is optional but if provided is [interval, timeout].
func WaitForResourceReady(ctx context.Context, t feature.T, namespace, name string, gvr schema.GroupVersionResource, timing ...time.Duration) error {
	return WaitForResourceCondition(ctx, t, namespace, name, gvr, isReady(t, name), timing...)
}

// WaitForResourceNotReady waits until the specified resource in the given namespace is not ready.
// Only the top level ready condition is considered (internal `happy` condition of knative.dev/pkg).
// Timing is optional but if provided is [interval, timeout].
func WaitForResourceNotReady(ctx context.Context, t feature.T, namespace, name string, gvr schema.GroupVersionResource, timing ...time.Duration) error {
	return WaitForResourceCondition(ctx, t, namespace, name, gvr, isNotReady(t, name), timing...)
}

// ConditionFunc is a function that determines whether a condition on a resource is satisfied.
type ConditionFunc func(resource duckv1.KResource) bool

// WaitForResourceCondition waits until the specified resource in the given namespace satisfies a given condition.
// Timing is optional but if provided is [interval, timeout].
func WaitForResourceCondition(ctx context.Context, t feature.T, namespace, name string, gvr schema.GroupVersionResource, condition ConditionFunc, timing ...time.Duration) error {
	interval, timeout := PollTimings(ctx, timing)

	like := &duckv1.KResource{}
	return wait.PollImmediate(interval, timeout, func() (bool, error) {
		client := dynamicclient.Get(ctx)

		us, err := client.Resource(gvr).Namespace(namespace).Get(ctx, name, metav1.GetOptions{})
		if err != nil {
			if apierrors.IsNotFound(err) {
				t.Log(namespace, name, "not found", err)
				// keep polling
				return false, nil
			}
			return false, err
		}
		obj := like.DeepCopy()
		if err = runtime.DefaultUnstructuredConverter.FromUnstructured(us.Object, obj); err != nil {
			t.Fatalf("Error DefaultUnstructured.Dynamiconverter. %v", err)
		}
		obj.ResourceVersion = gvr.Version
		obj.APIVersion = gvr.GroupVersion().String()

		// First see if the resource has conditions.
		if len(obj.Status.Conditions) == 0 {
			t.Log("Resource has no conditions")
			return false, nil // keep polling
		}

		// Verify condition.
		return condition(*obj), nil
	})
}

func isReady(t feature.T, name string) ConditionFunc {
	lastMsg := ""
	return func(obj duckv1.KResource) bool {
		if obj.Generation != obj.GetStatus().ObservedGeneration {
			return false
		}
		ready := readyCondition(obj)
		if ready != nil {
			if !ready.IsTrue() {
				msg := fmt.Sprintf("%s is not %s\n\nResource: %s\n", name, ready.Type, resource(obj))
				if msg != lastMsg {
					t.Log(msg)
					lastMsg = msg
				}
			}

			t.Logf("%s is %s, %s: %s\n", name, ready.Type, ready.Reason, ready.Message)
			return ready.IsTrue()
		}

		// Last resort, look at all conditions.
		// As a side-effect of this test,
		//   if a resource has no conditions, then it is ready.
		allReady := true
		for _, c := range obj.Status.Conditions {
			if !c.IsTrue() {
				msg := fmt.Sprintf("%s is not %s, %s: %s\n\nResource: %s\n", name, c.Type, c.Reason, c.Message, resource(obj))
				if msg != lastMsg {
					t.Log(msg)
					lastMsg = msg
				}
				allReady = false
				break
			}
		}
		return allReady
	}
}

func isNotReady(t feature.T, name string) ConditionFunc {
	lastMsg := ""
	return func(obj duckv1.KResource) bool {
		ready := readyCondition(obj)
		if ready == nil {
			msg := fmt.Sprintf("%s hasn't any of %s or %s conditions\n\nResource: %s\n", name, apis.ConditionReady, apis.ConditionSucceeded, resource(obj))
			if msg != lastMsg {
				t.Log(msg)
				lastMsg = msg
			}
			return false
		}
		t.Logf("%s is %s, %s: %s\n\nResource: %s\n", name, ready.Type, ready.Reason, ready.Message, resource(obj))
		return ready.IsFalse()
	}
}

func resource(obj duckv1.KResource) string {
	resource, err := json.MarshalIndent(obj, " ", " ")
	if err != nil {
		resource = []byte(err.Error())
	}
	return string(resource)
}

// readyCondition returns Ready or Succeeded condition.
func readyCondition(obj duckv1.KResource) *apis.Condition {
	// Ready type.
	ready := obj.Status.GetCondition(apis.ConditionReady)
	if ready != nil {
		return ready
	}
	// Succeeded type.
	return obj.Status.GetCondition(apis.ConditionSucceeded)
}

// WaitForServiceEndpointsOrFail polls the status of the specified Service
// every interval until number of service endpoints >= numOfEndpoints.
func WaitForServiceEndpointsOrFail(ctx context.Context, t feature.T, svcName string, numberOfExpectedEndpoints int) {
	endpointsService := kubeclient.Get(ctx).CoreV1().Endpoints(environment.FromContext(ctx).Namespace())
	interval, timeout := PollTimings(ctx, nil)
	err := wait.PollImmediate(interval, timeout, func() (bool, error) {
		endpoint, err := endpointsService.Get(ctx, svcName, metav1.GetOptions{})
		if err != nil {
			return false, err
		}

		return countEndpointsNum(endpoint) >= numberOfExpectedEndpoints, nil
	})
	if err != nil {
		t.Fatalf("Failed while waiting for %d endpoints in service %s: %+v", numberOfExpectedEndpoints, svcName, errors.WithStack(err))
	}
}

// WaitForPodRunningOrFail waits for the given pod to be in running state.
func WaitForPodRunningOrFail(ctx context.Context, t feature.T, podName string) {
	ns := environment.FromContext(ctx).Namespace()
	podClient := kubeclient.Get(ctx).CoreV1().Pods(ns)
	p := podClient
	interval, timeout := PollTimings(ctx, nil)
	err := wait.PollImmediate(interval, timeout, func() (bool, error) {
		p, err := p.Get(ctx, podName, metav1.GetOptions{})
		if err != nil {
			return true, err
		}
		isRunning := podRunning(p)

		if !isRunning {
			t.Log("Pod %s/%s is not running, dumping the pod object we got:", ns, podName)
			b, _ := json.MarshalIndent(p, "", " ")
			t.Log(string(b))
		}

		return isRunning, nil
	})
	if err != nil {
		sb := strings.Builder{}
		if p, err := podClient.Get(ctx, podName, metav1.GetOptions{}); err != nil {
			sb.WriteString(err.Error())
			sb.WriteString("\n")
		} else {
			for _, c := range p.Spec.Containers {
				if b, err := PodLogs(ctx, podName, c.Name, environment.FromContext(ctx).Namespace()); err != nil {
					sb.WriteString(err.Error())
				} else {
					sb.Write(b)
				}
				sb.WriteString("\n")
			}
		}
		t.Fatalf("Failed while waiting for pod %s running: %+v\n%s\n", podName, errors.WithStack(err), sb.String())
	}
}

// PodLogs returns Pod logs for given Pod and Container in the namespace
func PodLogs(ctx context.Context, podName, containerName, namespace string) ([]byte, error) {
	podClient := kubeclient.Get(ctx).CoreV1().Pods(namespace)
	podList, err := podClient.List(ctx, metav1.ListOptions{})
	if err != nil {
		return nil, err
	}
	for i := range podList.Items {
		// Pods are big, so avoid copying.
		pod := &podList.Items[i]
		if strings.Contains(pod.Name, podName) {
			result := podClient.GetLogs(pod.Name, &corev1.PodLogOptions{
				Container: containerName,
			}).Do(ctx)
			return result.Raw()
		}
	}
	return nil, fmt.Errorf("could not find logs for %s/%s:%s", namespace, podName, containerName)
}

// WaitForAddress waits until a resource has an address.
// Timing is optional but if provided is [interval, timeout].
func WaitForAddress(ctx context.Context, gvr schema.GroupVersionResource, name string, timing ...time.Duration) (*apis.URL, error) {
	interval, timeout := PollTimings(ctx, timing)

	var addr *apis.URL
	err := wait.PollImmediate(interval, timeout, func() (bool, error) {
		var err error
		addr, err = Address(ctx, gvr, name)
		if err == nil && addr == nil {
			// keep polling
			return false, nil
		}
		if err != nil {
			if apierrors.IsNotFound(err) {
				// keep polling
				return false, nil
			}
			// seems fatal.
			return false, err
		}
		// success!
		return true, nil
	})
	return addr, err
}

func countEndpointsNum(e *corev1.Endpoints) int {
	if e == nil || e.Subsets == nil {
		return 0
	}
	num := 0
	for _, sub := range e.Subsets {
		num += len(sub.Addresses)
	}
	return num
}

// podRunning will check the status conditions of the pod and return true if it's Running.
func podRunning(pod *corev1.Pod) bool {
	return pod.Status.Phase == corev1.PodRunning || pod.Status.Phase == corev1.PodSucceeded
}
<|endoftext|>


<commit_before>// Copyright 2019 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// ulog implements access to kernel logging.
package ulog

import (
	"fmt"
	"log"
	"os"
	"sync/atomic"
	"testing"

	"golang.org/x/sys/unix"
)

// Logger is any
type Logger interface {
	Printf(format string, v ...interface{})
	Print(v ...interface{})
}

// Log is a Logger that prints to stderr, like the default log package.
var Log = log.New(os.Stderr, "", log.LstdFlags)

// KernelLog is a logger that prints to the kernel syslog buffer.
//
// If the syslog buffer cannot be written to, KernelLog falls back to Log.
var KernelLog = &KLog{
	// Default log level is Info.
	LogLevel: uintptr(KLogInfo),
}

func init() {
	KernelLog.Reinit()
}

// KLog is a logger to the kernel syslog buffer.
type KLog struct {
	// FD for /dev/kmsg if it was openable.
	*os.File

	// LogLevel is the LogLevel to print with.
	//
	// Should only be accessed atomically.
	LogLevel uintptr
}

// Reinit reopens the /dev/kmsg file.
func (k *KLog) Reinit() {
	f, _ := os.OpenFile("/dev/kmsg", os.O_RDWR, 0)
	KernelLog.File = f
}

// writeString returns true iff it was able to write the log to /dev/kmsg.
func (k *KLog) writeString(s string) bool {
	if k.File == nil {
		return false
	}
	if _, err := k.File.WriteString(fmt.Sprintf("<%d>%s", atomic.LoadUintptr(&k.LogLevel), s)); err != nil {
		return false
	}
	return true
}

func (k *KLog) Printf(format string, v ...interface{}) {
	if !k.writeString(fmt.Sprintf(format, v...)) {
		Log.Printf(format, v...)
	}
}

func (k *KLog) Print(v ...interface{}) {
	if !k.writeString(fmt.Sprint(v...)) {
		Log.Print(v...)
	}
}

// KLogLevel are the log levels used by printk.
type KLogLevel uintptr

// These are the log levels used by printk as described in syslog(2).
const (
	KLogEmergency KLogLevel = 0
	KLogAlert     KLogLevel = 1
	KLogCritical  KLogLevel = 2
	KLogError     KLogLevel = 3
	KLogWarning   KLogLevel = 4
	KLogNotice    KLogLevel = 5
	KLogInfo      KLogLevel = 6
	KLogDebug     KLogLevel = 7
)

const (
	_SYSLOG_ACTION_READ_ALL      = 3
	_SYSLOG_ACTION_READ_CLEAR    = 4
	_SYSLOG_ACTION_CLEAR         = 5
	_SYSLOG_ACTION_CONSOLE_LEVEL = 8
)

// SetConsoleLogLevel sets the console level with syslog(2).
//
// After this call, only messages with a level value lower than the one
// specified will be printed to console by the kernel.
func (k *KLog) SetConsoleLogLevel(level KLogLevel) error {
	if _, _, err := unix.Syscall(unix.SYS_SYSLOG, _SYSLOG_ACTION_CONSOLE_LEVEL, 0, uintptr(level)); err != 0 {
		return fmt.Errorf("could not set syslog level to %d: %v", level, err)
	}
	return nil
}

// SetLogLevel sets the level that Printf and Print log to syslog with.
func (k *KLog) SetLogLevel(level KLogLevel) {
	atomic.StoreUintptr(&k.LogLevel, uintptr(level))
}

// ClearLog clears kernel logs back to empty.
func (k *KLog) ClearLog() error {
	_, err := unix.Klogctl(_SYSLOG_ACTION_CLEAR, nil)
	return err
}

// Read reads from the tail of the kernel log.
func (k *KLog) Read(b []byte) (int, error) {
	return unix.Klogctl(_SYSLOG_ACTION_READ_ALL, b)
}

// ReadClear reads from the tail of the kernel log and clears what was read.
func (k *KLog) ReadClear(b []byte) (int, error) {
	return unix.Klogctl(_SYSLOG_ACTION_READ_CLEAR, b)
}

// TestLogger is a Logger implementation that logs to testing.TB.Logf.
type TestLogger struct {
	TB testing.TB
}

func (tl TestLogger) Printf(format string, v ...interface{}) {
	tl.TB.Logf(format, v...)
}

func (tl TestLogger) Print(v ...interface{}) {
	tl.TB.Log(v...)
}
<commit_msg>ulog: improve documentation<commit_after>// Copyright 2019 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// ulog exposes logging via a Go interface.
//
// ulog has three implementations of the Logger interface: a Go standard
// library "log" package Logger, a kernel syslog (dmesg) Logger, and a test
// Logger that logs via a test's testing.TB.Logf.
package ulog

import (
	"fmt"
	"log"
	"os"
	"sync/atomic"
	"testing"

	"golang.org/x/sys/unix"
)

// Logger is a log receptacle.
//
// It puts your information somewhere for safekeeping.
type Logger interface {
	Printf(format string, v ...interface{})
	Print(v ...interface{})
}

// Log is a Logger that prints to stderr, like the default log package.
var Log = log.New(os.Stderr, "", log.LstdFlags)

// KernelLog is a logger that prints to the kernel syslog buffer.
//
// If the syslog buffer cannot be written to, KernelLog falls back to Log.
var KernelLog = &KLog{
	// Default log level is Info.
	LogLevel: uintptr(KLogInfo),
}

func init() {
	KernelLog.Reinit()
}

// KLog is a logger to the kernel syslog buffer.
type KLog struct {
	// FD for /dev/kmsg if it was openable.
	*os.File

	// LogLevel is the LogLevel to print with.
	//
	// Should only be accessed atomically.
	LogLevel uintptr
}

// Reinit reopens the /dev/kmsg file.
func (k *KLog) Reinit() {
	f, _ := os.OpenFile("/dev/kmsg", os.O_RDWR, 0)
	KernelLog.File = f
}

// writeString returns true iff it was able to write the log to /dev/kmsg.
func (k *KLog) writeString(s string) bool {
	if k.File == nil {
		return false
	}
	if _, err := k.File.WriteString(fmt.Sprintf("<%d>%s", atomic.LoadUintptr(&k.LogLevel), s)); err != nil {
		return false
	}
	return true
}

func (k *KLog) Printf(format string, v ...interface{}) {
	if !k.writeString(fmt.Sprintf(format, v...)) {
		Log.Printf(format, v...)
	}
}

func (k *KLog) Print(v ...interface{}) {
	if !k.writeString(fmt.Sprint(v...)) {
		Log.Print(v...)
	}
}

// KLogLevel are the log levels used by printk.
type KLogLevel uintptr

// These are the log levels used by printk as described in syslog(2).
const (
	KLogEmergency KLogLevel = 0
	KLogAlert     KLogLevel = 1
	KLogCritical  KLogLevel = 2
	KLogError     KLogLevel = 3
	KLogWarning   KLogLevel = 4
	KLogNotice    KLogLevel = 5
	KLogInfo      KLogLevel = 6
	KLogDebug     KLogLevel = 7
)

const (
	_SYSLOG_ACTION_READ_ALL      = 3
	_SYSLOG_ACTION_READ_CLEAR    = 4
	_SYSLOG_ACTION_CLEAR         = 5
	_SYSLOG_ACTION_CONSOLE_LEVEL = 8
)

// SetConsoleLogLevel sets the console level with syslog(2).
//
// After this call, only messages with a level value lower than the one
// specified will be printed to console by the kernel.
func (k *KLog) SetConsoleLogLevel(level KLogLevel) error {
	if _, _, err := unix.Syscall(unix.SYS_SYSLOG, _SYSLOG_ACTION_CONSOLE_LEVEL, 0, uintptr(level)); err != 0 {
		return fmt.Errorf("could not set syslog level to %d: %v", level, err)
	}
	return nil
}

// SetLogLevel sets the level that Printf and Print log to syslog with.
func (k *KLog) SetLogLevel(level KLogLevel) {
	atomic.StoreUintptr(&k.LogLevel, uintptr(level))
}

// ClearLog clears kernel logs back to empty.
func (k *KLog) ClearLog() error {
	_, err := unix.Klogctl(_SYSLOG_ACTION_CLEAR, nil)
	return err
}

// Read reads from the tail of the kernel log.
func (k *KLog) Read(b []byte) (int, error) {
	return unix.Klogctl(_SYSLOG_ACTION_READ_ALL, b)
}

// ReadClear reads from the tail of the kernel log and clears what was read.
func (k *KLog) ReadClear(b []byte) (int, error) {
	return unix.Klogctl(_SYSLOG_ACTION_READ_CLEAR, b)
}

// TestLogger is a Logger implementation that logs to testing.TB.Logf.
type TestLogger struct {
	TB testing.TB
}

func (tl TestLogger) Printf(format string, v ...interface{}) {
	tl.TB.Logf(format, v...)
}

func (tl TestLogger) Print(v ...interface{}) {
	tl.TB.Log(v...)
}
<|endoftext|>


<commit_before>package consul

import (
	"context"
	"errors"
	"fmt"
	"math"
	"strings"
	"time"

	"github.com/armon/go-metrics"
	"github.com/armon/go-metrics/prometheus"
	"github.com/hashicorp/go-hclog"

	"github.com/hashicorp/consul/agent/connect"
	"github.com/hashicorp/consul/agent/connect/ca"
	"github.com/hashicorp/consul/logging"
)

var metricsKeyMeshRootCAExpiry = []string{"mesh", "active-root-ca", "expiry"}
var metricsKeyMeshActiveSigningCAExpiry = []string{"mesh", "active-signing-ca", "expiry"}

var LeaderCertExpirationGauges = []prometheus.GaugeDefinition{
	{
		Name: metricsKeyMeshRootCAExpiry,
		Help: "Seconds until the service mesh root certificate expires. Updated every hour",
	},
	{
		Name: metricsKeyMeshActiveSigningCAExpiry,
		Help: "Seconds until the service mesh signing certificate expires. Updated every hour",
	},
}

func rootCAExpiryMonitor(s *Server) CertExpirationMonitor {
	return CertExpirationMonitor{
		Key:    metricsKeyMeshRootCAExpiry,
		Logger: s.logger.Named(logging.Connect),
		Query: func() (time.Duration, error) {
			return getRootCAExpiry(s)
		},
	}
}

func getRootCAExpiry(s *Server) (time.Duration, error) {
	state := s.fsm.State()
	_, root, err := state.CARootActive(nil)
	switch {
	case err != nil:
		return 0, fmt.Errorf("failed to retrieve root CA: %w", err)
	case root == nil:
		return 0, fmt.Errorf("no active root CA")
	}

	return time.Until(root.NotAfter), nil
}

func signingCAExpiryMonitor(s *Server) CertExpirationMonitor {
	isPrimary := s.config.Datacenter == s.config.PrimaryDatacenter
	if isPrimary {
		return CertExpirationMonitor{
			Key:    metricsKeyMeshActiveSigningCAExpiry,
			Logger: s.logger.Named(logging.Connect),
			Query: func() (time.Duration, error) {
				provider, _ := s.caManager.getCAProvider()

				if _, ok := provider.(ca.PrimaryUsesIntermediate); ok {
					return getActiveIntermediateExpiry(s)
				}
				return getRootCAExpiry(s)
			},
		}
	}

	return CertExpirationMonitor{
		Key:    metricsKeyMeshActiveSigningCAExpiry,
		Logger: s.logger.Named(logging.Connect),
		Query: func() (time.Duration, error) {
			return getActiveIntermediateExpiry(s)
		},
	}
}

func getActiveIntermediateExpiry(s *Server) (time.Duration, error) {
	state := s.fsm.State()
	_, root, err := state.CARootActive(nil)
	switch {
	case err != nil:
		return 0, fmt.Errorf("failed to retrieve root CA: %w", err)
	case root == nil:
		return 0, fmt.Errorf("no active root CA")
	}

	// the CA used in a secondary DC is the active intermediate,
	// which is the last in the IntermediateCerts stack
	if len(root.IntermediateCerts) == 0 {
		return 0, errors.New("no intermediate available")
	}
	cert, err := connect.ParseCert(root.IntermediateCerts[len(root.IntermediateCerts)-1])
	if err != nil {
		return 0, err
	}
	return time.Until(cert.NotAfter), nil
}

type CertExpirationMonitor struct {
	Key []string
	// Labels to be emitted along with the metric. It is very important that these
	// labels be included in the pre-declaration as well. Otherwise, if
	// telemetry.prometheus_retention_time is less than certExpirationMonitorInterval
	// then the metrics will expire before they are emitted again.
	Labels []metrics.Label
	Logger hclog.Logger
	// Query is called at each interval. It should return the duration until the
	// certificate expires, or an error if the query failed.
	Query func() (time.Duration, error)
}

const certExpirationMonitorInterval = time.Hour

func (m CertExpirationMonitor) Monitor(ctx context.Context) error {
	ticker := time.NewTicker(certExpirationMonitorInterval)
	defer ticker.Stop()

	logger := m.Logger.With("metric", strings.Join(m.Key, "."))

	emitMetric := func() {
		d, err := m.Query()
		if err != nil {
			logger.Warn("failed to emit certificate expiry metric", "error", err)
			return
		}

		if d < 24*time.Hour {
			logger.Warn("certificate will expire soon",
				"time_to_expiry", d, "expiration", time.Now().Add(d))
		}

		expiry := d / time.Second
		metrics.SetGaugeWithLabels(m.Key, float32(expiry), m.Labels)
	}

	// emit the metric immediately so that if a cert was just updated the
	// new metric will be updated to the new expiration time.
	emitMetric()

	for {
		select {
		case <-ctx.Done():
			// "Zero-out" the metric on exit so that when prometheus scrapes this
			// metric from a non-leader, it does not get a stale value.
			metrics.SetGaugeWithLabels(m.Key, float32(math.NaN()), m.Labels)
			return nil
		case <-ticker.C:
			emitMetric()
		}
	}
}

// initLeaderMetrics sets all metrics that are emitted only on leaders to a NaN
// value so that they don't incorrectly report 0 when a server starts as a
// follower.
func initLeaderMetrics() {
	for _, g := range LeaderCertExpirationGauges {
		metrics.SetGaugeWithLabels(g.Name, float32(math.NaN()), g.ConstLabels)
	}
}
<commit_msg>ca: use the new leaf signing lookup func in leader metrics<commit_after>package consul

import (
	"context"
	"errors"
	"fmt"
	"math"
	"strings"
	"time"

	"github.com/armon/go-metrics"
	"github.com/armon/go-metrics/prometheus"
	"github.com/hashicorp/go-hclog"

	"github.com/hashicorp/consul/agent/connect"
	"github.com/hashicorp/consul/logging"
)

var metricsKeyMeshRootCAExpiry = []string{"mesh", "active-root-ca", "expiry"}
var metricsKeyMeshActiveSigningCAExpiry = []string{"mesh", "active-signing-ca", "expiry"}

var LeaderCertExpirationGauges = []prometheus.GaugeDefinition{
	{
		Name: metricsKeyMeshRootCAExpiry,
		Help: "Seconds until the service mesh root certificate expires. Updated every hour",
	},
	{
		Name: metricsKeyMeshActiveSigningCAExpiry,
		Help: "Seconds until the service mesh signing certificate expires. Updated every hour",
	},
}

func rootCAExpiryMonitor(s *Server) CertExpirationMonitor {
	return CertExpirationMonitor{
		Key:    metricsKeyMeshRootCAExpiry,
		Logger: s.logger.Named(logging.Connect),
		Query: func() (time.Duration, error) {
			return getRootCAExpiry(s)
		},
	}
}

func getRootCAExpiry(s *Server) (time.Duration, error) {
	state := s.fsm.State()
	_, root, err := state.CARootActive(nil)
	switch {
	case err != nil:
		return 0, fmt.Errorf("failed to retrieve root CA: %w", err)
	case root == nil:
		return 0, fmt.Errorf("no active root CA")
	}

	return time.Until(root.NotAfter), nil
}

func signingCAExpiryMonitor(s *Server) CertExpirationMonitor {
	return CertExpirationMonitor{
		Key:    metricsKeyMeshActiveSigningCAExpiry,
		Logger: s.logger.Named(logging.Connect),
		Query: func() (time.Duration, error) {
			if s.caManager.isIntermediateUsedToSignLeaf() {
				return getActiveIntermediateExpiry(s)
			}
			return getRootCAExpiry(s)
		},
	}
}

func getActiveIntermediateExpiry(s *Server) (time.Duration, error) {
	state := s.fsm.State()
	_, root, err := state.CARootActive(nil)
	switch {
	case err != nil:
		return 0, fmt.Errorf("failed to retrieve root CA: %w", err)
	case root == nil:
		return 0, fmt.Errorf("no active root CA")
	}

	// the CA used in a secondary DC is the active intermediate,
	// which is the last in the IntermediateCerts stack
	if len(root.IntermediateCerts) == 0 {
		return 0, errors.New("no intermediate available")
	}
	cert, err := connect.ParseCert(root.IntermediateCerts[len(root.IntermediateCerts)-1])
	if err != nil {
		return 0, err
	}
	return time.Until(cert.NotAfter), nil
}

type CertExpirationMonitor struct {
	Key []string
	// Labels to be emitted along with the metric. It is very important that these
	// labels be included in the pre-declaration as well. Otherwise, if
	// telemetry.prometheus_retention_time is less than certExpirationMonitorInterval
	// then the metrics will expire before they are emitted again.
	Labels []metrics.Label
	Logger hclog.Logger
	// Query is called at each interval. It should return the duration until the
	// certificate expires, or an error if the query failed.
	Query func() (time.Duration, error)
}

const certExpirationMonitorInterval = time.Hour

func (m CertExpirationMonitor) Monitor(ctx context.Context) error {
	ticker := time.NewTicker(certExpirationMonitorInterval)
	defer ticker.Stop()

	logger := m.Logger.With("metric", strings.Join(m.Key, "."))

	emitMetric := func() {
		d, err := m.Query()
		if err != nil {
			logger.Warn("failed to emit certificate expiry metric", "error", err)
			return
		}

		if d < 24*time.Hour {
			logger.Warn("certificate will expire soon",
				"time_to_expiry", d, "expiration", time.Now().Add(d))
		}

		expiry := d / time.Second
		metrics.SetGaugeWithLabels(m.Key, float32(expiry), m.Labels)
	}

	// emit the metric immediately so that if a cert was just updated the
	// new metric will be updated to the new expiration time.
	emitMetric()

	for {
		select {
		case <-ctx.Done():
			// "Zero-out" the metric on exit so that when prometheus scrapes this
			// metric from a non-leader, it does not get a stale value.
			metrics.SetGaugeWithLabels(m.Key, float32(math.NaN()), m.Labels)
			return nil
		case <-ticker.C:
			emitMetric()
		}
	}
}

// initLeaderMetrics sets all metrics that are emitted only on leaders to a NaN
// value so that they don't incorrectly report 0 when a server starts as a
// follower.
func initLeaderMetrics() {
	for _, g := range LeaderCertExpirationGauges {
		metrics.SetGaugeWithLabels(g.Name, float32(math.NaN()), g.ConstLabels)
	}
}
<|endoftext|>


<commit_before>package agentintegration_test

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/garyburd/redigo/redis"
	"github.com/onsi/gomega/gexec"
	"github.com/pivotal-cf/cf-redis-broker/integration/helpers"
	"github.com/pivotal-cf/cf-redis-broker/redisconf"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var redisSession *gexec.Session
var agentSession *gexec.Session

var _ = Describe("DELETE /", func() {

	var (
		redisConn         redis.Conn
		aofPath           string
		originalRedisConf redisconf.Conf
		redisConf redisconf.Conf
	)

	BeforeEach(func() {
		agentSession = startAgent()
		redisSession, aofPath = startRedisAndBlockUntilUp()

		var err error
		originalRedisConf, err = redisconf.Load(redisConfPath)
		Expect(err).NotTo(HaveOccurred())

		originalRedisConn := helpers.BuildRedisClientFromConf(originalRedisConf)
		_, err = originalRedisConn.Do("SET", "key", "val")
		Expect(err).NotTo(HaveOccurred())

		redisStopped := make(chan bool)
		go checkRedisStopped(redisStopped)
		sendResetRequest()

		select {
		case <-redisStopped:
			// Sleep here to emulate the time it takes monit to do it's thing
			time.Sleep(time.Millisecond * 200)

			redisSession, err = gexec.Start(exec.Command("redis-server", redisConfPath), GinkgoWriter, GinkgoWriter)
			Expect(err).ShouldNot(HaveOccurred())

		case <-time.After(time.Second * 10):
			Fail("Test timed out after 10 seconds")
		}

		redisConf = checkRedisConfigured(redisConfPath)
		redisConn = helpers.BuildRedisClientFromConf(redisConf)
	})

	AfterEach(func() {
		stopAgent(agentSession)
		stopRedisAndDeleteData(redisConn, aofPath)
	})

	It("no longer uses the original password", func() {
		password := originalRedisConf.Get("requirepass")
		port := originalRedisConf.Get("port")
		uri := fmt.Sprintf("127.0.0.1:%s", port)
		redisConn, err := redis.Dial("tcp", uri)
		Expect(err).NotTo(HaveOccurred())

		_, err = redisConn.Do("AUTH", password)
		Expect(err).To(MatchError("ERR invalid password"))
	})

	It("resets the configuration", func() {
		config, err := redis.Strings(redisConn.Do("CONFIG", "GET", "maxmemory-policy"))

		Expect(err).NotTo(HaveOccurred())
		Expect(config[1]).To(Equal("noeviction"))
	})

	It("deletes all data from redis", func() {
		values, err := redis.Values(redisConn.Do("KEYS", "*"))
		Expect(err).NotTo(HaveOccurred())
		Expect(values).To(BeEmpty())
	})

	It("has an empty AOF file", func() {
		data, err := ioutil.ReadFile(aofPath)
		Expect(err).NotTo(HaveOccurred())
		Expect(string(data)).To(Equal(""))
	})
})

func checkRedisConfigured(redisConfPath string) redisconf.Conf {
	conf, err := redisconf.Load(redisConfPath)
	Expect(err).NotTo(HaveOccurred())

	port, err := strconv.Atoi(conf.Get("port"))
	Expect(err).NotTo(HaveOccurred())

	Expect(helpers.ServiceAvailable(uint(port))).To(BeTrue())
	redisConf, err := redisconf.Load(redisConfPath)
	Expect(err).NotTo(HaveOccurred())

	return redisConf
}

func startRedisAndBlockUntilUp() (*gexec.Session, string) {
	session, connection := startRedis(redisConfPath)

	_, err := connection.Do("CONFIG", "SET", "maxmemory-policy", "allkeys-lru")
	Ω(err).ShouldNot(HaveOccurred())

	cwd, err := os.Getwd()
	Ω(err).ShouldNot(HaveOccurred())
	aofPath := filepath.Join(cwd, "appendonly.aof")

	Eventually(redisNotWritingAof(connection)).Should(BeTrue())
	Expect(helpers.FileExists(aofPath)).To(BeTrue())

	return session, aofPath
}

func redisNotWritingAof(redisConn redis.Conn) func() bool {
	return func() bool {
		out, _ := redis.String(redisConn.Do("INFO", "persistence"))
		return strings.Contains(out, "aof_pending_rewrite:0") &&
			strings.Contains(out, "aof_rewrite_scheduled:0") &&
			strings.Contains(out, "aof_rewrite_in_progress:0")
	}
}

func sendResetRequest() {
	httpClient := &http.Client{
		Timeout:   300 * time.Second,
		Transport: http.DefaultTransport,
	}

	request, err := http.NewRequest("DELETE", "http://127.0.0.1:9876", nil)
	Expect(err).NotTo(HaveOccurred())

	request.SetBasicAuth("admin", "supersecretpassword")

	response, err := httpClient.Do(request)
	Expect(err).NotTo(HaveOccurred())

	defer response.Body.Close()
	bodyBytes, err := ioutil.ReadAll(response.Body)
	Expect(err).NotTo(HaveOccurred())
	fmt.Println(">>>>> HTTP Response from DELETE <<<<<<<<")
	fmt.Println(string(bodyBytes))
	fmt.Println("<<<<<<< END Response >>>>>>>")

	Expect(response.StatusCode).To(Equal(http.StatusOK))
}

func checkRedisStopped(c chan<- bool) {
	defer GinkgoRecover()
	Eventually(redisSession, "10s").Should(gexec.Exit())
	c <- true
}

func startRedis(confPath string) (*gexec.Session, redis.Conn) {
	redisSession, err := gexec.Start(exec.Command("redis-server", confPath), GinkgoWriter, GinkgoWriter)
	Ω(err).ShouldNot(HaveOccurred())

	conf, err := redisconf.Load(confPath)
	Ω(err).ShouldNot(HaveOccurred())

	port, err := strconv.Atoi(conf.Get("port"))
	Ω(err).ShouldNot(HaveOccurred())

	Expect(helpers.ServiceAvailable(uint(port))).To(BeTrue())
	return redisSession, helpers.BuildRedisClient(uint(port), "localhost", conf.Get("requirepass"))
}

func stopRedisAndDeleteData(redisConn redis.Conn, aofPath string) {
	redisSession.Kill().Wait()
	Eventually(redisSession).Should(gexec.Exit())

	os.Remove(aofPath)
	os.Remove(filepath.Join(aofPath, "..", "dump.rdb"))
}
<commit_msg>Delete reduntant test from cf-redis-broker<commit_after><|endoftext|>