<commit_before>package lint

import (
	"bytes"
	"os/exec"
	"runtime"
	"strings"
	"unicode/utf8"

	"github.com/errata-ai/vale/core"
	"github.com/gobwas/glob"
	"github.com/jdkato/regexp"
	"github.com/russross/blackfriday"
	"golang.org/x/net/html"
)

// reStructuredText configuration.
//
// reCodeBlock is used to convert Sphinx-style code directives to the regular
// `::` for rst2html.
var reCodeBlock = regexp.MustCompile(`.. (?:raw|code(?:-block)?):: (\w+)`)
var rstArgs = []string{
	"--quiet",
	"--halt=5",
	"--report=5",
	"--link-stylesheet",
	"--no-file-insertion",
	"--no-toc-backlinks",
	"--no-footnote-backlinks",
	"--no-section-numbering",
}

// AsciiDoc configuration.
var adocArgs = []string{
	"-s",
	"--quiet",
	"--safe-mode",
	"secure",
	"-",
}

// Blackfriday configuration.
var commonHTMLFlags = 0 | blackfriday.HTML_USE_XHTML
var commonExtensions = 0 |
	blackfriday.EXTENSION_NO_INTRA_EMPHASIS |
	blackfriday.EXTENSION_TABLES |
	blackfriday.EXTENSION_FENCED_CODE
var renderer = blackfriday.HtmlRenderer(commonHTMLFlags, "", "")
var options = blackfriday.Options{Extensions: commonExtensions}
var reFrontMatter = regexp.MustCompile(
	`^(?s)(?:---|\+\+\+)\n([^{]+?)\n(?:---|\+\+\+)`)

// HTML configuration.
var heading = regexp.MustCompile(`^h\d$`)

// skipTags are tags that we don't want to lint.
var skipTags = []string{"script", "style", "pre", "figure"}

// skipClasses are classes that we don't want to lint:
// 	- `problematic` is added by rst2html to processing errors which, in our
// 	  case, could be things like file-insertion URLs.
// 	- `pre` is added by rst2html to code spans.
var skipClasses = []string{"problematic", "pre"}
var inlineTags = []string{
	"b", "big", "i", "small", "abbr", "acronym", "cite", "dfn", "em", "kbd",
	"strong", "a", "br", "img", "span", "sub", "sup", "code", "tt"}
var tagToScope = map[string]string{
	"th": "text.table.header",
	"td": "text.table.cell",
	"li": "text.list",
}

func (l Linter) lintHTMLTokens(f *core.File, ctx string, fsrc []byte, offset int) {
	var txt, attr, raw string
	var tokt html.TokenType
	var tok html.Token
	var inBlock, inline, skip, skipClass bool

	lines := len(f.Lines) + offset
	buf := bytes.NewBufferString("")
	act := bytes.NewBufferString("")

	// queue holds each segment of text we encounter in a block, which we then
	// use to sequentially update our context.
	queue := []string{}

	// tagHistory holds the HTML tags we encounter in a given block -- e.g.,
	// if we see
,
,
, we'd get tagHistory = [ul li p]. It's reset
	// on every non-inline end tag.
	tagHistory := []string{}

	tokens := html.NewTokenizer(bytes.NewReader(fsrc))

	skipped := []string{"tt", "code"}
	if len(l.Config.IgnoredScopes) > 0 {
		skipped = l.Config.IgnoredScopes
	}

	for {
		tokt = tokens.Next()
		tok = tokens.Token()
		txt = html.UnescapeString(strings.TrimSpace(tok.Data))

		skipClass = core.StringInSlice(attr, skipClasses)
		if tokt == html.ErrorToken {
			break
		} else if tokt == html.StartTagToken && core.StringInSlice(txt, skipTags) {
			inBlock = true
		} else if inBlock && core.StringInSlice(txt, skipTags) {
			skip, inBlock = false, false
		} else if tokt == html.StartTagToken {
			inline = core.StringInSlice(txt, inlineTags)
			// skip = (txt == "tt" || txt == "code")
			skip = core.StringInSlice(txt, skipped)
			tagHistory = append(tagHistory, txt)
		} else if tokt == html.CommentToken {
			f.UpdateComments(txt)
		} else if tokt == html.TextToken {
			queue = append(queue, txt)
			if !inBlock {
				txt, raw, skip = clean(txt, f.NormedExt, skip, skipClass, inline)
				buf.WriteString(txt)
				act.WriteString(raw)
			}
		}

		if tokt == html.EndTagToken && !core.StringInSlice(txt, inlineTags) {
			content := buf.String()
			actual := act.String()
			if content != "" {
				l.lintScope(f, ctx, content, actual, tagHistory, lines)
			}
			for _, s := range queue {
				ctx = updateCtx(ctx, s, html.TextToken)
			}
			queue = []string{}
			tagHistory = []string{}
			buf.Reset()
			act.Reset()
		}

		attr = getAttribute(tok, "class")
		ctx = clearElements(ctx, tok)

		if tok.Data == "img" {
			for _, a := range tok.Attr {
				if a.Key == "alt" || a.Key == "title" {
					l.lintScope(f, ctx, a.Val, a.Val, tagHistory, lines)
				}
			}
		}
	}

	summary := NewBlock("", f.Summary.String(), "", "summary."+f.RealExt)
	l.lintText(f, summary, lines, 0)
}

func (l Linter) lintScope(f *core.File, ctx, txt, raw string, tags []string, lines int) {
	for _, tag := range tags {
		scope, match := tagToScope[tag]
		if match || heading.MatchString(tag) {
			if match {
				scope = scope + f.RealExt
			} else {
				scope = "text.heading." + tag + f.RealExt
			}
			txt = strings.TrimLeft(txt, " ")
			l.lintText(f, NewBlock(ctx, txt, raw, scope), lines, 0)
			return
		}
	}

	// NOTE: We don't include headings, list items, or table cells (which are
	// processed above) in our Summary content.
	f.Summary.WriteString(raw + " ")
	l.lintProse(f, ctx, txt, raw, lines, 0)
}

func codify(ext, text string) string {
	if ext == ".md" || ext == ".adoc" {
		return "`" + text + "`"
	} else if ext == ".rst" {
		return "``" + text + "``"
	}
	return text
}

func clean(txt, ext string, skip, skipClass, inline bool) (string, string, bool) {
	punct := []string{".", "?", "!", ",", ":", ";"}
	first, _ := utf8.DecodeRuneInString(txt)
	starter := core.StringInSlice(string(first), punct) && !skip
	raw := txt
	if skip || skipClass {
		raw = codify(ext, txt)
		txt, _ = core.Substitute(txt, txt, '*')
		txt = codify(ext, txt)
		skip = false
	}
	if inline && !starter {
		txt = " " + txt
		raw = " " + raw
	}
	return txt, raw, skip
}

func getAttribute(tok html.Token, key string) string {
	for _, attr := range tok.Attr {
		if attr.Key == key {
			return attr.Val
		}
	}
	return ""
}

func updateCtx(ctx string, txt string, tokt html.TokenType) string {
	var found bool
	if (tokt == html.TextToken || tokt == html.CommentToken) && txt != "" {
		for _, s := range strings.Split(txt, "\n") {
			ctx, found = core.Substitute(ctx, s, '@')
			if !found {
				for _, w := range strings.Fields(s) {
					ctx, _ = core.Substitute(ctx, w, '@')
				}
			}
		}
	}
	return ctx
}

func clearElements(ctx string, tok html.Token) string {
	if tok.Data == "img" || tok.Data == "a" || tok.Data == "p" || tok.Data == "script" {
		for _, a := range tok.Attr {
			if a.Key == "href" || a.Key == "id" || a.Key == "src" {
				ctx = updateCtx(ctx, a.Val, html.TextToken)
			}
		}
	}
	return ctx
}

func (l Linter) lintHTML(f *core.File) {
	l.lintHTMLTokens(f, f.Content, []byte(f.Content), 0)
}

func (l Linter) prepMarkdown(content string) string {
	s := reFrontMatter.ReplaceAllString(content, "```\n$1\n```")
	for syntax, regexes := range l.Config.IgnorePatterns {
		sec, err := glob.Compile(syntax)
		if core.CheckError(err) && sec.Match(".md") {
			for _, r := range regexes {
				pat, err := regexp.Compile(r)
				if err == nil {
					s = pat.ReplaceAllString(s, "\n```\n$1\n```\n")
				}
			}
		}
	}

	return s
}

func (l Linter) lintMarkdown(f *core.File) {
	s := l.prepMarkdown(f.Content)
	html := blackfriday.MarkdownOptions([]byte(s), renderer, options)
	l.lintHTMLTokens(f, f.Content, html, 0)
}

func (l Linter) lintRST(f *core.File, python string, rst2html string) {
	var out bytes.Buffer
	var cmd *exec.Cmd

	if runtime.GOOS == "windows" {
		// rst2html is executable by default on Windows.
		cmd = exec.Command(python, append([]string{rst2html}, rstArgs...)...)
	} else {
		cmd = exec.Command(rst2html, rstArgs...)
	}

	cmd.Stdin = strings.NewReader(reCodeBlock.ReplaceAllString(f.Content, "::"))
	cmd.Stdout = &out

	if core.CheckError(cmd.Run()) {
		html := bytes.Replace(out.Bytes(), []byte("\r"), []byte(""), -1)
		bodyStart := bytes.Index(html, []byte("\n"))
		if bodyStart < 0 {
			bodyStart = -7
		}
		bodyEnd := bytes.Index(html, []byte("\n"))
		if bodyEnd < 0 || bodyEnd >= len(html) {
			bodyEnd = len(html) - 1
			if bodyEnd < 0 {
				bodyEnd = 0
			}
		}
		l.lintHTMLTokens(f, f.Content, html[bodyStart+7:bodyEnd], 0)
	}
}

func (l Linter) lintADoc(f *core.File, asciidoctor string) {
	var out bytes.Buffer
	cmd := exec.Command(asciidoctor, adocArgs...)
	cmd.Stdin = strings.NewReader(f.Content)
	cmd.Stdout = &out
	if core.CheckError(cmd.Run()) {
		l.lintHTMLTokens(f, f.Content, out.Bytes(), 0)
	}
}

func (l Linter) lintCustom(f *core.File, command string, args []string) {
	var out bytes.Buffer
	s := f.Content
	if f.NormedExt == ".md" {
		s = l.prepMarkdown(s)
	}
	cmd := exec.Command(command, args...)
	cmd.Stdin = strings.NewReader(s)
	cmd.Stdout = &out
	if core.CheckError(cmd.Run()) {
		l.lintHTMLTokens(f, f.Content, out.Bytes(), 0)
	}
}
<commit_msg>WIP: math<commit_after>package lint

import (
	"bytes"
	"os/exec"
	"runtime"
	"strings"
	"unicode/utf8"

	"github.com/errata-ai/vale/core"
	"github.com/gobwas/glob"
	"github.com/jdkato/regexp"
	"github.com/russross/blackfriday"
	"golang.org/x/net/html"
)

// reStructuredText configuration.
//
// reCodeBlock is used to convert Sphinx-style code directives to the regular
// `::` for rst2html.
var reCodeBlock = regexp.MustCompile(`.. (?:raw|code(?:-block)?):: (\w+)`)
var rstArgs = []string{
	"--quiet",
	"--halt=5",
	"--report=5",
	"--link-stylesheet",
	"--no-file-insertion",
	"--no-toc-backlinks",
	"--no-footnote-backlinks",
	"--no-section-numbering",
}

// AsciiDoc configuration.
var adocArgs = []string{
	"-s",
	"--quiet",
	"--safe-mode",
	"secure",
	"-",
}

// Blackfriday configuration.
var commonHTMLFlags = 0 | blackfriday.HTML_USE_XHTML
var commonExtensions = 0 |
	blackfriday.EXTENSION_NO_INTRA_EMPHASIS |
	blackfriday.EXTENSION_TABLES |
	blackfriday.EXTENSION_FENCED_CODE
var renderer = blackfriday.HtmlRenderer(commonHTMLFlags, "", "")
var options = blackfriday.Options{Extensions: commonExtensions}
var reFrontMatter = regexp.MustCompile(
	`^(?s)(?:---|\+\+\+)\n([^{]+?)\n(?:---|\+\+\+)`)
var reMathInline = regexp.MustCompile(`\${1,2}([^\n$]+)\${1,2}`)

// HTML configuration.
var heading = regexp.MustCompile(`^h\d$`)

// skipTags are tags that we don't want to lint.
var skipTags = []string{"script", "style", "pre", "figure"}

// skipClasses are classes that we don't want to lint:
// 	- `problematic` is added by rst2html to processing errors which, in our
// 	  case, could be things like file-insertion URLs.
// 	- `pre` is added by rst2html to code spans.
var skipClasses = []string{"problematic", "pre"}
var inlineTags = []string{
	"b", "big", "i", "small", "abbr", "acronym", "cite", "dfn", "em", "kbd",
	"strong", "a", "br", "img", "span", "sub", "sup", "code", "tt"}
var tagToScope = map[string]string{
	"th": "text.table.header",
	"td": "text.table.cell",
	"li": "text.list",
}

func (l Linter) lintHTMLTokens(f *core.File, ctx string, fsrc []byte, offset int) {
	var txt, attr, raw string
	var tokt html.TokenType
	var tok html.Token
	var inBlock, inline, skip, skipClass bool

	lines := len(f.Lines) + offset
	buf := bytes.NewBufferString("")
	act := bytes.NewBufferString("")

	// queue holds each segment of text we encounter in a block, which we then
	// use to sequentially update our context.
	queue := []string{}

	// tagHistory holds the HTML tags we encounter in a given block -- e.g.,
	// if we see
,
,
, we'd get tagHistory = [ul li p]. It's reset
	// on every non-inline end tag.
	tagHistory := []string{}

	tokens := html.NewTokenizer(bytes.NewReader(fsrc))

	skipped := []string{"tt", "code"}
	if len(l.Config.IgnoredScopes) > 0 {
		skipped = l.Config.IgnoredScopes
	}

	for {
		tokt = tokens.Next()
		tok = tokens.Token()
		txt = html.UnescapeString(strings.TrimSpace(tok.Data))

		skipClass = core.StringInSlice(attr, skipClasses)
		if tokt == html.ErrorToken {
			break
		} else if tokt == html.StartTagToken && core.StringInSlice(txt, skipTags) {
			inBlock = true
		} else if inBlock && core.StringInSlice(txt, skipTags) {
			skip, inBlock = false, false
		} else if tokt == html.StartTagToken {
			inline = core.StringInSlice(txt, inlineTags)
			// skip = (txt == "tt" || txt == "code")
			skip = core.StringInSlice(txt, skipped)
			tagHistory = append(tagHistory, txt)
		} else if tokt == html.CommentToken {
			f.UpdateComments(txt)
		} else if tokt == html.TextToken {
			queue = append(queue, txt)
			if !inBlock {
				txt, raw, skip = clean(txt, f.NormedExt, skip, skipClass, inline)
				buf.WriteString(txt)
				act.WriteString(raw)
			}
		}

		if tokt == html.EndTagToken && !core.StringInSlice(txt, inlineTags) {
			content := buf.String()
			actual := act.String()
			if content != "" {
				l.lintScope(f, ctx, content, actual, tagHistory, lines)
			}
			for _, s := range queue {
				ctx = updateCtx(ctx, s, html.TextToken)
			}
			queue = []string{}
			tagHistory = []string{}
			buf.Reset()
			act.Reset()
		}

		attr = getAttribute(tok, "class")
		ctx = clearElements(ctx, tok)

		if tok.Data == "img" {
			for _, a := range tok.Attr {
				if a.Key == "alt" || a.Key == "title" {
					l.lintScope(f, ctx, a.Val, a.Val, tagHistory, lines)
				}
			}
		}
	}

	summary := NewBlock("", f.Summary.String(), "", "summary."+f.RealExt)
	l.lintText(f, summary, lines, 0)
}

func (l Linter) lintScope(f *core.File, ctx, txt, raw string, tags []string, lines int) {
	for _, tag := range tags {
		scope, match := tagToScope[tag]
		if match || heading.MatchString(tag) {
			if match {
				scope = scope + f.RealExt
			} else {
				scope = "text.heading." + tag + f.RealExt
			}
			txt = strings.TrimLeft(txt, " ")
			l.lintText(f, NewBlock(ctx, txt, raw, scope), lines, 0)
			return
		}
	}

	// NOTE: We don't include headings, list items, or table cells (which are
	// processed above) in our Summary content.
	f.Summary.WriteString(raw + " ")
	l.lintProse(f, ctx, txt, raw, lines, 0)
}

func codify(ext, text string) string {
	if ext == ".md" || ext == ".adoc" {
		return "`" + text + "`"
	} else if ext == ".rst" {
		return "``" + text + "``"
	}
	return text
}

func clean(txt, ext string, skip, skipClass, inline bool) (string, string, bool) {
	punct := []string{".", "?", "!", ",", ":", ";"}
	first, _ := utf8.DecodeRuneInString(txt)
	starter := core.StringInSlice(string(first), punct) && !skip
	raw := txt
	if skip || skipClass {
		raw = codify(ext, txt)
		txt, _ = core.Substitute(txt, txt, '*')
		txt = codify(ext, txt)
		skip = false
	}
	if inline && !starter {
		txt = " " + txt
		raw = " " + raw
	}
	return txt, raw, skip
}

func getAttribute(tok html.Token, key string) string {
	for _, attr := range tok.Attr {
		if attr.Key == key {
			return attr.Val
		}
	}
	return ""
}

func updateCtx(ctx string, txt string, tokt html.TokenType) string {
	var found bool
	if (tokt == html.TextToken || tokt == html.CommentToken) && txt != "" {
		for _, s := range strings.Split(txt, "\n") {
			ctx, found = core.Substitute(ctx, s, '@')
			if !found {
				for _, w := range strings.Fields(s) {
					ctx, _ = core.Substitute(ctx, w, '@')
				}
			}
		}
	}
	return ctx
}

func clearElements(ctx string, tok html.Token) string {
	if tok.Data == "img" || tok.Data == "a" || tok.Data == "p" || tok.Data == "script" {
		for _, a := range tok.Attr {
			if a.Key == "href" || a.Key == "id" || a.Key == "src" {
				ctx = updateCtx(ctx, a.Val, html.TextToken)
			}
		}
	}
	return ctx
}

func (l Linter) lintHTML(f *core.File) {
	l.lintHTMLTokens(f, f.Content, []byte(f.Content), 0)
}

var reCite = regexp.MustCompile(`(\^\w{2,})\b`)

func (l Linter) prepMarkdown(content string) string {
	// TODO: Make format-agnostic
	//
	// TODO: Introduce IgnoredTokens for things like `reCite`
	s := reFrontMatter.ReplaceAllString(content, "```\n$1\n```")
	s = reMathInline.ReplaceAllString(s, "`$1`")
	s = reCite.ReplaceAllString(s, "`$1`")

	for syntax, regexes := range l.Config.IgnorePatterns {
		sec, err := glob.Compile(syntax)
		if core.CheckError(err) && sec.Match(".md") {
			for _, r := range regexes {
				pat, err := regexp.Compile(r)
				if err == nil {
					s = pat.ReplaceAllString(s, "\n```\n$1\n```\n")
				}
			}
		}
	}

	return s
}

func (l Linter) lintMarkdown(f *core.File) {
	s := l.prepMarkdown(f.Content)
	html := blackfriday.MarkdownOptions([]byte(s), renderer, options)
	l.lintHTMLTokens(f, f.Content, html, 0)
}

func (l Linter) lintRST(f *core.File, python string, rst2html string) {
	var out bytes.Buffer
	var cmd *exec.Cmd

	if runtime.GOOS == "windows" {
		// rst2html is executable by default on Windows.
		cmd = exec.Command(python, append([]string{rst2html}, rstArgs...)...)
	} else {
		cmd = exec.Command(rst2html, rstArgs...)
	}

	cmd.Stdin = strings.NewReader(reCodeBlock.ReplaceAllString(f.Content, "::"))
	cmd.Stdout = &out

	if core.CheckError(cmd.Run()) {
		html := bytes.Replace(out.Bytes(), []byte("\r"), []byte(""), -1)
		bodyStart := bytes.Index(html, []byte("\n"))
		if bodyStart < 0 {
			bodyStart = -7
		}
		bodyEnd := bytes.Index(html, []byte("\n"))
		if bodyEnd < 0 || bodyEnd >= len(html) {
			bodyEnd = len(html) - 1
			if bodyEnd < 0 {
				bodyEnd = 0
			}
		}
		l.lintHTMLTokens(f, f.Content, html[bodyStart+7:bodyEnd], 0)
	}
}

func (l Linter) lintADoc(f *core.File, asciidoctor string) {
	var out bytes.Buffer
	cmd := exec.Command(asciidoctor, adocArgs...)
	cmd.Stdin = strings.NewReader(f.Content)
	cmd.Stdout = &out
	if core.CheckError(cmd.Run()) {
		l.lintHTMLTokens(f, f.Content, out.Bytes(), 0)
	}
}

func (l Linter) lintCustom(f *core.File, command string, args []string) {
	var out bytes.Buffer
	s := f.Content
	if f.NormedExt == ".md" {
		s = l.prepMarkdown(s)
	}
	cmd := exec.Command(command, args...)
	cmd.Stdin = strings.NewReader(s)
	cmd.Stdout = &out
	if core.CheckError(cmd.Run()) {
		l.lintHTMLTokens(f, f.Content, out.Bytes(), 0)
	}
}
<|endoftext|>


<commit_before>package filter
<commit_msg>Pair program NOP & blacklist filters<commit_after>package filter
import (
	"encoding/json"
	"strings"
)

type FilterConfig struct {
	ignore []string `json:"ignore"`
}

type Filter interface {
	Start(c FilterConfig, sending chan<- string, receiving <-chan string)
}

type FSFilter struct { }

type NOP struct { }

type ZachsInotifyData struct {
	Date	 string `json:"DATE"`
	Event	 string `json:"EVENT"`
	FilePath string `json:"PATH"`
	Type	 string `json:"TYPE"`
}

func (f FSFilter) Start (c FilterConfig, sending chan<- string, receiving <-chan string){
	for {
		select {
			case message := <-receiving:
				zid := ZachsInotifyData{}
				err := json.Unmarshal([]byte(message), &zid)
				if err != nil {
					panic(err)
				}
				notblacklisted := true
				for _, i := range c.ignore {
					if strings.HasPrefix(zid.FilePath,i) {
						notblacklisted = false
						break
					}
				}
				if notblacklisted {
					sending <- message
				}
		}
	}

}

func (N NOP) Start( c FilterConfig, sending chan<- string, receiving <-chan string){
	for {
		select{
			case message := <- receiving:
				sending <- message
		}
	}

}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto"
	"crypto/rand"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/pem"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"strings"
	"time"

	"github.com/yahoo/athenz/clients/go/zts"
	"github.com/yahoo/athenz/libs/go/zmssvctoken"
)

type signer struct {
	key crypto.Signer
}

func main() {

	var ztsUrl, privateKeyFile, domain, service, keyVersion, certFile, dnsDomain string
	flag.StringVar(&certFile, "cert-file", "", "output certificate file")
	flag.StringVar(&privateKeyFile, "private-key", "", "private key file")
	flag.StringVar(&domain, "domain", "", "domain of service")
	flag.StringVar(&service, "service", "", "name of service")
	flag.StringVar(&keyVersion, "key-version", "", "key version")
	flag.StringVar(&ztsUrl, "zts", "", "url of the ZTS Service")
	flag.StringVar(&dnsDomain, "dns-domain", "", "dns domain suffix to be included in the csr")
	flag.StringVar(&hdr, "hdr", "Athenz-Principal-Auth", "Header name")
	flag.Parse()

	if privateKeyFile == "" || domain == "" || service == "" ||
		keyVersion == "" || ztsUrl == "" || dnsDomain == "" {
		log.Fatalln("usage: zts-svccert -domain <domain> -service <service> -private-key <key-file> -key-version <version> -zts <zts-server-url> -dns-domain <dns-domain> [-cert-file <output-cert-file>]")
	}

	// load private key
	bytes, err := ioutil.ReadFile(privateKeyFile)
	if err != nil {
		log.Fatalln(err)
	}

	// get token builder instance
	builder, err := zmssvctoken.NewTokenBuilder(domain, service, bytes, keyVersion)
	if err != nil {
		log.Fatalln(err)
	}

	// set optional attributes
	builder.SetExpiration(10 * time.Minute)

	// get a token instance that always gives you unexpired tokens values
	// safe for concurrent use
	tok := builder.Token()

	// get a token for use
	ntoken, err := tok.Value()
	if err != nil {
		log.Fatalln(err)
	}

	// use the ntoken to talk to Athenz
	client := zts.NewClient(ztsUrl, nil)
	client.AddCredentials(hdr, ntoken)

	// get our private key signer for csr
	pkSigner, err := newSigner(bytes)
	if err != nil {
		log.Fatalln(err)
	}

	// generate a csr for this service

	hyphenDomain := strings.Replace(domain, ".", "-", -1)
	host := fmt.Sprintf("%s.%s.%s", service, hyphenDomain, dnsDomain)
	commonName := fmt.Sprintf("%s.%s", domain, service)
	csr, err := generateCSR(pkSigner, commonName, host)
	if err != nil {
		log.Fatalln(err)
	}
	req := &zts.InstanceRefreshRequest{Csr: csr}

	// request a tls certificate for this service
	identity, err := client.PostInstanceRefreshRequest(zts.CompoundName(domain), zts.SimpleName(service), req)
	if err != nil {
		log.Fatalln(err)
	}

	if certFile != "" {
		err = ioutil.WriteFile(certFile, []byte(identity.Certificate), 0444)
		if err != nil {
			log.Fatalln(err)
		}
	} else {
		fmt.Println(identity.Certificate)
	}
}

func newSigner(privateKeyPEM []byte) (*signer, error) {
	block, _ := pem.Decode(privateKeyPEM)
	if block == nil {
		return nil, fmt.Errorf("Unable to load private key")
	}

	switch block.Type {
	case "EC PRIVATE KEY":
		key, err := x509.ParseECPrivateKey(block.Bytes)
		if err != nil {
			return nil, err
		}
		return &signer{key: key}, nil
	case "RSA PRIVATE KEY":
		key, err := x509.ParsePKCS1PrivateKey(block.Bytes)
		if err != nil {
			return nil, err
		}
		return &signer{key: key}, nil
	default:
		return nil, fmt.Errorf("Unsupported private key type: %s", block.Type)
	}
}

func generateCSR(keySigner *signer, commonName, host string) (string, error) {
	//note: RFC 6125 states that if the SAN (Subject Alternative Name) exists,
	//it is used, not the CA. So, we will always put the Athenz name in the CN
	//(it is *not* a DNS domain name), and put the host name into the SAN.
	subj := pkix.Name{CommonName: commonName}
	subj.OrganizationalUnit = []string{"Athenz"}

	template := x509.CertificateRequest{
		Subject:            subj,
		SignatureAlgorithm: x509.SHA256WithRSA,
	}
	if host != "" {
		template.DNSNames = []string{host}
	}
	csr, err := x509.CreateCertificateRequest(rand.Reader, &template, keySigner.key)
	if err != nil {
		return "", fmt.Errorf("Cannot create CSR: %v", err)
	}
	block := &pem.Block{
		Type:  "CERTIFICATE REQUEST",
		Bytes: csr,
	}
	var buf bytes.Buffer
	err = pem.Encode(&buf, block)
	if err != nil {
		return "", fmt.Errorf("Cannot encode CSR to PEM: %v", err)
	}
	return buf.String(), nil
}
<commit_msg>Make hdr field configurable<commit_after>package main

import (
	"bytes"
	"crypto"
	"crypto/rand"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/pem"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"strings"
	"time"

	"github.com/yahoo/athenz/clients/go/zts"
	"github.com/yahoo/athenz/libs/go/zmssvctoken"
)

type signer struct {
	key crypto.Signer
}

func main() {

	var ztsUrl, privateKeyFile, domain, service, keyVersion, certFile, dnsDomain, hdr string
	flag.StringVar(&certFile, "cert-file", "", "output certificate file")
	flag.StringVar(&privateKeyFile, "private-key", "", "private key file")
	flag.StringVar(&domain, "domain", "", "domain of service")
	flag.StringVar(&service, "service", "", "name of service")
	flag.StringVar(&keyVersion, "key-version", "", "key version")
	flag.StringVar(&ztsUrl, "zts", "", "url of the ZTS Service")
	flag.StringVar(&dnsDomain, "dns-domain", "", "dns domain suffix to be included in the csr")
	flag.StringVar(&hdr, "hdr", "Athenz-Principal-Auth", "Header name")
	flag.Parse()

	if privateKeyFile == "" || domain == "" || service == "" ||
		keyVersion == "" || ztsUrl == "" || dnsDomain == "" {
		log.Fatalln("usage: zts-svccert -domain <domain> -service <service> -private-key <key-file> -key-version <version> -zts <zts-server-url> -dns-domain <dns-domain> [-cert-file <output-cert-file>]")
	}

	// load private key
	bytes, err := ioutil.ReadFile(privateKeyFile)
	if err != nil {
		log.Fatalln(err)
	}

	// get token builder instance
	builder, err := zmssvctoken.NewTokenBuilder(domain, service, bytes, keyVersion)
	if err != nil {
		log.Fatalln(err)
	}

	// set optional attributes
	builder.SetExpiration(10 * time.Minute)

	// get a token instance that always gives you unexpired tokens values
	// safe for concurrent use
	tok := builder.Token()

	// get a token for use
	ntoken, err := tok.Value()
	if err != nil {
		log.Fatalln(err)
	}

	// use the ntoken to talk to Athenz
	client := zts.NewClient(ztsUrl, nil)
	client.AddCredentials(hdr, ntoken)

	// get our private key signer for csr
	pkSigner, err := newSigner(bytes)
	if err != nil {
		log.Fatalln(err)
	}

	// generate a csr for this service

	hyphenDomain := strings.Replace(domain, ".", "-", -1)
	host := fmt.Sprintf("%s.%s.%s", service, hyphenDomain, dnsDomain)
	commonName := fmt.Sprintf("%s.%s", domain, service)
	csr, err := generateCSR(pkSigner, commonName, host)
	if err != nil {
		log.Fatalln(err)
	}
	req := &zts.InstanceRefreshRequest{Csr: csr}

	// request a tls certificate for this service
	identity, err := client.PostInstanceRefreshRequest(zts.CompoundName(domain), zts.SimpleName(service), req)
	if err != nil {
		log.Fatalln(err)
	}

	if certFile != "" {
		err = ioutil.WriteFile(certFile, []byte(identity.Certificate), 0444)
		if err != nil {
			log.Fatalln(err)
		}
	} else {
		fmt.Println(identity.Certificate)
	}
}

func newSigner(privateKeyPEM []byte) (*signer, error) {
	block, _ := pem.Decode(privateKeyPEM)
	if block == nil {
		return nil, fmt.Errorf("Unable to load private key")
	}

	switch block.Type {
	case "EC PRIVATE KEY":
		key, err := x509.ParseECPrivateKey(block.Bytes)
		if err != nil {
			return nil, err
		}
		return &signer{key: key}, nil
	case "RSA PRIVATE KEY":
		key, err := x509.ParsePKCS1PrivateKey(block.Bytes)
		if err != nil {
			return nil, err
		}
		return &signer{key: key}, nil
	default:
		return nil, fmt.Errorf("Unsupported private key type: %s", block.Type)
	}
}

func generateCSR(keySigner *signer, commonName, host string) (string, error) {
	//note: RFC 6125 states that if the SAN (Subject Alternative Name) exists,
	//it is used, not the CA. So, we will always put the Athenz name in the CN
	//(it is *not* a DNS domain name), and put the host name into the SAN.
	subj := pkix.Name{CommonName: commonName}
	subj.OrganizationalUnit = []string{"Athenz"}

	template := x509.CertificateRequest{
		Subject:            subj,
		SignatureAlgorithm: x509.SHA256WithRSA,
	}
	if host != "" {
		template.DNSNames = []string{host}
	}
	csr, err := x509.CreateCertificateRequest(rand.Reader, &template, keySigner.key)
	if err != nil {
		return "", fmt.Errorf("Cannot create CSR: %v", err)
	}
	block := &pem.Block{
		Type:  "CERTIFICATE REQUEST",
		Bytes: csr,
	}
	var buf bytes.Buffer
	err = pem.Encode(&buf, block)
	if err != nil {
		return "", fmt.Errorf("Cannot encode CSR to PEM: %v", err)
	}
	return buf.String(), nil
}
<|endoftext|>


<commit_before>package btcwallet

import (
	"github.com/btcsuite/btcd/btcec"
	"github.com/btcsuite/btcd/chaincfg/chainhash"
	"github.com/btcsuite/btcd/txscript"
	"github.com/btcsuite/btcd/wire"
	"github.com/btcsuite/btcutil"
	"github.com/btcsuite/btcwallet/waddrmgr"
	base "github.com/btcsuite/btcwallet/wallet"
	"github.com/btcsuite/btcwallet/walletdb"
	"github.com/go-errors/errors"
	"github.com/lightningnetwork/lnd/keychain"
	"github.com/lightningnetwork/lnd/lnwallet"
)

// FetchInputInfo queries for the WalletController's knowledge of the passed
// outpoint. If the base wallet determines this output is under its control,
// then the original txout should be returned. Otherwise, a non-nil error value
// of ErrNotMine should be returned instead.
//
// This is a part of the WalletController interface.
func (b *BtcWallet) FetchInputInfo(prevOut *wire.OutPoint) (*wire.TxOut, error) {
	var (
		err    error
		output *wire.TxOut
	)

	// First check to see if the output is already within the utxo cache.
	// If so we can return directly saving a disk access.
	b.cacheMtx.RLock()
	if output, ok := b.utxoCache[*prevOut]; ok {
		b.cacheMtx.RUnlock()
		return output, nil
	}
	b.cacheMtx.RUnlock()

	// Otherwise, we manually look up the output within the tx store.
	txid := &prevOut.Hash
	txDetail, err := base.UnstableAPI(b.wallet).TxDetails(txid)
	if err != nil {
		return nil, err
	} else if txDetail == nil {
		return nil, lnwallet.ErrNotMine
	}

	output = txDetail.TxRecord.MsgTx.TxOut[prevOut.Index]

	b.cacheMtx.Lock()
	b.utxoCache[*prevOut] = output
	b.cacheMtx.Unlock()

	return output, nil
}

// fetchOutputAddr attempts to fetch the managed address corresponding to the
// passed output script. This function is used to look up the proper key which
// should be used to sign a specified input.
func (b *BtcWallet) fetchOutputAddr(script []byte) (waddrmgr.ManagedAddress, error) {
	_, addrs, _, err := txscript.ExtractPkScriptAddrs(script, b.netParams)
	if err != nil {
		return nil, err
	}

	// If the case of a multi-sig output, several address may be extracted.
	// Therefore, we simply select the key for the first address we know
	// of.
	for _, addr := range addrs {
		addr, err := b.wallet.AddressInfo(addr)
		if err == nil {
			return addr, nil
		}
	}

	return nil, errors.Errorf("address not found")
}

// fetchPrivKey attempts to retrieve the raw private key corresponding to the
// passed public key if populated, or the key descriptor path (if non-empty).
func (b *BtcWallet) fetchPrivKey(keyDesc *keychain.KeyDescriptor) (*btcec.PrivateKey, error) {
	// If the key locator within the descriptor *isn't* empty, then we can
	// directly derive the keys raw.
	if !keyDesc.KeyLocator.IsEmpty() {
		// We'll assume the special lightning key scope in this case.
		scopedMgr, err := b.wallet.Manager.FetchScopedKeyManager(
			b.chainKeyScope,
		)
		if err != nil {
			return nil, err
		}

		var key *btcec.PrivateKey
		err = walletdb.View(b.db, func(tx walletdb.ReadTx) error {
			addrmgrNs := tx.ReadBucket(waddrmgrNamespaceKey)

			path := waddrmgr.DerivationPath{
				Account: uint32(keyDesc.Family),
				Branch:  0,
				Index:   uint32(keyDesc.Index),
			}
			addr, err := scopedMgr.DeriveFromKeyPath(addrmgrNs, path)
			if err != nil {
				return err
			}

			key, err = addr.(waddrmgr.ManagedPubKeyAddress).PrivKey()
			return err
		})
		if err != nil {
			return nil, err
		}

		return key, nil
	}

	hash160 := btcutil.Hash160(keyDesc.PubKey.SerializeCompressed())
	addr, err := btcutil.NewAddressWitnessPubKeyHash(hash160, b.netParams)
	if err != nil {
		return nil, err
	}

	return b.wallet.PrivKeyForAddress(addr)
}

// maybeTweakPrivKey examines the single and double tweak parameters on the
// passed sign descriptor and may perform a mapping on the passed private key
// in order to utilize the tweaks, if populated.
func maybeTweakPrivKey(signDesc *lnwallet.SignDescriptor,
	privKey *btcec.PrivateKey) (*btcec.PrivateKey, error) {

	var retPriv *btcec.PrivateKey
	switch {

	case signDesc.SingleTweak != nil:
		retPriv = lnwallet.TweakPrivKey(privKey,
			signDesc.SingleTweak)

	case signDesc.DoubleTweak != nil:
		retPriv = lnwallet.DeriveRevocationPrivKey(privKey,
			signDesc.DoubleTweak)

	default:
		retPriv = privKey
	}

	return retPriv, nil
}

// SignOutputRaw generates a signature for the passed transaction according to
// the data within the passed SignDescriptor.
//
// This is a part of the WalletController interface.
func (b *BtcWallet) SignOutputRaw(tx *wire.MsgTx,
	signDesc *lnwallet.SignDescriptor) ([]byte, error) {

	witnessScript := signDesc.WitnessScript

	// First attempt to fetch the private key which corresponds to the
	// specified public key.
	privKey, err := b.fetchPrivKey(&signDesc.KeyDesc)
	if err != nil {
		return nil, err
	}

	// If a tweak (single or double) is specified, then we'll need to use
	// this tweak to derive the final private key to be used for signing
	// this output.
	privKey, err = maybeTweakPrivKey(signDesc, privKey)
	if err != nil {
		return nil, err
	}

	// TODO(roasbeef): generate sighash midstate if not present?

	amt := signDesc.Output.Value
	sig, err := txscript.RawTxInWitnessSignature(tx, signDesc.SigHashes,
		signDesc.InputIndex, amt, witnessScript, signDesc.HashType,
		privKey)
	if err != nil {
		return nil, err
	}

	// Chop off the sighash flag at the end of the signature.
	return sig[:len(sig)-1], nil
}

// ComputeInputScript generates a complete InputScript for the passed
// transaction with the signature as defined within the passed SignDescriptor.
// This method is capable of generating the proper input script for both
// regular p2wkh output and p2wkh outputs nested within a regular p2sh output.
//
// This is a part of the WalletController interface.
func (b *BtcWallet) ComputeInputScript(tx *wire.MsgTx,
	signDesc *lnwallet.SignDescriptor) (*lnwallet.InputScript, error) {

	outputScript := signDesc.Output.PkScript
	walletAddr, err := b.fetchOutputAddr(outputScript)
	if err != nil {
		return nil, nil
	}

	pka := walletAddr.(waddrmgr.ManagedPubKeyAddress)
	privKey, err := pka.PrivKey()
	if err != nil {
		return nil, err
	}

	var witnessProgram []byte
	inputScript := &lnwallet.InputScript{}

	switch {

	// If we're spending p2wkh output nested within a p2sh output, then
	// we'll need to attach a sigScript in addition to witness data.
	case pka.AddrType() == waddrmgr.NestedWitnessPubKey:
		pubKey := privKey.PubKey()
		pubKeyHash := btcutil.Hash160(pubKey.SerializeCompressed())

		// Next, we'll generate a valid sigScript that will allow us to
		// spend the p2sh output. The sigScript will contain only a
		// single push of the p2wkh witness program corresponding to
		// the matching public key of this address.
		p2wkhAddr, err := btcutil.NewAddressWitnessPubKeyHash(pubKeyHash,
			b.netParams)
		if err != nil {
			return nil, err
		}
		witnessProgram, err = txscript.PayToAddrScript(p2wkhAddr)
		if err != nil {
			return nil, err
		}

		bldr := txscript.NewScriptBuilder()
		bldr.AddData(witnessProgram)
		sigScript, err := bldr.Script()
		if err != nil {
			return nil, err
		}

		inputScript.ScriptSig = sigScript

	// Otherwise, this is a regular p2wkh output, so we include the
	// witness program itself as the subscript to generate the proper
	// sighash digest. As part of the new sighash digest algorithm, the
	// p2wkh witness program will be expanded into a regular p2kh
	// script.
	default:
		witnessProgram = outputScript
	}

	// If a tweak (single or double) is specified, then we'll need to use
	// this tweak to derive the final private key to be used for signing
	// this output.
	privKey, err = maybeTweakPrivKey(signDesc, privKey)
	if err != nil {
		return nil, err
	}

	// Generate a valid witness stack for the input.
	// TODO(roasbeef): adhere to passed HashType
	witnessScript, err := txscript.WitnessSignature(tx, signDesc.SigHashes,
		signDesc.InputIndex, signDesc.Output.Value, witnessProgram,
		signDesc.HashType, privKey, true)
	if err != nil {
		return nil, err
	}

	inputScript.Witness = witnessScript

	return inputScript, nil
}

// A compile time check to ensure that BtcWallet implements the Signer
// interface.
var _ lnwallet.Signer = (*BtcWallet)(nil)

// SignMessage attempts to sign a target message with the private key that
// corresponds to the passed public key. If the target private key is unable to
// be found, then an error will be returned. The actual digest signed is the
// double SHA-256 of the passed message.
//
// NOTE: This is a part of the MessageSigner interface.
func (b *BtcWallet) SignMessage(pubKey *btcec.PublicKey,
	msg []byte) (*btcec.Signature, error) {

	// First attempt to fetch the private key which corresponds to the
	// specified public key.
	privKey, err := b.fetchPrivKey(&keychain.KeyDescriptor{
		PubKey: pubKey,
	})
	if err != nil {
		return nil, err
	}

	// Double hash and sign the data.
	msgDigest := chainhash.DoubleHashB(msg)
	sign, err := privKey.Sign(msgDigest)
	if err != nil {
		return nil, errors.Errorf("unable sign the message: %v", err)
	}

	return sign, nil
}

// A compile time check to ensure that BtcWallet implements the MessageSigner
// interface.
var _ lnwallet.MessageSigner = (*BtcWallet)(nil)
<commit_msg>lnwallet/btcwallet: check output is under our control in FetchInputInfo<commit_after>package btcwallet

import (
	"github.com/btcsuite/btcd/btcec"
	"github.com/btcsuite/btcd/chaincfg/chainhash"
	"github.com/btcsuite/btcd/txscript"
	"github.com/btcsuite/btcd/wire"
	"github.com/btcsuite/btcutil"
	"github.com/btcsuite/btcwallet/waddrmgr"
	base "github.com/btcsuite/btcwallet/wallet"
	"github.com/btcsuite/btcwallet/walletdb"
	"github.com/go-errors/errors"
	"github.com/lightningnetwork/lnd/keychain"
	"github.com/lightningnetwork/lnd/lnwallet"
)

// FetchInputInfo queries for the WalletController's knowledge of the passed
// outpoint. If the base wallet determines this output is under its control,
// then the original txout should be returned. Otherwise, a non-nil error value
// of ErrNotMine should be returned instead.
//
// This is a part of the WalletController interface.
func (b *BtcWallet) FetchInputInfo(prevOut *wire.OutPoint) (*wire.TxOut, error) {
	var (
		err    error
		output *wire.TxOut
	)

	// First check to see if the output is already within the utxo cache.
	// If so we can return directly saving a disk access.
	b.cacheMtx.RLock()
	if output, ok := b.utxoCache[*prevOut]; ok {
		b.cacheMtx.RUnlock()
		return output, nil
	}
	b.cacheMtx.RUnlock()

	// Otherwise, we manually look up the output within the tx store.
	txid := &prevOut.Hash
	txDetail, err := base.UnstableAPI(b.wallet).TxDetails(txid)
	if err != nil {
		return nil, err
	} else if txDetail == nil {
		return nil, lnwallet.ErrNotMine
	}

	// With the output retrieved, we'll make an additional check to ensure
	// we actually have control of this output. We do this because the check
	// above only guarantees that the transaction is somehow relevant to us,
	// like in the event of us being the sender of the transaction.
	output = txDetail.TxRecord.MsgTx.TxOut[prevOut.Index]
	if _, err := b.fetchOutputAddr(output.PkScript); err != nil {
		return nil, err
	}

	b.cacheMtx.Lock()
	b.utxoCache[*prevOut] = output
	b.cacheMtx.Unlock()

	return output, nil
}

// fetchOutputAddr attempts to fetch the managed address corresponding to the
// passed output script. This function is used to look up the proper key which
// should be used to sign a specified input.
func (b *BtcWallet) fetchOutputAddr(script []byte) (waddrmgr.ManagedAddress, error) {
	_, addrs, _, err := txscript.ExtractPkScriptAddrs(script, b.netParams)
	if err != nil {
		return nil, err
	}

	// If the case of a multi-sig output, several address may be extracted.
	// Therefore, we simply select the key for the first address we know
	// of.
	for _, addr := range addrs {
		addr, err := b.wallet.AddressInfo(addr)
		if err == nil {
			return addr, nil
		}
	}

	return nil, lnwallet.ErrNotMine
}

// fetchPrivKey attempts to retrieve the raw private key corresponding to the
// passed public key if populated, or the key descriptor path (if non-empty).
func (b *BtcWallet) fetchPrivKey(keyDesc *keychain.KeyDescriptor) (*btcec.PrivateKey, error) {
	// If the key locator within the descriptor *isn't* empty, then we can
	// directly derive the keys raw.
	if !keyDesc.KeyLocator.IsEmpty() {
		// We'll assume the special lightning key scope in this case.
		scopedMgr, err := b.wallet.Manager.FetchScopedKeyManager(
			b.chainKeyScope,
		)
		if err != nil {
			return nil, err
		}

		var key *btcec.PrivateKey
		err = walletdb.View(b.db, func(tx walletdb.ReadTx) error {
			addrmgrNs := tx.ReadBucket(waddrmgrNamespaceKey)

			path := waddrmgr.DerivationPath{
				Account: uint32(keyDesc.Family),
				Branch:  0,
				Index:   uint32(keyDesc.Index),
			}
			addr, err := scopedMgr.DeriveFromKeyPath(addrmgrNs, path)
			if err != nil {
				return err
			}

			key, err = addr.(waddrmgr.ManagedPubKeyAddress).PrivKey()
			return err
		})
		if err != nil {
			return nil, err
		}

		return key, nil
	}

	hash160 := btcutil.Hash160(keyDesc.PubKey.SerializeCompressed())
	addr, err := btcutil.NewAddressWitnessPubKeyHash(hash160, b.netParams)
	if err != nil {
		return nil, err
	}

	return b.wallet.PrivKeyForAddress(addr)
}

// maybeTweakPrivKey examines the single and double tweak parameters on the
// passed sign descriptor and may perform a mapping on the passed private key
// in order to utilize the tweaks, if populated.
func maybeTweakPrivKey(signDesc *lnwallet.SignDescriptor,
	privKey *btcec.PrivateKey) (*btcec.PrivateKey, error) {

	var retPriv *btcec.PrivateKey
	switch {

	case signDesc.SingleTweak != nil:
		retPriv = lnwallet.TweakPrivKey(privKey,
			signDesc.SingleTweak)

	case signDesc.DoubleTweak != nil:
		retPriv = lnwallet.DeriveRevocationPrivKey(privKey,
			signDesc.DoubleTweak)

	default:
		retPriv = privKey
	}

	return retPriv, nil
}

// SignOutputRaw generates a signature for the passed transaction according to
// the data within the passed SignDescriptor.
//
// This is a part of the WalletController interface.
func (b *BtcWallet) SignOutputRaw(tx *wire.MsgTx,
	signDesc *lnwallet.SignDescriptor) ([]byte, error) {

	witnessScript := signDesc.WitnessScript

	// First attempt to fetch the private key which corresponds to the
	// specified public key.
	privKey, err := b.fetchPrivKey(&signDesc.KeyDesc)
	if err != nil {
		return nil, err
	}

	// If a tweak (single or double) is specified, then we'll need to use
	// this tweak to derive the final private key to be used for signing
	// this output.
	privKey, err = maybeTweakPrivKey(signDesc, privKey)
	if err != nil {
		return nil, err
	}

	// TODO(roasbeef): generate sighash midstate if not present?

	amt := signDesc.Output.Value
	sig, err := txscript.RawTxInWitnessSignature(tx, signDesc.SigHashes,
		signDesc.InputIndex, amt, witnessScript, signDesc.HashType,
		privKey)
	if err != nil {
		return nil, err
	}

	// Chop off the sighash flag at the end of the signature.
	return sig[:len(sig)-1], nil
}

// ComputeInputScript generates a complete InputScript for the passed
// transaction with the signature as defined within the passed SignDescriptor.
// This method is capable of generating the proper input script for both
// regular p2wkh output and p2wkh outputs nested within a regular p2sh output.
//
// This is a part of the WalletController interface.
func (b *BtcWallet) ComputeInputScript(tx *wire.MsgTx,
	signDesc *lnwallet.SignDescriptor) (*lnwallet.InputScript, error) {

	outputScript := signDesc.Output.PkScript
	walletAddr, err := b.fetchOutputAddr(outputScript)
	if err != nil {
		return nil, err
	}

	pka := walletAddr.(waddrmgr.ManagedPubKeyAddress)
	privKey, err := pka.PrivKey()
	if err != nil {
		return nil, err
	}

	var witnessProgram []byte
	inputScript := &lnwallet.InputScript{}

	switch {

	// If we're spending p2wkh output nested within a p2sh output, then
	// we'll need to attach a sigScript in addition to witness data.
	case pka.AddrType() == waddrmgr.NestedWitnessPubKey:
		pubKey := privKey.PubKey()
		pubKeyHash := btcutil.Hash160(pubKey.SerializeCompressed())

		// Next, we'll generate a valid sigScript that will allow us to
		// spend the p2sh output. The sigScript will contain only a
		// single push of the p2wkh witness program corresponding to
		// the matching public key of this address.
		p2wkhAddr, err := btcutil.NewAddressWitnessPubKeyHash(pubKeyHash,
			b.netParams)
		if err != nil {
			return nil, err
		}
		witnessProgram, err = txscript.PayToAddrScript(p2wkhAddr)
		if err != nil {
			return nil, err
		}

		bldr := txscript.NewScriptBuilder()
		bldr.AddData(witnessProgram)
		sigScript, err := bldr.Script()
		if err != nil {
			return nil, err
		}

		inputScript.ScriptSig = sigScript

	// Otherwise, this is a regular p2wkh output, so we include the
	// witness program itself as the subscript to generate the proper
	// sighash digest. As part of the new sighash digest algorithm, the
	// p2wkh witness program will be expanded into a regular p2kh
	// script.
	default:
		witnessProgram = outputScript
	}

	// If a tweak (single or double) is specified, then we'll need to use
	// this tweak to derive the final private key to be used for signing
	// this output.
	privKey, err = maybeTweakPrivKey(signDesc, privKey)
	if err != nil {
		return nil, err
	}

	// Generate a valid witness stack for the input.
	// TODO(roasbeef): adhere to passed HashType
	witnessScript, err := txscript.WitnessSignature(tx, signDesc.SigHashes,
		signDesc.InputIndex, signDesc.Output.Value, witnessProgram,
		signDesc.HashType, privKey, true)
	if err != nil {
		return nil, err
	}

	inputScript.Witness = witnessScript

	return inputScript, nil
}

// A compile time check to ensure that BtcWallet implements the Signer
// interface.
var _ lnwallet.Signer = (*BtcWallet)(nil)

// SignMessage attempts to sign a target message with the private key that
// corresponds to the passed public key. If the target private key is unable to
// be found, then an error will be returned. The actual digest signed is the
// double SHA-256 of the passed message.
//
// NOTE: This is a part of the MessageSigner interface.
func (b *BtcWallet) SignMessage(pubKey *btcec.PublicKey,
	msg []byte) (*btcec.Signature, error) {

	// First attempt to fetch the private key which corresponds to the
	// specified public key.
	privKey, err := b.fetchPrivKey(&keychain.KeyDescriptor{
		PubKey: pubKey,
	})
	if err != nil {
		return nil, err
	}

	// Double hash and sign the data.
	msgDigest := chainhash.DoubleHashB(msg)
	sign, err := privKey.Sign(msgDigest)
	if err != nil {
		return nil, errors.Errorf("unable sign the message: %v", err)
	}

	return sign, nil
}

// A compile time check to ensure that BtcWallet implements the MessageSigner
// interface.
var _ lnwallet.MessageSigner = (*BtcWallet)(nil)
<|endoftext|>


<commit_before>package cli

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strconv"
	"strings"

	"github.com/flynn/flynn/Godeps/_workspace/src/github.com/flynn/go-docopt"
	"github.com/flynn/flynn/Godeps/_workspace/src/gopkg.in/inconshreveable/log15.v2"
	"github.com/flynn/flynn/pkg/cluster"
)

var flynnHostLogs = map[string]string{
	"flynn-host.log": "/var/log/flynn/flynn-host.log",

	// the following two entries are legacy paths from when flynn-host used
	// to log to stdout (which would be redirected to these files)
	"upstart-flynn-host.log": "/var/log/upstart/flynn-host.log",
	"tmp-flynn-host.log":     "/tmp/flynn-host.log",
}

var debugCmds = [][]string{
	{"ps", "faux"},
	{"ifconfig"},
	{"uname", "-a"},
	{"lsb_release", "-a"},
	{"date"},
	{"free", "-m"},
	{"df", "-h"},
	{"dpkg-query", "-W", "-f", "${Package}: ${Version}\n", "libvirt-bin"},
	{os.Args[0], "version"},
	{"virsh", "-c", "lxc:///", "list"},
	{"virsh", "-c", "lxc:///", "net-list"},
	{"route", "-n"},
	{"iptables", "-L", "-v", "-n", "--line-numbers"},
}

func init() {
	Register("collect-debug-info", runCollectDebugInfo, `
usage: flynn-host collect-debug-info [options]

Options:
  --tarball          Create a tarball instead of uploading to a gist
  --include-env      Include sensitive environment variables
  --job-lines=<num>  Maximum number of lines to fetch from each job [default: 1000]

Collect debug information into an anonymous gist or tarball`)
}

func runCollectDebugInfo(args *docopt.Args) error {
	log := log15.New()
	lines, err := strconv.Atoi(args.String["--job-lines"])
	if err != nil {
		return err
	}

	if args.Bool["--tarball"] {
		log.Info("creating a tarball containing logs and debug information")
	} else {
		log.Info("uploading logs and debug information to a private, anonymous gist")
	}
	log.Info("this may take a while depending on the size of your logs")

	gist := &Gist{
		Description: "Flynn debug information",
		Public:      false,
		Files:       make(map[string]File),
	}

	log.Info("getting flynn-host logs")
	for name, filepath := range flynnHostLogs {
		if err := gist.AddLocalFile(name, filepath); err != nil && !os.IsNotExist(err) {
			log.Error(fmt.Sprintf("error getting flynn-host log %q", filepath), "err", err)
		}
	}

	log.Info("getting job logs")
	if err := captureJobs(gist, args.Bool["--include-env"], lines); err != nil {
		log.Error("error getting job logs, falling back to on-disk logs", "err", err)
		debugCmds = append(debugCmds, []string{"bash", "-c", fmt.Sprintf("tail -n %d /var/log/flynn/*.log", lines)})
	}

	log.Info("getting system information")
	var debugOutput string
	for _, cmd := range debugCmds {
		output, err := captureCmd(cmd[0], cmd[1:]...)
		if err != nil {
			log.Error(fmt.Sprintf("error capturing output of %q", strings.Join(cmd, " ")), "err", err)
			continue
		}
		debugOutput += fmt.Sprintln("===>", strings.Replace(strings.Join(cmd, " "), "\n", `\n`, -1), "\n", output)
	}
	gist.AddFile("0-debug-output.log", debugOutput)

	if args.Bool["--tarball"] {
		path, err := gist.CreateTarball()
		if err != nil {
			log.Error("error creating tarball", "err", err)
			return err
		}
		log.Info(fmt.Sprintf("created tarball containing debug information at %s", path))
		return nil
	}

	if err := gist.Upload(log); err != nil {
		return err
	}

	log.Info(fmt.Sprintf("debug information uploaded to: %s", gist.URL))
	return nil
}

func captureCmd(name string, arg ...string) (string, error) {
	var buf bytes.Buffer
	c := exec.Command(name, arg...)
	c.Stdout = &buf
	if err := c.Run(); err != nil {
		return "", err
	}
	return buf.String(), nil
}

func captureJobs(gist *Gist, env bool, lines int) error {
	client := cluster.NewClient()

	jobs, err := jobList(client, true)
	if err != nil {
		return err
	}

	var buf bytes.Buffer
	printJobs(jobs, &buf)
	gist.AddFile("1-jobs.log", buf.String())

	for _, job := range jobs {
		var name string
		if app, ok := job.Job.Metadata["flynn-controller.app_name"]; ok {
			name += app + "-"
		}
		if typ, ok := job.Job.Metadata["flynn-controller.type"]; ok {
			name += typ + "-"
		}
		name += job.Job.ID + ".log"

		var content bytes.Buffer
		printJobDesc(&job, &content, env)
		fmt.Fprint(&content, "\n\n***** ***** ***** ***** ***** ***** ***** ***** ***** *****\n\n")
		stdoutR, stdoutW := io.Pipe()
		stderrR, stderrW := io.Pipe()
		go func() {
			getLog(job.HostID, job.Job.ID, client, false, true, stdoutW, stderrW)
			stdoutW.Close()
			stderrW.Close()
		}()
		tailLogs(stdoutR, stderrR, lines, &content, &content)

		gist.AddFile(name, content.String())
	}

	return nil
}
<commit_msg>host/cli: Only collect system app logs<commit_after>package cli

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strconv"
	"strings"

	"github.com/flynn/flynn/Godeps/_workspace/src/github.com/flynn/go-docopt"
	"github.com/flynn/flynn/Godeps/_workspace/src/gopkg.in/inconshreveable/log15.v2"
	"github.com/flynn/flynn/pkg/cluster"
)

var flynnHostLogs = map[string]string{
	"flynn-host.log": "/var/log/flynn/flynn-host.log",

	// the following two entries are legacy paths from when flynn-host used
	// to log to stdout (which would be redirected to these files)
	"upstart-flynn-host.log": "/var/log/upstart/flynn-host.log",
	"tmp-flynn-host.log":     "/tmp/flynn-host.log",
}

var debugCmds = [][]string{
	{"ps", "faux"},
	{"ifconfig"},
	{"uname", "-a"},
	{"lsb_release", "-a"},
	{"date"},
	{"free", "-m"},
	{"df", "-h"},
	{"dpkg-query", "-W", "-f", "${Package}: ${Version}\n", "libvirt-bin"},
	{os.Args[0], "version"},
	{"virsh", "-c", "lxc:///", "list"},
	{"virsh", "-c", "lxc:///", "net-list"},
	{"route", "-n"},
	{"iptables", "-L", "-v", "-n", "--line-numbers"},
}

func init() {
	Register("collect-debug-info", runCollectDebugInfo, `
usage: flynn-host collect-debug-info [options]

Options:
  --tarball          Create a tarball instead of uploading to a gist
  --include-env      Include sensitive environment variables
  --job-lines=<num>  Maximum number of lines to fetch from each job [default: 1000]

Collect debug information into an anonymous gist or tarball`)
}

func runCollectDebugInfo(args *docopt.Args) error {
	log := log15.New()
	lines, err := strconv.Atoi(args.String["--job-lines"])
	if err != nil {
		return err
	}

	if args.Bool["--tarball"] {
		log.Info("creating a tarball containing logs and debug information")
	} else {
		log.Info("uploading logs and debug information to a private, anonymous gist")
	}
	log.Info("this may take a while depending on the size of your logs")

	gist := &Gist{
		Description: "Flynn debug information",
		Public:      false,
		Files:       make(map[string]File),
	}

	log.Info("getting flynn-host logs")
	for name, filepath := range flynnHostLogs {
		if err := gist.AddLocalFile(name, filepath); err != nil && !os.IsNotExist(err) {
			log.Error(fmt.Sprintf("error getting flynn-host log %q", filepath), "err", err)
		}
	}

	log.Info("getting job logs")
	if err := captureJobs(gist, args.Bool["--include-env"], lines); err != nil {
		log.Error("error getting job logs, falling back to on-disk logs", "err", err)
		debugCmds = append(debugCmds, []string{"bash", "-c", fmt.Sprintf("tail -n %d /var/log/flynn/*.log", lines)})
	}

	log.Info("getting system information")
	var debugOutput string
	for _, cmd := range debugCmds {
		output, err := captureCmd(cmd[0], cmd[1:]...)
		if err != nil {
			log.Error(fmt.Sprintf("error capturing output of %q", strings.Join(cmd, " ")), "err", err)
			continue
		}
		debugOutput += fmt.Sprintln("===>", strings.Replace(strings.Join(cmd, " "), "\n", `\n`, -1), "\n", output)
	}
	gist.AddFile("0-debug-output.log", debugOutput)

	if args.Bool["--tarball"] {
		path, err := gist.CreateTarball()
		if err != nil {
			log.Error("error creating tarball", "err", err)
			return err
		}
		log.Info(fmt.Sprintf("created tarball containing debug information at %s", path))
		return nil
	}

	if err := gist.Upload(log); err != nil {
		return err
	}

	log.Info(fmt.Sprintf("debug information uploaded to: %s", gist.URL))
	return nil
}

func captureCmd(name string, arg ...string) (string, error) {
	var buf bytes.Buffer
	c := exec.Command(name, arg...)
	c.Stdout = &buf
	if err := c.Run(); err != nil {
		return "", err
	}
	return buf.String(), nil
}

func captureJobs(gist *Gist, env bool, lines int) error {
	client := cluster.NewClient()

	jobs, err := jobList(client, true)
	if err != nil {
		return err
	}

	var buf bytes.Buffer
	printJobs(jobs, &buf)
	gist.AddFile("1-jobs.log", buf.String())

	for _, job := range jobs {
		var name string
		if system, ok := job.Job.Metadata["flynn-system-app"]; !ok || system != "true" {
			continue // Skip non-system applications
		}
		if app, ok := job.Job.Metadata["flynn-controller.app_name"]; ok {
			name += app + "-"
		}
		if typ, ok := job.Job.Metadata["flynn-controller.type"]; ok {
			name += typ + "-"
		}
		name += job.Job.ID + ".log"

		var content bytes.Buffer
		printJobDesc(&job, &content, env)
		fmt.Fprint(&content, "\n\n***** ***** ***** ***** ***** ***** ***** ***** ***** *****\n\n")
		stdoutR, stdoutW := io.Pipe()
		stderrR, stderrW := io.Pipe()
		go func() {
			getLog(job.HostID, job.Job.ID, client, false, true, stdoutW, stderrW)
			stdoutW.Close()
			stderrW.Close()
		}()
		tailLogs(stdoutR, stderrR, lines, &content, &content)

		gist.AddFile(name, content.String())
	}

	return nil
}
<|endoftext|>


<commit_before>// Twackup -- Backs up your tweets
//
//
// Only works on non-protected accounts. Why would you be on twitter
// if you're not being public?

// http://api.twitter.com/version/statuses/user_timeline.format

package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
)

func FindEndpoints(dir string) (oldest, newest uint64, err error) {
	file, err := os.Open(dir)
	if err != nil {
		return
	}
	defer file.Close()
	for {
		var list []string
		list, err = file.Readdirnames(1000)
		if err != nil {
			return
		}
		if len(list) == 0 {
			break
		}
		for _, name := range list {
			JSON_SUFFIX := ".json"
			if strings.HasSuffix(name, JSON_SUFFIX) {
				name = name[:len(name)-len(JSON_SUFFIX)]
				seen, err := strconv.ParseUint(name, 10, 64)
				if err != nil {
					// it's an unrelated json file?
					continue
				}
				if oldest == 0 || seen < oldest {
					oldest = seen
				}
				if newest == 0 || seen > newest {
					newest = seen
				}
			}
		}
	}
	return
}

// Gets tweets from max_id backwards.
// max_id==0 means latest tweet.
func GetTweets(user string, max_id uint64, since_id uint64) (tweets []map[string]interface{}, err error) {
	args := url.Values{
		"screen_name":      []string{user},
		"trim_user":        []string{"true"},
		"include_rts":      []string{"true"},
		"include_entities": []string{"true"},
		"count":            []string{"200"},
	}
	if max_id != 0 {
		args["max_id"] = []string{strconv.FormatUint(max_id, 10)}
	}
	if since_id != 0 {
		args["since_id"] = []string{strconv.FormatUint(since_id, 10)}
	}
	query := args.Encode()
	url := "http://api.twitter.com/1/statuses/user_timeline.json?" + query
	log.Printf("Fetching url %v\n", url)
	r, err := http.Get(url)
	if err != nil {
		return
	}
	ctype := r.Header.Get("Content-Type")
	if ctype != "application/json; charset=utf-8" {
		return
	}

	//TODO ponder rate-limiting http headers
	//X-Ratelimit-Remaining: 143
	//X-Ratelimit-Limit: 150
	//X-Ratelimit-Reset: 1303182615

	buf, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return
	}
	err = r.Body.Close()
	if err != nil {
		return
	}
	err = json.Unmarshal(buf, &tweets)
	return
}

func IdFromTweet(tweet map[string]interface{}) (id uint64, err error) {
	id_raw := tweet["id_str"]
	id_str, ok := id_raw.(string)
	if !ok {
		msg := fmt.Sprintf("tweet id is not a string: %v", id_raw)
		err = errors.New(msg)
		return
	}

	id, err = strconv.ParseUint(id_str, 10, 64)
	if err != nil {
		return
	}
	return
}

func SaveTweet(dir string, tweet map[string]interface{}) (id uint64, err error) {
	// clean up Twitter's mistakes
	delete(tweet, "id")

	id, err = IdFromTweet(tweet)
	if err != nil {
		return
	}
	out, err := json.MarshalIndent(tweet, "", "  ")
	if err != nil {
		return
	}

	// roundtrip it from number back to string to canonicalize it
	filename := dir + "/" + strconv.FormatUint(id, 10) + ".json"
	tmp := filename + "." + strconv.Itoa(os.Getpid()) + ".tmp"
	f, err := os.Create(tmp)
	if err != nil {
		return
	}
	_, err = f.Write(out)
	if err != nil {
		_ = os.Remove(tmp)
		return
	}
	err = f.Close()
	if err != nil {
		_ = os.Remove(tmp)
		return
	}

	err = os.Rename(tmp, filename)
	if err != nil {
		_ = os.Remove(tmp)
		return
	}
	return
}

// Get tweets backwards in history until end of time (or Twitter API
// limit).
func getOldTweets(dir string, user string, oldest uint64) (error) {
	for {
		var max_id uint64
		if oldest != 0 {
			// max_id is inclusive, so do -1 to avoid duplicating
			// the one we have
			max_id = oldest - 1
		}
		tweets, err := GetTweets(user, max_id, 0)
		if err != nil {
			return err
		}
		if len(tweets) == 0 {
			break
		}
		for _, tweet := range tweets {
			_, err := SaveTweet(dir, tweet)
			if err != nil {
				return err
			}
		}
		oldest, err = IdFromTweet(tweets[len(tweets)-1])
		if err != nil {
			return err
		}
		log.Printf("Saved %d old tweets\n", len(tweets))
	}
	return nil
}

// Get tweets forwards in time; since Twitter gives you the *latest*
// chunk, not the oldest chunk, we need to save these to disk in
// reverse order.
func getNewTweets(dir string, user string, newest uint64) (error) {
	// gather them in RAM
	var tweets []map[string]interface{}
	for {
		var since_id uint64
		if newest != 0 {
			since_id = newest
		}
		chunk, err := GetTweets(user, 0, since_id)
		if err != nil {
			return err
		}
		if len(chunk) == 0 {
			break
		}
		tweets = append(tweets, chunk...)

		newest, err = IdFromTweet(tweets[0])
		if err != nil {
			return err
		}
	}

	// now save them to disk in reverse order
	for i := len(tweets) - 1; i >= 0; i-- {
		tweet := tweets[i]
		_, err := SaveTweet(dir, tweet)
		if err != nil {
			return err
		}
	}
	log.Printf("Saved %d new tweets\n", len(tweets))
	return nil
}

func main() {
	log.SetFlags(0)
	if len(os.Args) != 3 || os.Args[1][0] == '-' {
		log.Printf("%s: usage: %s USER DIR", os.Args[0], os.Args[0])
		os.Exit(2)
	}
	user := os.Args[1]
	dir := os.Args[2]

	oldest, newest, err := FindEndpoints(dir)
	if err != nil {
		log.Fatalf("%s: %s", os.Args[0], err)
	}

	// usually prioritize fetching new tweets first, but not on
	// the first run; it'd load them all to RAM, where as
	// getOldTweets is more efficient
	if newest != 0 {
		log.Printf("Fetching tweets newer than %d\n", newest)
		getNewTweets(dir, user, newest)
	}

	log.Printf("Fetching tweets older than %d\n", oldest)
	getOldTweets(dir, user, oldest)
}
<commit_msg>go1 fix: file.Open().Readdirnames() now gives EOF.<commit_after>// Twackup -- Backs up your tweets
//
//
// Only works on non-protected accounts. Why would you be on twitter
// if you're not being public?

// http://api.twitter.com/version/statuses/user_timeline.format

package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
)

func FindEndpoints(dir string) (oldest, newest uint64, err error) {
	file, err := os.Open(dir)
	if err != nil {
		return
	}
	defer file.Close()
	for {
		var list []string
		list, err = file.Readdirnames(1000)
		if err == io.EOF {
			err = nil
			break
		}
		if err != nil {
			return
		}
		if len(list) == 0 {
			break
		}
		for _, name := range list {
			JSON_SUFFIX := ".json"
			if strings.HasSuffix(name, JSON_SUFFIX) {
				name = name[:len(name)-len(JSON_SUFFIX)]
				seen, err := strconv.ParseUint(name, 10, 64)
				if err != nil {
					// it's an unrelated json file?
					continue
				}
				if oldest == 0 || seen < oldest {
					oldest = seen
				}
				if newest == 0 || seen > newest {
					newest = seen
				}
			}
		}
	}
	return
}

// Gets tweets from max_id backwards.
// max_id==0 means latest tweet.
func GetTweets(user string, max_id uint64, since_id uint64) (tweets []map[string]interface{}, err error) {
	args := url.Values{
		"screen_name":      []string{user},
		"trim_user":        []string{"true"},
		"include_rts":      []string{"true"},
		"include_entities": []string{"true"},
		"count":            []string{"200"},
	}
	if max_id != 0 {
		args["max_id"] = []string{strconv.FormatUint(max_id, 10)}
	}
	if since_id != 0 {
		args["since_id"] = []string{strconv.FormatUint(since_id, 10)}
	}
	query := args.Encode()
	url := "http://api.twitter.com/1/statuses/user_timeline.json?" + query
	log.Printf("Fetching url %v\n", url)
	r, err := http.Get(url)
	if err != nil {
		return
	}
	ctype := r.Header.Get("Content-Type")
	if ctype != "application/json; charset=utf-8" {
		return
	}

	//TODO ponder rate-limiting http headers
	//X-Ratelimit-Remaining: 143
	//X-Ratelimit-Limit: 150
	//X-Ratelimit-Reset: 1303182615

	buf, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return
	}
	err = r.Body.Close()
	if err != nil {
		return
	}
	err = json.Unmarshal(buf, &tweets)
	return
}

func IdFromTweet(tweet map[string]interface{}) (id uint64, err error) {
	id_raw := tweet["id_str"]
	id_str, ok := id_raw.(string)
	if !ok {
		msg := fmt.Sprintf("tweet id is not a string: %v", id_raw)
		err = errors.New(msg)
		return
	}

	id, err = strconv.ParseUint(id_str, 10, 64)
	if err != nil {
		return
	}
	return
}

func SaveTweet(dir string, tweet map[string]interface{}) (id uint64, err error) {
	// clean up Twitter's mistakes
	delete(tweet, "id")

	id, err = IdFromTweet(tweet)
	if err != nil {
		return
	}
	out, err := json.MarshalIndent(tweet, "", "  ")
	if err != nil {
		return
	}

	// roundtrip it from number back to string to canonicalize it
	filename := dir + "/" + strconv.FormatUint(id, 10) + ".json"
	tmp := filename + "." + strconv.Itoa(os.Getpid()) + ".tmp"
	f, err := os.Create(tmp)
	if err != nil {
		return
	}
	_, err = f.Write(out)
	if err != nil {
		_ = os.Remove(tmp)
		return
	}
	err = f.Close()
	if err != nil {
		_ = os.Remove(tmp)
		return
	}

	err = os.Rename(tmp, filename)
	if err != nil {
		_ = os.Remove(tmp)
		return
	}
	return
}

// Get tweets backwards in history until end of time (or Twitter API
// limit).
func getOldTweets(dir string, user string, oldest uint64) (error) {
	for {
		var max_id uint64
		if oldest != 0 {
			// max_id is inclusive, so do -1 to avoid duplicating
			// the one we have
			max_id = oldest - 1
		}
		tweets, err := GetTweets(user, max_id, 0)
		if err != nil {
			return err
		}
		if len(tweets) == 0 {
			break
		}
		for _, tweet := range tweets {
			_, err := SaveTweet(dir, tweet)
			if err != nil {
				return err
			}
		}
		oldest, err = IdFromTweet(tweets[len(tweets)-1])
		if err != nil {
			return err
		}
		log.Printf("Saved %d old tweets\n", len(tweets))
	}
	return nil
}

// Get tweets forwards in time; since Twitter gives you the *latest*
// chunk, not the oldest chunk, we need to save these to disk in
// reverse order.
func getNewTweets(dir string, user string, newest uint64) (error) {
	// gather them in RAM
	var tweets []map[string]interface{}
	for {
		var since_id uint64
		if newest != 0 {
			since_id = newest
		}
		chunk, err := GetTweets(user, 0, since_id)
		if err != nil {
			return err
		}
		if len(chunk) == 0 {
			break
		}
		tweets = append(tweets, chunk...)

		newest, err = IdFromTweet(tweets[0])
		if err != nil {
			return err
		}
	}

	// now save them to disk in reverse order
	for i := len(tweets) - 1; i >= 0; i-- {
		tweet := tweets[i]
		_, err := SaveTweet(dir, tweet)
		if err != nil {
			return err
		}
	}
	log.Printf("Saved %d new tweets\n", len(tweets))
	return nil
}

func main() {
	log.SetFlags(0)
	if len(os.Args) != 3 || os.Args[1][0] == '-' {
		log.Printf("%s: usage: %s USER DIR", os.Args[0], os.Args[0])
		os.Exit(2)
	}
	user := os.Args[1]
	dir := os.Args[2]

	oldest, newest, err := FindEndpoints(dir)
	if err != nil {
		log.Fatalf("%s: Error in handling old tweets: %s", os.Args[0], err)
	}

	// usually prioritize fetching new tweets first, but not on
	// the first run; it'd load them all to RAM, where as
	// getOldTweets is more efficient
	if newest != 0 {
		log.Printf("Fetching tweets newer than %d\n", newest)
		getNewTweets(dir, user, newest)
	}

	log.Printf("Fetching tweets older than %d\n", oldest)
	getOldTweets(dir, user, oldest)
}
<|endoftext|>


<commit_before>package main

import (
    "archive/tar"
    "bufio"
    "compress/gzip"
    "fmt"
    "io"
    "net/http"
    "os"
    "regexp"
    "sort"
    "strconv"
    "strings"
)

const URL = "https://storage.googleapis.com/golang/"

type Interface struct {
    Name       string
    Package    string
    SourceFile string
    LineNumber string
}

type ByName []Interface

func (b ByName) Len() int           {return len(b)}
func (b ByName) Swap(i, j int)      {b[i], b[j] = b[j], b[i]}
func (b ByName) Less(i, j int) bool {return b[i].Name < b[j].Name}

func parseSourceFile(filename string, source io.Reader) []Interface {
    regexpInterface := regexp.MustCompile(`\s*type\s+([A-Z]\w*)\s+interface\s+{`)
    interfaces := make([]Interface, 0)
    reader := bufio.NewReader(source)
    pack := filename[11:strings.LastIndex(filename, "/")]
    lineNumber := 1
    for {
        line, err := reader.ReadBytes('\n')
        if err != nil && err != io.EOF {
            panic("Error parsing source file")
        }
        matches := regexpInterface.FindSubmatch(line)
        if len(matches) > 0 {
            interf := Interface {
                Name:       string(matches[1]),
                Package:    string(pack),
                SourceFile: filename,
                LineNumber: strconv.Itoa(lineNumber),
            }
            interfaces = append(interfaces, interf)
        }
        if err == io.EOF {
            break
        }
        lineNumber += 1
    }
    return interfaces
}

func printInterfaces(interfaces []Interface) {
    lenName := 0
    lenPackage := 0
    lenSourceFile := 0
    lenLineNumber := 0
    for _, i := range interfaces {
        if len(i.Name) > lenName {
            lenName = len(i.Name)
        }
        if len(i.Package) > lenPackage {
            lenPackage = len(i.Package)
        }
        if len(i.SourceFile) > lenSourceFile {
            lenSourceFile = len(i.SourceFile)
        }
        if len(i.LineNumber) > lenLineNumber {
            lenLineNumber = len(i.LineNumber)
        }
    }
    formatLine := "%-" + strconv.Itoa(lenName) + "s  %-" + strconv.Itoa(lenPackage) +
        "s  %-" + strconv.Itoa(lenSourceFile) + "s  %-" + strconv.Itoa(lenLineNumber) +
        "s\n"
    fmt.Printf(formatLine, "Interface", "Package", "Source File", "Line")
    separator := strings.Repeat("-", lenName) + "  " + strings.Repeat("-", lenPackage) +
        "  " + strings.Repeat("-", lenSourceFile) + "  " + strings.Repeat("-", lenLineNumber)
    fmt.Println(separator)
    for _, i := range interfaces {
        fmt.Printf(formatLine, i.Name, i.Package, i.SourceFile, i.LineNumber)
    }
}

func main() {
    // read version on command line
    if len(os.Args) != 2 {
        panic("Must pass go version on command line")
    }
    version := os.Args[1]
    // download compressed archive
    println("Downloading archive...")
    response, err := http.Get(URL+"go"+version+".src.tar.gz")
    if err!=nil {
        panic(err)
    }
    defer response.Body.Close()
    // gunzip the archive stream
    gzipReader, err := gzip.NewReader(response.Body)
    if err != nil {
        panic(err)
    }
    // parse tar source files in go/src/pkg
    println("Parsing archive...")
    tarReader := tar.NewReader(gzipReader)
    interfaces := make([]Interface, 0)
    for {
        header, err := tarReader.Next()
        if err != nil {
            break
        }
        if strings.HasPrefix(header.Name, "go/src/pkg") &&
           strings.HasSuffix(header.Name, ".go") &&
           !strings.HasSuffix(header.Name, "doc.go") {
            newInterfaces := parseSourceFile(header.Name, tarReader)
            interfaces = append(interfaces, newInterfaces...)
        }
    }
    // print the result
    sort.Sort(ByName(interfaces))
    printInterfaces(interfaces)
}
<commit_msg>Added a link to the sources<commit_after>package main

import (
    "archive/tar"
    "bufio"
    "compress/gzip"
    "fmt"
    "io"
    "net/http"
    "os"
    "regexp"
    "sort"
    "strconv"
    "strings"
)

const (
    URL = "https://storage.googleapis.com/golang/"
    SOURCE_URL = "https://code.google.com/p/go/source/browse/"
)

type Interface struct {
    Name       string
    Package    string
    SourceFile string
    LineNumber string
    Link       string
}

type ByName []Interface

func (b ByName) Len() int           {return len(b)}
func (b ByName) Swap(i, j int)      {b[i], b[j] = b[j], b[i]}
func (b ByName) Less(i, j int) bool {return b[i].Name < b[j].Name}

func parseSourceFile(filename string, source io.Reader) []Interface {
    regexpInterface := regexp.MustCompile(`\s*type\s+([A-Z]\w*)\s+interface\s+{`)
    interfaces := make([]Interface, 0)
    reader := bufio.NewReader(source)
    pack := filename[11:strings.LastIndex(filename, "/")]
    lineNumber := 1
    for {
        line, err := reader.ReadBytes('\n')
        if err != nil && err != io.EOF {
            panic("Error parsing source file")
        }
        matches := regexpInterface.FindSubmatch(line)
        if len(matches) > 0 {
            interf := Interface {
                Name:       string(matches[1]),
                Package:    string(pack),
                SourceFile: filename[3:],
                LineNumber: strconv.Itoa(lineNumber),
                Link:       SOURCE_URL+filename[3:]+"?name=release#"+strconv.Itoa(lineNumber),
            }
            interfaces = append(interfaces, interf)
        }
        if err == io.EOF {
            break
        }
        lineNumber += 1
    }
    return interfaces
}

func printInterfaces(interfaces []Interface) {
    lenName := 0
    lenPackage := 0
    lenSourceFile := 0
    lenLineNumber := 0
    for _, i := range interfaces {
        if len(i.Name)+len(i.Link)+4 > lenName {
            lenName = len(i.Name)+len(i.Link)+4
        }
        if len(i.Package) > lenPackage {
            lenPackage = len(i.Package)
        }
        if len(i.SourceFile) > lenSourceFile {
            lenSourceFile = len(i.SourceFile)
        }
        if len(i.LineNumber) > lenLineNumber {
            lenLineNumber = len(i.LineNumber)
        }
     }
    formatLine := "%-" + strconv.Itoa(lenName) + "s  %-" + strconv.Itoa(lenPackage) +
        "s  %-" + strconv.Itoa(lenSourceFile) + "s  %-" + strconv.Itoa(lenLineNumber) +
        "s\n"
    fmt.Printf(formatLine, "Interface", "Package", "Source File", "Line")
    separator := strings.Repeat("-", lenName) + "  " + strings.Repeat("-", lenPackage) +
        "  " + strings.Repeat("-", lenSourceFile) + "  " + strings.Repeat("-", lenLineNumber)
    fmt.Println(separator)
    for _, i := range interfaces {
        link := "["+i.Name+"]("+i.Link+")"
        fmt.Printf(formatLine, link, i.Package, i.SourceFile, i.LineNumber)
    }
}

func main() {
    // read version on command line
    if len(os.Args) != 2 {
        panic("Must pass go version on command line")
    }
    version := os.Args[1]
    // download compressed archive
    println("Downloading archive...")
    response, err := http.Get(URL+"go"+version+".src.tar.gz")
    if err!=nil {
        panic(err)
    }
    defer response.Body.Close()
    // gunzip the archive stream
    gzipReader, err := gzip.NewReader(response.Body)
    if err != nil {
        panic(err)
    }
    // parse tar source files in go/src/pkg
    println("Parsing archive...")
    tarReader := tar.NewReader(gzipReader)
    interfaces := make([]Interface, 0)
    for {
        header, err := tarReader.Next()
        if err != nil {
            break
        }
        if strings.HasPrefix(header.Name, "go/src/pkg") &&
           strings.HasSuffix(header.Name, ".go") &&
           !strings.HasSuffix(header.Name, "doc.go") {
            newInterfaces := parseSourceFile(header.Name, tarReader)
            interfaces = append(interfaces, newInterfaces...)
        }
    }
    // print the result
    sort.Sort(ByName(interfaces))
    printInterfaces(interfaces)
}
<|endoftext|>


<commit_before>// Contains functions that don't really belong anywhere else.

package google

import (
	"fmt"
	"log"
	"sort"
	"strings"

	"github.com/hashicorp/errwrap"
	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
	"google.golang.org/api/googleapi"
)

type TerraformResourceData interface {
	HasChange(string) bool
	GetOkExists(string) (interface{}, bool)
	GetOk(string) (interface{}, bool)
	Get(string) interface{}
	Set(string, interface{}) error
	SetId(string)
	Id() string
}

type TerraformResourceDiff interface {
	HasChange(string) bool
	GetChange(string) (interface{}, interface{})
	Get(string) interface{}
	Clear(string) error
	ForceNew(string) error
}

// getRegionFromZone returns the region from a zone for Google cloud.
func getRegionFromZone(zone string) string {
	if zone != "" && len(zone) > 2 {
		region := zone[:len(zone)-2]
		return region
	}
	return ""
}

// Infers the region based on the following (in order of priority):
// - `region` field in resource schema
// - region extracted from the `zone` field in resource schema
// - provider-level region
// - region extracted from the provider-level zone
func getRegion(d TerraformResourceData, config *Config) (string, error) {
	return getRegionFromSchema("region", "zone", d, config)
}

// getProject reads the "project" field from the given resource data and falls
// back to the provider's value if not given. If the provider's value is not
// given, an error is returned.
func getProject(d TerraformResourceData, config *Config) (string, error) {
	return getProjectFromSchema("project", d, config)
}

// getBillingProject reads the "billing_project" field from the given resource data and falls
// back to the provider's value if not given. If no value is found, an error is returned.
func getBillingProject(d TerraformResourceData, config *Config) (string, error) {
	return getBillingProjectFromSchema("billing_project", d, config)
}

// getProjectFromDiff reads the "project" field from the given diff and falls
// back to the provider's value if not given. If the provider's value is not
// given, an error is returned.
func getProjectFromDiff(d *schema.ResourceDiff, config *Config) (string, error) {
	res, ok := d.GetOk("project")
	if ok {
		return res.(string), nil
	}
	if config.Project != "" {
		return config.Project, nil
	}
	return "", fmt.Errorf("%s: required field is not set", "project")
}

func getRouterLockName(region string, router string) string {
	return fmt.Sprintf("router/%s/%s", region, router)
}

func handleNotFoundError(err error, d *schema.ResourceData, resource string) error {
	if isGoogleApiErrorWithCode(err, 404) {
		log.Printf("[WARN] Removing %s because it's gone", resource)
		// The resource doesn't exist anymore
		d.SetId("")

		return nil
	}

	return errwrap.Wrapf(
		fmt.Sprintf("Error when reading or editing %s: {{err}}", resource), err)
}

func isGoogleApiErrorWithCode(err error, errCode int) bool {
	gerr, ok := errwrap.GetType(err, &googleapi.Error{}).(*googleapi.Error)
	return ok && gerr != nil && gerr.Code == errCode
}

func isApiNotEnabledError(err error) bool {
	gerr, ok := errwrap.GetType(err, &googleapi.Error{}).(*googleapi.Error)
	if !ok {
		return false
	}
	if gerr == nil {
		return false
	}
	if gerr.Code != 403 {
		return false
	}
	for _, e := range gerr.Errors {
		if e.Reason == "accessNotConfigured" {
			return true
		}
	}
	return false
}

func isFailedPreconditionError(err error) bool {
	gerr, ok := errwrap.GetType(err, &googleapi.Error{}).(*googleapi.Error)
	if !ok {
		return false
	}
	if gerr == nil {
		return false
	}
	if gerr.Code != 400 {
		return false
	}
	for _, e := range gerr.Errors {
		if e.Reason == "failedPrecondition" {
			return true
		}
	}
	return false
}

func isConflictError(err error) bool {
	if e, ok := err.(*googleapi.Error); ok && e.Code == 409 {
		return true
	} else if !ok && errwrap.ContainsType(err, &googleapi.Error{}) {
		e := errwrap.GetType(err, &googleapi.Error{}).(*googleapi.Error)
		if e.Code == 409 {
			return true
		}
	}
	return false
}

// expandLabels pulls the value of "labels" out of a TerraformResourceData as a map[string]string.
func expandLabels(d TerraformResourceData) map[string]string {
	return expandStringMap(d, "labels")
}

// expandEnvironmentVariables pulls the value of "environment_variables" out of a schema.ResourceData as a map[string]string.
func expandEnvironmentVariables(d *schema.ResourceData) map[string]string {
	return expandStringMap(d, "environment_variables")
}

// expandStringMap pulls the value of key out of a TerraformResourceData as a map[string]string.
func expandStringMap(d TerraformResourceData, key string) map[string]string {
	v, ok := d.GetOk(key)

	if !ok {
		return map[string]string{}
	}

	return convertStringMap(v.(map[string]interface{}))
}

func convertStringMap(v map[string]interface{}) map[string]string {
	m := make(map[string]string)
	for k, val := range v {
		m[k] = val.(string)
	}
	return m
}

func convertStringArr(ifaceArr []interface{}) []string {
	return convertAndMapStringArr(ifaceArr, func(s string) string { return s })
}

func convertAndMapStringArr(ifaceArr []interface{}, f func(string) string) []string {
	var arr []string
	for _, v := range ifaceArr {
		if v == nil {
			continue
		}
		arr = append(arr, f(v.(string)))
	}
	return arr
}

func mapStringArr(original []string, f func(string) string) []string {
	var arr []string
	for _, v := range original {
		arr = append(arr, f(v))
	}
	return arr
}

func convertStringArrToInterface(strs []string) []interface{} {
	arr := make([]interface{}, len(strs))
	for i, str := range strs {
		arr[i] = str
	}
	return arr
}

func convertStringSet(set *schema.Set) []string {
	s := make([]string, 0, set.Len())
	for _, v := range set.List() {
		s = append(s, v.(string))
	}
	sort.Strings(s)

	return s
}

func golangSetFromStringSlice(strings []string) map[string]struct{} {
	set := map[string]struct{}{}
	for _, v := range strings {
		set[v] = struct{}{}
	}

	return set
}

func stringSliceFromGolangSet(sset map[string]struct{}) []string {
	ls := make([]string, 0, len(sset))
	for s := range sset {
		ls = append(ls, s)
	}
	sort.Strings(ls)

	return ls
}

func reverseStringMap(m map[string]string) map[string]string {
	o := map[string]string{}
	for k, v := range m {
		o[v] = k
	}
	return o
}

func mergeStringMaps(a, b map[string]string) map[string]string {
	merged := make(map[string]string)

	for k, v := range a {
		merged[k] = v
	}

	for k, v := range b {
		merged[k] = v
	}

	return merged
}

func mergeSchemas(a, b map[string]*schema.Schema) map[string]*schema.Schema {
	merged := make(map[string]*schema.Schema)

	for k, v := range a {
		merged[k] = v
	}

	for k, v := range b {
		merged[k] = v
	}

	return merged
}

func mergeResourceMaps(ms ...map[string]*schema.Resource) (map[string]*schema.Resource, error) {
	merged := make(map[string]*schema.Resource)
	duplicates := []string{}

	for _, m := range ms {
		for k, v := range m {
			if _, ok := merged[k]; ok {
				duplicates = append(duplicates, k)
			}

			merged[k] = v
		}
	}

	var err error
	if len(duplicates) > 0 {
		err = fmt.Errorf("saw duplicates in mergeResourceMaps: %v", duplicates)
	}

	return merged, err
}

func extractFirstMapConfig(m []interface{}) map[string]interface{} {
	if len(m) == 0 {
		return map[string]interface{}{}
	}

	return m[0].(map[string]interface{})
}

func lockedCall(lockKey string, f func() error) error {
	mutexKV.Lock(lockKey)
	defer mutexKV.Unlock(lockKey)

	return f()
}

// This is a Printf sibling (Nprintf; Named Printf), which handles strings like
// Nprintf("Hello %{target}!", map[string]interface{}{"target":"world"}) == "Hello world!".
// This is particularly useful for generated tests, where we don't want to use Printf,
// since that would require us to generate a very particular ordering of arguments.
func Nprintf(format string, params map[string]interface{}) string {
	for key, val := range params {
		format = strings.Replace(format, "%{"+key+"}", fmt.Sprintf("%v", val), -1)
	}
	return format
}

// serviceAccountFQN will attempt to generate the fully qualified name in the format of:
// "projects/(-|<project>)/serviceAccounts/<service_account_id>@<project>.iam.gserviceaccount.com"
// A project is required if we are trying to build the FQN from a service account id and
// and error will be returned in this case if no project is set in the resource or the
// provider-level config
func serviceAccountFQN(serviceAccount string, d TerraformResourceData, config *Config) (string, error) {
	// If the service account id is already the fully qualified name
	if strings.HasPrefix(serviceAccount, "projects/") {
		return serviceAccount, nil
	}

	// If the service account id is an email
	if strings.Contains(serviceAccount, "@") {
		return "projects/-/serviceAccounts/" + serviceAccount, nil
	}

	// Get the project from the resource or fallback to the project
	// in the provider configuration
	project, err := getProject(d, config)
	if err != nil {
		return "", err
	}

	return fmt.Sprintf("projects/-/serviceAccounts/%s@%s.iam.gserviceaccount.com", serviceAccount, project), nil
}

func paginatedListRequest(project, baseUrl, userAgent string, config *Config, flattener func(map[string]interface{}) []interface{}) ([]interface{}, error) {
	res, err := sendRequest(config, "GET", project, baseUrl, userAgent, nil)
	if err != nil {
		return nil, err
	}

	ls := flattener(res)
	pageToken, ok := res["pageToken"]
	for ok {
		if pageToken.(string) == "" {
			break
		}
		url := fmt.Sprintf("%s?pageToken=%s", baseUrl, pageToken.(string))
		res, err = sendRequest(config, "GET", project, url, userAgent, nil)
		if err != nil {
			return nil, err
		}
		ls = append(ls, flattener(res))
		pageToken, ok = res["pageToken"]
	}

	return ls, nil
}

func getInterconnectAttachmentLink(config *Config, project, region, ic, userAgent string) (string, error) {
	if !strings.Contains(ic, "/") {
		icData, err := config.NewComputeClient(userAgent).InterconnectAttachments.Get(
			project, region, ic).Do()
		if err != nil {
			return "", fmt.Errorf("Error reading interconnect attachment: %s", err)
		}
		ic = icData.SelfLink
	}

	return ic, nil
}

// Given two sets of references (with "from" values in self link form),
// determine which need to be added or removed // during an update using
// addX/removeX APIs.
func calcAddRemove(from []string, to []string) (add, remove []string) {
	add = make([]string, 0)
	remove = make([]string, 0)
	for _, u := range to {
		found := false
		for _, v := range from {
			if compareSelfLinkOrResourceName("", v, u, nil) {
				found = true
				break
			}
		}
		if !found {
			add = append(add, u)
		}
	}
	for _, u := range from {
		found := false
		for _, v := range to {
			if compareSelfLinkOrResourceName("", u, v, nil) {
				found = true
				break
			}
		}
		if !found {
			remove = append(remove, u)
		}
	}
	return add, remove
}

func stringInSlice(arr []string, str string) bool {
	for _, i := range arr {
		if i == str {
			return true
		}
	}

	return false
}

func migrateStateNoop(v int, is *terraform.InstanceState, meta interface{}) (*terraform.InstanceState, error) {
	return is, nil
}

func expandString(v interface{}, d TerraformResourceData, config *Config) (string, error) {
	return v.(string), nil
}

func changeFieldSchemaToForceNew(sch *schema.Schema) {
	sch.ForceNew = true
	switch sch.Type {
	case schema.TypeList:
	case schema.TypeSet:
		if nestedR, ok := sch.Elem.(*schema.Resource); ok {
			for _, nestedSch := range nestedR.Schema {
				changeFieldSchemaToForceNew(nestedSch)
			}
		}
	}
}

func generateUserAgentString(d *schema.ResourceData, currentUserAgent string) (string, error) {
	var m providerMeta

	err := d.GetProviderMeta(&m)
	if err != nil {
		return currentUserAgent, err
	}

	if m.ModuleName != "" {
		return strings.Join([]string{currentUserAgent, m.ModuleName}, " "), nil
	}

	return currentUserAgent, nil
}
<commit_msg>Add Build Enviroment Vars to Cloud Functions (#4133) (#559)<commit_after>// Contains functions that don't really belong anywhere else.

package google

import (
	"fmt"
	"log"
	"sort"
	"strings"

	"github.com/hashicorp/errwrap"
	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
	"google.golang.org/api/googleapi"
)

type TerraformResourceData interface {
	HasChange(string) bool
	GetOkExists(string) (interface{}, bool)
	GetOk(string) (interface{}, bool)
	Get(string) interface{}
	Set(string, interface{}) error
	SetId(string)
	Id() string
}

type TerraformResourceDiff interface {
	HasChange(string) bool
	GetChange(string) (interface{}, interface{})
	Get(string) interface{}
	Clear(string) error
	ForceNew(string) error
}

// getRegionFromZone returns the region from a zone for Google cloud.
func getRegionFromZone(zone string) string {
	if zone != "" && len(zone) > 2 {
		region := zone[:len(zone)-2]
		return region
	}
	return ""
}

// Infers the region based on the following (in order of priority):
// - `region` field in resource schema
// - region extracted from the `zone` field in resource schema
// - provider-level region
// - region extracted from the provider-level zone
func getRegion(d TerraformResourceData, config *Config) (string, error) {
	return getRegionFromSchema("region", "zone", d, config)
}

// getProject reads the "project" field from the given resource data and falls
// back to the provider's value if not given. If the provider's value is not
// given, an error is returned.
func getProject(d TerraformResourceData, config *Config) (string, error) {
	return getProjectFromSchema("project", d, config)
}

// getBillingProject reads the "billing_project" field from the given resource data and falls
// back to the provider's value if not given. If no value is found, an error is returned.
func getBillingProject(d TerraformResourceData, config *Config) (string, error) {
	return getBillingProjectFromSchema("billing_project", d, config)
}

// getProjectFromDiff reads the "project" field from the given diff and falls
// back to the provider's value if not given. If the provider's value is not
// given, an error is returned.
func getProjectFromDiff(d *schema.ResourceDiff, config *Config) (string, error) {
	res, ok := d.GetOk("project")
	if ok {
		return res.(string), nil
	}
	if config.Project != "" {
		return config.Project, nil
	}
	return "", fmt.Errorf("%s: required field is not set", "project")
}

func getRouterLockName(region string, router string) string {
	return fmt.Sprintf("router/%s/%s", region, router)
}

func handleNotFoundError(err error, d *schema.ResourceData, resource string) error {
	if isGoogleApiErrorWithCode(err, 404) {
		log.Printf("[WARN] Removing %s because it's gone", resource)
		// The resource doesn't exist anymore
		d.SetId("")

		return nil
	}

	return errwrap.Wrapf(
		fmt.Sprintf("Error when reading or editing %s: {{err}}", resource), err)
}

func isGoogleApiErrorWithCode(err error, errCode int) bool {
	gerr, ok := errwrap.GetType(err, &googleapi.Error{}).(*googleapi.Error)
	return ok && gerr != nil && gerr.Code == errCode
}

func isApiNotEnabledError(err error) bool {
	gerr, ok := errwrap.GetType(err, &googleapi.Error{}).(*googleapi.Error)
	if !ok {
		return false
	}
	if gerr == nil {
		return false
	}
	if gerr.Code != 403 {
		return false
	}
	for _, e := range gerr.Errors {
		if e.Reason == "accessNotConfigured" {
			return true
		}
	}
	return false
}

func isFailedPreconditionError(err error) bool {
	gerr, ok := errwrap.GetType(err, &googleapi.Error{}).(*googleapi.Error)
	if !ok {
		return false
	}
	if gerr == nil {
		return false
	}
	if gerr.Code != 400 {
		return false
	}
	for _, e := range gerr.Errors {
		if e.Reason == "failedPrecondition" {
			return true
		}
	}
	return false
}

func isConflictError(err error) bool {
	if e, ok := err.(*googleapi.Error); ok && e.Code == 409 {
		return true
	} else if !ok && errwrap.ContainsType(err, &googleapi.Error{}) {
		e := errwrap.GetType(err, &googleapi.Error{}).(*googleapi.Error)
		if e.Code == 409 {
			return true
		}
	}
	return false
}

// expandLabels pulls the value of "labels" out of a TerraformResourceData as a map[string]string.
func expandLabels(d TerraformResourceData) map[string]string {
	return expandStringMap(d, "labels")
}

// expandEnvironmentVariables pulls the value of "environment_variables" out of a schema.ResourceData as a map[string]string.
func expandEnvironmentVariables(d *schema.ResourceData) map[string]string {
	return expandStringMap(d, "environment_variables")
}

// expandBuildEnvironmentVariables pulls the value of "build_environment_variables" out of a schema.ResourceData as a map[string]string.
func expandBuildEnvironmentVariables(d *schema.ResourceData) map[string]string {
	return expandStringMap(d, "build_environment_variables")
}

// expandStringMap pulls the value of key out of a TerraformResourceData as a map[string]string.
func expandStringMap(d TerraformResourceData, key string) map[string]string {
	v, ok := d.GetOk(key)

	if !ok {
		return map[string]string{}
	}

	return convertStringMap(v.(map[string]interface{}))
}

func convertStringMap(v map[string]interface{}) map[string]string {
	m := make(map[string]string)
	for k, val := range v {
		m[k] = val.(string)
	}
	return m
}

func convertStringArr(ifaceArr []interface{}) []string {
	return convertAndMapStringArr(ifaceArr, func(s string) string { return s })
}

func convertAndMapStringArr(ifaceArr []interface{}, f func(string) string) []string {
	var arr []string
	for _, v := range ifaceArr {
		if v == nil {
			continue
		}
		arr = append(arr, f(v.(string)))
	}
	return arr
}

func mapStringArr(original []string, f func(string) string) []string {
	var arr []string
	for _, v := range original {
		arr = append(arr, f(v))
	}
	return arr
}

func convertStringArrToInterface(strs []string) []interface{} {
	arr := make([]interface{}, len(strs))
	for i, str := range strs {
		arr[i] = str
	}
	return arr
}

func convertStringSet(set *schema.Set) []string {
	s := make([]string, 0, set.Len())
	for _, v := range set.List() {
		s = append(s, v.(string))
	}
	sort.Strings(s)

	return s
}

func golangSetFromStringSlice(strings []string) map[string]struct{} {
	set := map[string]struct{}{}
	for _, v := range strings {
		set[v] = struct{}{}
	}

	return set
}

func stringSliceFromGolangSet(sset map[string]struct{}) []string {
	ls := make([]string, 0, len(sset))
	for s := range sset {
		ls = append(ls, s)
	}
	sort.Strings(ls)

	return ls
}

func reverseStringMap(m map[string]string) map[string]string {
	o := map[string]string{}
	for k, v := range m {
		o[v] = k
	}
	return o
}

func mergeStringMaps(a, b map[string]string) map[string]string {
	merged := make(map[string]string)

	for k, v := range a {
		merged[k] = v
	}

	for k, v := range b {
		merged[k] = v
	}

	return merged
}

func mergeSchemas(a, b map[string]*schema.Schema) map[string]*schema.Schema {
	merged := make(map[string]*schema.Schema)

	for k, v := range a {
		merged[k] = v
	}

	for k, v := range b {
		merged[k] = v
	}

	return merged
}

func mergeResourceMaps(ms ...map[string]*schema.Resource) (map[string]*schema.Resource, error) {
	merged := make(map[string]*schema.Resource)
	duplicates := []string{}

	for _, m := range ms {
		for k, v := range m {
			if _, ok := merged[k]; ok {
				duplicates = append(duplicates, k)
			}

			merged[k] = v
		}
	}

	var err error
	if len(duplicates) > 0 {
		err = fmt.Errorf("saw duplicates in mergeResourceMaps: %v", duplicates)
	}

	return merged, err
}

func extractFirstMapConfig(m []interface{}) map[string]interface{} {
	if len(m) == 0 {
		return map[string]interface{}{}
	}

	return m[0].(map[string]interface{})
}

func lockedCall(lockKey string, f func() error) error {
	mutexKV.Lock(lockKey)
	defer mutexKV.Unlock(lockKey)

	return f()
}

// This is a Printf sibling (Nprintf; Named Printf), which handles strings like
// Nprintf("Hello %{target}!", map[string]interface{}{"target":"world"}) == "Hello world!".
// This is particularly useful for generated tests, where we don't want to use Printf,
// since that would require us to generate a very particular ordering of arguments.
func Nprintf(format string, params map[string]interface{}) string {
	for key, val := range params {
		format = strings.Replace(format, "%{"+key+"}", fmt.Sprintf("%v", val), -1)
	}
	return format
}

// serviceAccountFQN will attempt to generate the fully qualified name in the format of:
// "projects/(-|<project>)/serviceAccounts/<service_account_id>@<project>.iam.gserviceaccount.com"
// A project is required if we are trying to build the FQN from a service account id and
// and error will be returned in this case if no project is set in the resource or the
// provider-level config
func serviceAccountFQN(serviceAccount string, d TerraformResourceData, config *Config) (string, error) {
	// If the service account id is already the fully qualified name
	if strings.HasPrefix(serviceAccount, "projects/") {
		return serviceAccount, nil
	}

	// If the service account id is an email
	if strings.Contains(serviceAccount, "@") {
		return "projects/-/serviceAccounts/" + serviceAccount, nil
	}

	// Get the project from the resource or fallback to the project
	// in the provider configuration
	project, err := getProject(d, config)
	if err != nil {
		return "", err
	}

	return fmt.Sprintf("projects/-/serviceAccounts/%s@%s.iam.gserviceaccount.com", serviceAccount, project), nil
}

func paginatedListRequest(project, baseUrl, userAgent string, config *Config, flattener func(map[string]interface{}) []interface{}) ([]interface{}, error) {
	res, err := sendRequest(config, "GET", project, baseUrl, userAgent, nil)
	if err != nil {
		return nil, err
	}

	ls := flattener(res)
	pageToken, ok := res["pageToken"]
	for ok {
		if pageToken.(string) == "" {
			break
		}
		url := fmt.Sprintf("%s?pageToken=%s", baseUrl, pageToken.(string))
		res, err = sendRequest(config, "GET", project, url, userAgent, nil)
		if err != nil {
			return nil, err
		}
		ls = append(ls, flattener(res))
		pageToken, ok = res["pageToken"]
	}

	return ls, nil
}

func getInterconnectAttachmentLink(config *Config, project, region, ic, userAgent string) (string, error) {
	if !strings.Contains(ic, "/") {
		icData, err := config.NewComputeClient(userAgent).InterconnectAttachments.Get(
			project, region, ic).Do()
		if err != nil {
			return "", fmt.Errorf("Error reading interconnect attachment: %s", err)
		}
		ic = icData.SelfLink
	}

	return ic, nil
}

// Given two sets of references (with "from" values in self link form),
// determine which need to be added or removed // during an update using
// addX/removeX APIs.
func calcAddRemove(from []string, to []string) (add, remove []string) {
	add = make([]string, 0)
	remove = make([]string, 0)
	for _, u := range to {
		found := false
		for _, v := range from {
			if compareSelfLinkOrResourceName("", v, u, nil) {
				found = true
				break
			}
		}
		if !found {
			add = append(add, u)
		}
	}
	for _, u := range from {
		found := false
		for _, v := range to {
			if compareSelfLinkOrResourceName("", u, v, nil) {
				found = true
				break
			}
		}
		if !found {
			remove = append(remove, u)
		}
	}
	return add, remove
}

func stringInSlice(arr []string, str string) bool {
	for _, i := range arr {
		if i == str {
			return true
		}
	}

	return false
}

func migrateStateNoop(v int, is *terraform.InstanceState, meta interface{}) (*terraform.InstanceState, error) {
	return is, nil
}

func expandString(v interface{}, d TerraformResourceData, config *Config) (string, error) {
	return v.(string), nil
}

func changeFieldSchemaToForceNew(sch *schema.Schema) {
	sch.ForceNew = true
	switch sch.Type {
	case schema.TypeList:
	case schema.TypeSet:
		if nestedR, ok := sch.Elem.(*schema.Resource); ok {
			for _, nestedSch := range nestedR.Schema {
				changeFieldSchemaToForceNew(nestedSch)
			}
		}
	}
}

func generateUserAgentString(d *schema.ResourceData, currentUserAgent string) (string, error) {
	var m providerMeta

	err := d.GetProviderMeta(&m)
	if err != nil {
		return currentUserAgent, err
	}

	if m.ModuleName != "" {
		return strings.Join([]string{currentUserAgent, m.ModuleName}, " "), nil
	}

	return currentUserAgent, nil
}
<|endoftext|>