<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package middleware

import (
	"context"
	"net"
	"net/http"
	"strconv"
	"strings"
	"sync"

	"github.com/golang/groupcache/lru"
	"go.opencensus.io/stats"
	"go.opencensus.io/stats/view"
	"go.opencensus.io/tag"
	"golang.org/x/pkgsite/internal/config"
	"golang.org/x/pkgsite/internal/log"
	"golang.org/x/time/rate"
)

var (
	keyQuotaBlocked = tag.MustNewKey("quota.blocked")
	quotaResults    = stats.Int64(
		"go-discovery/quota_result_count",
		"The result of a quota check.",
		stats.UnitDimensionless,
	)
	// QuotaResultCount is a counter of quota results, by whether the request was blocked or not.
	QuotaResultCount = &view.View{
		Name:        "go-discovery/quota/result_count",
		Measure:     quotaResults,
		Aggregation: view.Count(),
		Description: "quota results, by blocked or allowed",
		TagKeys:     []tag.Key{keyQuotaBlocked},
	}
)

// Quota implements a simple IP-based rate limiter. Each set of incoming IP
// addresses with the same low-order byte gets qps requests per second, with the
// given burst.
// Information is kept in an LRU cache of size maxEntries.
//
// If a request is disallowed, a 429 (TooManyRequests) will be served.
func Quota(settings config.QuotaSettings) Middleware {
	var mu sync.Mutex
	cache := lru.New(settings.MaxEntries)

	return func(h http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			authVal := r.Header.Get(settings.AuthHeader)
			for _, wantVal := range settings.AuthValues {
				if authVal == wantVal {
					recordQuotaMetric("accepted")
					log.Infof(r.Context(), "Quota: accepting %q", authVal)
					h.ServeHTTP(w, r)
					return
				}
			}

			key := ipKey(r.Header.Get("X-Forwarded-For"))
			// key is empty if we couldn't parse an IP, or there is no IP.
			// Fail open in this case: allow serving.
			var limiter *rate.Limiter
			if key != "" {
				mu.Lock()
				if v, ok := cache.Get(key); ok {
					limiter = v.(*rate.Limiter)
				} else {
					limiter = rate.NewLimiter(rate.Limit(settings.QPS), settings.Burst)
					cache.Add(key, limiter)
				}
				mu.Unlock()
			}
			blocked := limiter != nil && !limiter.Allow()
			recordQuotaMetric(strconv.FormatBool(blocked))
			if blocked && settings.RecordOnly != nil && !*settings.RecordOnly {
				const tmr = http.StatusTooManyRequests
				http.Error(w, http.StatusText(tmr), tmr)
				return
			}
			h.ServeHTTP(w, r)
		})
	}
}

func recordQuotaMetric(blocked string) {
	stats.RecordWithTags(context.Background(), []tag.Mutator{
		tag.Upsert(keyQuotaBlocked, blocked),
	}, quotaResults.M(1))
}

func ipKey(s string) string {
	fields := strings.SplitN(s, ",", 2)
	// First field is the originating IP address.
	origin := strings.TrimSpace(fields[0])
	ip := net.ParseIP(origin)
	if ip == nil {
		return ""
	}
	// Zero out last byte, to cover ranges.
	ip[len(ip)-1] = 0
	return ip.String()
}
<commit_msg>internal/middleware: pass context to quota metric recording<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package middleware

import (
	"context"
	"net"
	"net/http"
	"strconv"
	"strings"
	"sync"

	"github.com/golang/groupcache/lru"
	"go.opencensus.io/stats"
	"go.opencensus.io/stats/view"
	"go.opencensus.io/tag"
	"golang.org/x/pkgsite/internal/config"
	"golang.org/x/pkgsite/internal/log"
	"golang.org/x/time/rate"
)

var (
	keyQuotaBlocked = tag.MustNewKey("quota.blocked")
	quotaResults    = stats.Int64(
		"go-discovery/quota_result_count",
		"The result of a quota check.",
		stats.UnitDimensionless,
	)
	// QuotaResultCount is a counter of quota results, by whether the request was blocked or not.
	QuotaResultCount = &view.View{
		Name:        "go-discovery/quota/result_count",
		Measure:     quotaResults,
		Aggregation: view.Count(),
		Description: "quota results, by blocked or allowed",
		TagKeys:     []tag.Key{keyQuotaBlocked},
	}
)

// Quota implements a simple IP-based rate limiter. Each set of incoming IP
// addresses with the same low-order byte gets qps requests per second, with the
// given burst.
// Information is kept in an LRU cache of size maxEntries.
//
// If a request is disallowed, a 429 (TooManyRequests) will be served.
func Quota(settings config.QuotaSettings) Middleware {
	var mu sync.Mutex
	cache := lru.New(settings.MaxEntries)

	return func(h http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			authVal := r.Header.Get(settings.AuthHeader)
			for _, wantVal := range settings.AuthValues {
				if authVal == wantVal {
					recordQuotaMetric(r.Context(), "accepted")
					log.Infof(r.Context(), "Quota: accepting %q", authVal)
					h.ServeHTTP(w, r)
					return
				}
			}

			key := ipKey(r.Header.Get("X-Forwarded-For"))
			// key is empty if we couldn't parse an IP, or there is no IP.
			// Fail open in this case: allow serving.
			var limiter *rate.Limiter
			if key != "" {
				mu.Lock()
				if v, ok := cache.Get(key); ok {
					limiter = v.(*rate.Limiter)
				} else {
					limiter = rate.NewLimiter(rate.Limit(settings.QPS), settings.Burst)
					cache.Add(key, limiter)
				}
				mu.Unlock()
			}
			blocked := limiter != nil && !limiter.Allow()
			recordQuotaMetric(r.Context(), strconv.FormatBool(blocked))
			if blocked && settings.RecordOnly != nil && !*settings.RecordOnly {
				const tmr = http.StatusTooManyRequests
				http.Error(w, http.StatusText(tmr), tmr)
				return
			}
			h.ServeHTTP(w, r)
		})
	}
}

func recordQuotaMetric(ctx context.Context, blocked string) {
	stats.RecordWithTags(ctx, []tag.Mutator{
		tag.Upsert(keyQuotaBlocked, blocked),
	}, quotaResults.M(1))
}

func ipKey(s string) string {
	fields := strings.SplitN(s, ",", 2)
	// First field is the originating IP address.
	origin := strings.TrimSpace(fields[0])
	ip := net.ParseIP(origin)
	if ip == nil {
		return ""
	}
	// Zero out last byte, to cover ranges.
	ip[len(ip)-1] = 0
	return ip.String()
}
<|endoftext|>


<commit_before>// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
//
// This program is free software: you can redistribute it and/or modify it
// under the terms of the GNU General Public License as published by the Free
// Software Foundation, either version 3 of the License, or (at your option)
// any later version.
//
// This program is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
// more details.
//
// You should have received a copy of the GNU General Public License along
// with this program. If not, see <http://www.gnu.org/licenses/>.

package model

import "github.com/syncthing/syncthing/internal/protocol"

type bqAdd struct {
	file protocol.FileInfo
	have []protocol.BlockInfo
	need []protocol.BlockInfo
}

type bqBlock struct {
	file  protocol.FileInfo
	block protocol.BlockInfo   // get this block from the network
	copy  []protocol.BlockInfo // copy these blocks from the old version of the file
	first bool
	last  bool
}

type blockQueue struct {
	queued []bqBlock
}

func (q *blockQueue) put(a bqAdd) {
	// If we already have it queued, return
	for _, b := range q.queued {
		if b.file.Name == a.file.Name {
			return
		}
	}

	l := len(a.need)

	if len(a.have) > 0 {
		// First queue a copy operation
		q.queued = append(q.queued, bqBlock{
			file:  a.file,
			copy:  a.have,
			first: true,
			last:  l == 0,
		})
	}

	// Queue the needed blocks individually
	for i, b := range a.need {
		q.queued = append(q.queued, bqBlock{
			file:  a.file,
			block: b,
			first: len(a.have) == 0 && i == 0,
			last:  i == l-1,
		})
	}

	if len(a.need)+len(a.have) == 0 {
		// If we didn't have anything to fetch, queue an empty block with the "last" flag set to close the file.
		q.queued = append(q.queued, bqBlock{
			file: a.file,
			last: true,
		})
	}
}

func (q *blockQueue) get() (bqBlock, bool) {
	if len(q.queued) == 0 {
		return bqBlock{}, false
	}
	b := q.queued[0]
	q.queued = q.queued[1:]
	return b, true
}
<commit_msg>Remove dead code<commit_after><|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

// Build the parser:
//go:generate goyacc -v y.output -o parser.go -p mtail parser.y

// Package parser implements the parse phase of the mtail program compilation.
package parser

import (
	"flag"
	"fmt"
	"io"
	"strconv"
	"time"

	"github.com/golang/glog"
	"github.com/google/mtail/internal/vm/ast"
	"github.com/google/mtail/internal/vm/errors"
	"github.com/google/mtail/internal/vm/position"
)

// Parse reads the program named name from the input, and if successful returns
// an ast.Node for the root of the AST, otherwise parser errors.
func Parse(name string, input io.Reader) (ast.Node, error) {
	p := newParser(name, input)
	r := mtailParse(p)
	if r != 0 || p == nil || p.errors != nil {
		return nil, p.errors
	}
	return p.root, nil
}

// EOF is a marker for end of file.
const EOF = 0

// parser defines the data structure for parsing an mtail program.
type parser struct {
	name   string
	root   ast.Node
	errors errors.ErrorList
	l      *Lexer
	t      Token             // Most recently lexed token.
	pos    position.Position // Optionally contains the position of the start of a production
}

func newParser(name string, input io.Reader) *parser {
	return &parser{name: name, l: NewLexer(name, input)}
}

func (p *parser) ErrorP(s string, pos *position.Position) {
	p.errors.Add(pos, s)
}

func (p *parser) Error(s string) {
	p.errors.Add(&p.t.Pos, s)
}

// Lex reads the next token from the Lexer, turning it into a form useful for the goyacc generated parser.
// The variable lval is modified to carry token information, and the token type is returned.
func (p *parser) Lex(lval *mtailSymType) int {
	p.t = p.l.NextToken()
	switch p.t.Kind {
	case INVALID:
		p.Error(p.t.Spelling)
		lval.text = p.t.Spelling
		return INVALID
	case INTLITERAL:
		var err error
		lval.intVal, err = strconv.ParseInt(p.t.Spelling, 10, 64)
		if err != nil {
			p.Error(fmt.Sprintf("bad number '%s': %s", p.t.Spelling, err))
			return INVALID
		}
	case FLOATLITERAL:
		var err error
		lval.floatVal, err = strconv.ParseFloat(p.t.Spelling, 64)
		if err != nil {
			p.Error(fmt.Sprintf("bad number '%s': %s", p.t.Spelling, err))
			return INVALID
		}
	case DURATIONLITERAL:
		var err error
		lval.duration, err = time.ParseDuration(p.t.Spelling)
		if err != nil {
			p.Error(fmt.Sprintf("%s", err))
			return INVALID
		}
	case LT, GT, LE, GE, NE, EQ, SHL, SHR, BITAND, BITOR, AND, OR, XOR, NOT, INC, DEC, DIV, MUL, MINUS, PLUS, ASSIGN, ADD_ASSIGN, POW, MOD, CONCAT, MATCH, NOT_MATCH:
		lval.op = int(p.t.Kind)
	default:
		lval.text = p.t.Spelling
	}
	return int(p.t.Kind)
}

func (p *parser) inRegex() {
	glog.V(2).Info("Entering regex")
	p.l.InRegex = true
}

func init() {
	flag.IntVar(&mtailDebug, "mtailDebug", 0, "Set parser debug level.")
	mtailErrorVerbose = true
}
<commit_msg>docstrings<commit_after>// Copyright 2016 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

// Build the parser:
//go:generate goyacc -v y.output -o parser.go -p mtail parser.y

// Package parser implements the parse phase of the mtail program compilation.
// The parser itself is defined in parser.y, and goyacc generates the program
// code and token definitions.  The parser fetches tokens from the lexer, which
// scans the input converting the program source into a token stream.  The
// driver code wraps the generated parser and marshals the ast and errors back
// to the caller.
//
// Two pretty-printers are used for debugging: the unparser, which converts an
// ast back into program text, and an approximation of an s-expression printer,
// which tries to model in indented text the structure of the ast.
package parser

import (
	"flag"
	"fmt"
	"io"
	"strconv"
	"time"

	"github.com/golang/glog"
	"github.com/google/mtail/internal/vm/ast"
	"github.com/google/mtail/internal/vm/errors"
	"github.com/google/mtail/internal/vm/position"
)

// Parse reads the program named name from the input, and if successful returns
// an ast.Node for the root of the AST, otherwise parser errors.
func Parse(name string, input io.Reader) (ast.Node, error) {
	p := newParser(name, input)
	r := mtailParse(p)
	if r != 0 || p == nil || p.errors != nil {
		return nil, p.errors
	}
	return p.root, nil
}

// EOF is a marker for end of file.  It has the same value as the goyacc internal Kind `$end`.
const EOF = 0

// parser defines the data structure for parsing an mtail program.
type parser struct {
	name   string
	root   ast.Node
	errors errors.ErrorList
	l      *Lexer
	t      Token             // Most recently lexed token.
	pos    position.Position // Optionally contains the position of the start of a production
}

func newParser(name string, input io.Reader) *parser {
	return &parser{name: name, l: NewLexer(name, input)}
}

func (p *parser) ErrorP(s string, pos *position.Position) {
	p.errors.Add(pos, s)
}

func (p *parser) Error(s string) {
	p.errors.Add(&p.t.Pos, s)
}

// Lex reads the next token from the Lexer, turning it into a form useful for the goyacc generated parser.
// The variable lval is modified to carry token information, and the token type is returned.
func (p *parser) Lex(lval *mtailSymType) int {
	p.t = p.l.NextToken()
	switch p.t.Kind {
	case INVALID:
		p.Error(p.t.Spelling)
		lval.text = p.t.Spelling
		return INVALID
	case INTLITERAL:
		var err error
		lval.intVal, err = strconv.ParseInt(p.t.Spelling, 10, 64)
		if err != nil {
			p.Error(fmt.Sprintf("bad number '%s': %s", p.t.Spelling, err))
			return INVALID
		}
	case FLOATLITERAL:
		var err error
		lval.floatVal, err = strconv.ParseFloat(p.t.Spelling, 64)
		if err != nil {
			p.Error(fmt.Sprintf("bad number '%s': %s", p.t.Spelling, err))
			return INVALID
		}
	case DURATIONLITERAL:
		var err error
		lval.duration, err = time.ParseDuration(p.t.Spelling)
		if err != nil {
			p.Error(fmt.Sprintf("%s", err))
			return INVALID
		}
	case LT, GT, LE, GE, NE, EQ, SHL, SHR, BITAND, BITOR, AND, OR, XOR, NOT, INC, DEC, DIV, MUL, MINUS, PLUS, ASSIGN, ADD_ASSIGN, POW, MOD, CONCAT, MATCH, NOT_MATCH:
		lval.op = int(p.t.Kind)
	default:
		lval.text = p.t.Spelling
	}
	return int(p.t.Kind)
}

func (p *parser) inRegex() {
	glog.V(2).Info("Entering regex")
	p.l.InRegex = true
}

func init() {
	flag.IntVar(&mtailDebug, "mtailDebug", 0, "Set parser debug level.")
	mtailErrorVerbose = true
}
<|endoftext|>


<commit_before>// email 类库
// create by gloomy 2017-04-12 11:16:21
package common

import (
	"net/smtp"
)

// 发送邮件
//
func SendEmail() {
	auth := smtp.PlainAuth("", "shaow@axon.com.cn", "Sw435464", "smtp.qiye.163.com")
	to := []string{"sw.gloomysw@gmail.com", "chenm@axon.com.cn"}
	msg := []byte("To: bain@axon.com.cn;shaow@axon.com.cn;chenm@axon.com.cn\r\n" +
		"Subject: 测试golang邮件发送 \r\n" +
		"Content-Type: text/plain; charset=UTF-8" + "\r\n\r\n")
	//for _, value := range sendMailContent {
	msg = append(msg, []byte("测试邮件发送")...)
	//}
	err := smtp.SendMail("smtp.qiye.163.com:25", auth, "shaow@axon.com.cn", to, msg)
	if err != nil {
		//glog.Error("邮件发送失败! err: %s \n", err.Error())
		//	fmt.Println(err.Error())
		return
	}
}
<commit_msg>完善邮件发送程序<commit_after>// email 类库
// create by gloomy 2017-04-12 11:16:21
package common

import (
	"fmt"
	"net/smtp"
	"strings"
)

// 发送邮件
// create by gloomy 2017-04-12 11:18:23
func SendEmail(account, passWord, smtpOption, emailSendPort, emailTitle string, sendContent *[]byte, toEmailUser []string) error {
	auth := smtp.PlainAuth("", account, passWord, smtpOption)
	to := toEmailUser
	msg := []byte(fmt.Sprintf("To: %s \r\n Subject: %s \r\n Content-Type: text/plain; charset=UTF-8\r\n\r\n", strings.Join(toEmailUser, ";"), emailTitle))
	msg = append(msg, (*sendContent)...)
	return smtp.SendMail(fmt.Sprintf("%s:%s", smtpOption, emailSendPort), auth, account, to, msg)
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/aclements/go-misc/internal/loganal"
)

// TODO: If searching dashboard logs, optionally print to builder URLs
// instead of local file names.

// TODO: Optionally extract failures and show only those.

// TODO: Optionally classify matched logs by failure (and show either
// file name or extracted failure).

// TODO: Option to print Markdown-friendly output for GitHub.

// TODO: Option to print failure summary versus full failure message.

// TODO: Option to only show failures matching regexp? Currently we
// show all failures in files matching regexp, but sometimes you want
// to search the failures themselves. We could pre-filter the files by
// regexp, extract failures, and then match the failure messages.

var (
	// TODO: Allow mulitple -e's like grep.
	flagRegexp = flag.String("e", "", "show files matching `regexp`")
	re         *regexp.Regexp

	flagDashboard = flag.Bool("dashboard", false, "search dashboard logs from fetchlogs")
)

func main() {
	// XXX What I want right now is just to point it at a bunch of
	// logs and have it extract the failures.
	flag.Parse()

	// Validate flags.
	if *flagRegexp != "" {
		var err error
		re, err = regexp.Compile(*flagRegexp)
		if err != nil {
			fmt.Fprintf(os.Stderr, "bad regexp: %v\n", err)
			os.Exit(2)
		}
	}
	if *flagDashboard && flag.NArg() > 0 {
		fmt.Fprintf(os.Stderr, "-dashboard and paths are incompatible\n")
		os.Exit(2)
	}

	// Gather paths.
	var paths []string
	var stripDir string
	if *flagDashboard {
		revDir := filepath.Join(xdgCacheDir(), "fetchlogs", "rev")
		paths = []string{revDir}
		stripDir = revDir + "/"
	} else {
		paths = flag.Args()
	}

	// Process files
	status := 1
	for _, path := range paths {
		filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
			if err != nil {
				status = 2
				fmt.Fprintf(os.Stderr, "%s: %v\n", path, err)
				return nil
			}
			if info.IsDir() || strings.HasPrefix(filepath.Base(path), ".") {
				return nil
			}

			nicePath := path
			if stripDir != "" && strings.HasPrefix(path, stripDir) {
				nicePath = path[len(stripDir):]
			}

			found, err := process(path, nicePath)
			if err != nil {
				status = 2
				fmt.Fprintf(os.Stderr, "%s: %v\n", path, err)
			} else if found && status == 1 {
				status = 0
			}
			return nil
		})
	}
	os.Exit(status)
}

func process(path, nicePath string) (found bool, err error) {
	// TODO: Use streaming if possible.
	data, err := ioutil.ReadFile(path)
	if err != nil {
		return false, err
	}

	// Check regexp match.
	if re != nil && !re.Match(data) {
		return false, nil
	}

	// Extract failures.
	failures, err := loganal.Extract(string(data), "", "")
	if err != nil {
		return false, err
	}

	// Print failures.
	for _, failure := range failures {
		msg := failure.FullMessage
		if msg == "" {
			msg = failure.Message
		}
		fmt.Printf("%s:\n%s\n\n", nicePath, msg)
	}
	return true, nil
}
<commit_msg>greplogs: compile regexp in multi-line mode<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/aclements/go-misc/internal/loganal"
)

// TODO: If searching dashboard logs, optionally print to builder URLs
// instead of local file names.

// TODO: Optionally extract failures and show only those.

// TODO: Optionally classify matched logs by failure (and show either
// file name or extracted failure).

// TODO: Option to print Markdown-friendly output for GitHub.

// TODO: Option to print failure summary versus full failure message.

// TODO: Option to only show failures matching regexp? Currently we
// show all failures in files matching regexp, but sometimes you want
// to search the failures themselves. We could pre-filter the files by
// regexp, extract failures, and then match the failure messages.

var (
	// TODO: Allow mulitple -e's like grep.
	flagRegexp = flag.String("e", "", "show files matching `regexp`")
	re         *regexp.Regexp

	flagDashboard = flag.Bool("dashboard", false, "search dashboard logs from fetchlogs")
)

func main() {
	// XXX What I want right now is just to point it at a bunch of
	// logs and have it extract the failures.
	flag.Parse()

	// Validate flags.
	if *flagRegexp != "" {
		var err error
		re, err = regexp.Compile("(?m)" + *flagRegexp)
		if err != nil {
			fmt.Fprintf(os.Stderr, "bad regexp: %v\n", err)
			os.Exit(2)
		}
	}
	if *flagDashboard && flag.NArg() > 0 {
		fmt.Fprintf(os.Stderr, "-dashboard and paths are incompatible\n")
		os.Exit(2)
	}

	// Gather paths.
	var paths []string
	var stripDir string
	if *flagDashboard {
		revDir := filepath.Join(xdgCacheDir(), "fetchlogs", "rev")
		paths = []string{revDir}
		stripDir = revDir + "/"
	} else {
		paths = flag.Args()
	}

	// Process files
	status := 1
	for _, path := range paths {
		filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
			if err != nil {
				status = 2
				fmt.Fprintf(os.Stderr, "%s: %v\n", path, err)
				return nil
			}
			if info.IsDir() || strings.HasPrefix(filepath.Base(path), ".") {
				return nil
			}

			nicePath := path
			if stripDir != "" && strings.HasPrefix(path, stripDir) {
				nicePath = path[len(stripDir):]
			}

			found, err := process(path, nicePath)
			if err != nil {
				status = 2
				fmt.Fprintf(os.Stderr, "%s: %v\n", path, err)
			} else if found && status == 1 {
				status = 0
			}
			return nil
		})
	}
	os.Exit(status)
}

func process(path, nicePath string) (found bool, err error) {
	// TODO: Use streaming if possible.
	data, err := ioutil.ReadFile(path)
	if err != nil {
		return false, err
	}

	// Check regexp match.
	if re != nil && !re.Match(data) {
		return false, nil
	}

	// Extract failures.
	failures, err := loganal.Extract(string(data), "", "")
	if err != nil {
		return false, err
	}

	// Print failures.
	for _, failure := range failures {
		msg := failure.FullMessage
		if msg == "" {
			msg = failure.Message
		}
		fmt.Printf("%s:\n%s\n\n", nicePath, msg)
	}
	return true, nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"log"
	"net/http"
	"net/url"

	"github.com/docopt/docopt-go"

	bleed "github.com/FiloSottile/Heartbleed/bleed"
	cache "github.com/FiloSottile/Heartbleed/server/cache"
)

var PAYLOAD = []byte("filippo.io/Heartbleed")

var withCache bool

type result struct {
	Code  int    `json:"code"`
	Data  string `json:"data"`
	Error string `json:"error"`
	Host  string `json:"host"`
}

func handleRequest(tgt *bleed.Target, w http.ResponseWriter, r *http.Request, skip bool) {
	w.Header().Set("Access-Control-Allow-Origin", "*")

	var rc int
	var errS string
	var data string

	var cacheOk bool
	if withCache {
		cReply, ok := cache.Check(tgt.HostIp)
		if ok {
			rc = int(cReply.Status)
			errS = cReply.Error
			data = cReply.Data
			cacheOk = true
		}
	}

	if !withCache || !cacheOk {
		out, err := bleed.Heartbleed(tgt, PAYLOAD, skip)

		if err == bleed.Safe || err == bleed.Closed {
			rc = 1
		} else if err != nil {
			rc = 2
		} else {
			rc = 0
			// _, err := bleed.Heartbleed(tgt, PAYLOAD)
			// if err == nil {
			// 	// Two VULN in a row
			// 	rc = 0
			// } else {
			// 	// One VULN and one not
			// 	_, err := bleed.Heartbleed(tgt, PAYLOAD)
			// 	if err == nil {
			// 		// 2 VULN on 3 tries
			// 		rc = 0
			// 	} else {
			// 		// 1 VULN on 3 tries
			// 		if err == bleed.Safe {
			// 			rc = 1
			// 		} else {
			// 			rc = 2
			// 		}
			// 	}
			// }
		}

		switch rc {
		case 0:
			data = out
			log.Printf("%v (%v) - VULNERABLE [skip: %v]", tgt.HostIp, tgt.Service, skip)
		case 1:
			log.Printf("%v (%v) - SAFE", tgt.HostIp, tgt.Service)
		case 2:
			errS = err.Error()
			if errS == "Please try again" {
				log.Printf("%v (%v) - MISMATCH", tgt.HostIp, tgt.Service)
			} else {
				log.Printf("%v (%v) - ERROR [%v]", tgt.HostIp, tgt.Service, errS)
			}
		}
	}

	if withCache {
		cache.Set(tgt.HostIp, rc, data, errS)
	}

	res := result{rc, data, errS, tgt.HostIp}
	j, err := json.Marshal(res)
	if err != nil {
		log.Println("[json] ERROR:", err)
	} else {
		w.Write(j)
	}
}

func bleedHandler(w http.ResponseWriter, r *http.Request) {
	host := r.URL.Path[len("/bleed/"):]

	tgt := bleed.Target{
		HostIp:  string(host),
		Service: "https",
	}
	handleRequest(&tgt, w, r, true)
}

func bleedQueryHandler(w http.ResponseWriter, r *http.Request) {
	q, ok := r.URL.Query()["u"]
	if !ok || len(q) != 1 {
		return
	}

	skip, ok := r.URL.Query()["skip"]
	s := false
	if ok && len(skip) == 1 {
		s = true
	}

	tgt := bleed.Target{
		HostIp:  string(q[0]),
		Service: "https",
	}

	u, err := url.Parse(tgt.HostIp)
	if err == nil && u.Host != "" {
		tgt.HostIp = u.Host
		if u.Scheme != "" {
			tgt.Service = u.Scheme
		}
	}

	handleRequest(&tgt, w, r, s)
}

var Usage = `Heartbleed test server.

Usage:
  HBserver --redir-host=<host> [--listen=<addr:port> --expiry=<duration>]
  HBserver -h | --help
  HBserver --version

Options:
  --redir-host HOST   Redirect requests to "/" to this host.
  --listen ADDR:PORT  Listen and serve requests to this address:port [default: :8082].
  --expiry DURATION   ENABLE CACHING. Expire records after this period.
                      Uses Go's parse syntax
                      e.g. 10m = 10 minutes, 600s = 600 seconds, 1d = 1 day, etc.
  -h --help           Show this screen.
  --version           Show version.`

func main() {
	arguments, _ := docopt.Parse(Usage, nil, true, "HBserver 0.3", false)

	if arguments["--expiry"] != nil {
		withCache = true
	}

	if withCache {
		cache.Init(arguments["--expiry"].(string))
	}

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, arguments["--redir-host"].(string), http.StatusFound)
	})

	// Required for some ELBs
	http.HandleFunc("/status", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte("OK"))
	})

	http.HandleFunc("/bleed/", bleedHandler)
	http.HandleFunc("/bleed/query", bleedQueryHandler)

	log.Printf("Starting server on %s\n", arguments["--listen"].(string))
	err := http.ListenAndServe(arguments["--listen"].(string), nil)
	if err != nil {
		log.Fatal("ListenAndServe: ", err)
	}
}
<commit_msg>Include the service in cache keys; ignore empty requests<commit_after>package main

import (
	"encoding/json"
	"log"
	"net/http"
	"net/url"

	"github.com/docopt/docopt-go"

	bleed "github.com/FiloSottile/Heartbleed/bleed"
	cache "github.com/FiloSottile/Heartbleed/server/cache"
)

var PAYLOAD = []byte("filippo.io/Heartbleed")

var withCache bool

type result struct {
	Code  int    `json:"code"`
	Data  string `json:"data"`
	Error string `json:"error"`
	Host  string `json:"host"`
}

func handleRequest(tgt *bleed.Target, w http.ResponseWriter, r *http.Request, skip bool) {
	if tgt.HostIp == "" {
		// tens of empty requests per minute, mah...
		return
	}

	w.Header().Set("Access-Control-Allow-Origin", "*")

	var rc int
	var errS string
	var data string

	var cacheOk bool
	if withCache {
		cReply, ok := cache.Check(tgt.Service + "://" + tgt.HostIp)
		if ok {
			rc = int(cReply.Status)
			errS = cReply.Error
			data = cReply.Data
			cacheOk = true
		}
	}

	if !withCache || !cacheOk {
		out, err := bleed.Heartbleed(tgt, PAYLOAD, skip)

		if err == bleed.Safe || err == bleed.Closed {
			rc = 1
		} else if err != nil {
			rc = 2
		} else {
			rc = 0
			// _, err := bleed.Heartbleed(tgt, PAYLOAD)
			// if err == nil {
			// 	// Two VULN in a row
			// 	rc = 0
			// } else {
			// 	// One VULN and one not
			// 	_, err := bleed.Heartbleed(tgt, PAYLOAD)
			// 	if err == nil {
			// 		// 2 VULN on 3 tries
			// 		rc = 0
			// 	} else {
			// 		// 1 VULN on 3 tries
			// 		if err == bleed.Safe {
			// 			rc = 1
			// 		} else {
			// 			rc = 2
			// 		}
			// 	}
			// }
		}

		switch rc {
		case 0:
			data = out
			log.Printf("%v (%v) - VULNERABLE [skip: %v]", tgt.HostIp, tgt.Service, skip)
		case 1:
			log.Printf("%v (%v) - SAFE", tgt.HostIp, tgt.Service)
		case 2:
			errS = err.Error()
			if errS == "Please try again" {
				log.Printf("%v (%v) - MISMATCH", tgt.HostIp, tgt.Service)
			} else {
				log.Printf("%v (%v) - ERROR [%v]", tgt.HostIp, tgt.Service, errS)
			}
		}
	}

	if withCache {
		cache.Set(tgt.Service+"://"+tgt.HostIp, rc, data, errS)
	}

	res := result{rc, data, errS, tgt.HostIp}
	j, err := json.Marshal(res)
	if err != nil {
		log.Println("[json] ERROR:", err)
	} else {
		w.Write(j)
	}
}

func bleedHandler(w http.ResponseWriter, r *http.Request) {
	host := r.URL.Path[len("/bleed/"):]

	tgt := bleed.Target{
		HostIp:  string(host),
		Service: "https",
	}
	handleRequest(&tgt, w, r, true)
}

func bleedQueryHandler(w http.ResponseWriter, r *http.Request) {
	q, ok := r.URL.Query()["u"]
	if !ok || len(q) != 1 {
		return
	}

	skip, ok := r.URL.Query()["skip"]
	s := false
	if ok && len(skip) == 1 {
		s = true
	}

	tgt := bleed.Target{
		HostIp:  string(q[0]),
		Service: "https",
	}

	u, err := url.Parse(tgt.HostIp)
	if err == nil && u.Host != "" {
		tgt.HostIp = u.Host
		if u.Scheme != "" {
			tgt.Service = u.Scheme
		}
	}

	handleRequest(&tgt, w, r, s)
}

var Usage = `Heartbleed test server.

Usage:
  HBserver --redir-host=<host> [--listen=<addr:port> --expiry=<duration>]
  HBserver -h | --help
  HBserver --version

Options:
  --redir-host HOST   Redirect requests to "/" to this host.
  --listen ADDR:PORT  Listen and serve requests to this address:port [default: :8082].
  --expiry DURATION   ENABLE CACHING. Expire records after this period.
                      Uses Go's parse syntax
                      e.g. 10m = 10 minutes, 600s = 600 seconds, 1d = 1 day, etc.
  -h --help           Show this screen.
  --version           Show version.`

func main() {
	arguments, _ := docopt.Parse(Usage, nil, true, "HBserver 0.3", false)

	if arguments["--expiry"] != nil {
		withCache = true
	}

	if withCache {
		cache.Init(arguments["--expiry"].(string))
	}

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, arguments["--redir-host"].(string), http.StatusFound)
	})

	// Required for some ELBs
	http.HandleFunc("/status", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte("OK"))
	})

	http.HandleFunc("/bleed/", bleedHandler)
	http.HandleFunc("/bleed/query", bleedQueryHandler)

	log.Printf("Starting server on %s\n", arguments["--listen"].(string))
	err := http.ListenAndServe(arguments["--listen"].(string), nil)
	if err != nil {
		log.Fatal("ListenAndServe: ", err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fs

import (
	"fmt"
	"io"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/gcloud/syncutil"
	"github.com/jacobsa/gcsfuse/fs/inode"
	"golang.org/x/net/context"
)

// State required for reading from directories.
type dirHandle struct {
	/////////////////////////
	// Constant data
	/////////////////////////

	in *inode.DirInode

	/////////////////////////
	// Mutable state
	/////////////////////////

	Mu syncutil.InvariantMutex

	// Entries that we have buffered from a previous call to in.ReadEntries.
	//
	// INVARIANT: For each i in range, entries[i+1].Offset == entries[i].Offset + 1
	//
	// GUARDED_BY(Mu)
	entries []fuseutil.Dirent

	// The logical offset at which entries[0] lies.
	//
	// INVARIANT: If len(entries) > 0, entriesOffset + 1 == entries[0].Offset
	//
	// GUARDED_BY(Mu)
	entriesOffset fuse.DirOffset

	// The continuation token to supply in the next call to in.ReadEntries.
	//
	// GUARDED_BY(Mu)
	tok string
}

// Create a directory handle that obtains listings from the supplied inode.
func newDirHandle(in *inode.DirInode) (dh *dirHandle) {
	// Set up the basic struct.
	dh = &dirHandle{
		in: in,
	}

	// Set up invariant checking.
	dh.Mu = syncutil.NewInvariantMutex(dh.checkInvariants)

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func (dh *dirHandle) checkInvariants() {
	// Check that the offsets increment by one each time.
	for i := 0; i < len(dh.entries)-1; i++ {
		if dh.entries[i].Offset+1 != dh.entries[i+1].Offset {
			panic(
				fmt.Sprintf(
					"Unexpected offset sequence: %v, %v",
					dh.entries[i].Offset,
					dh.entries[i+1].Offset))
		}
	}

	// Check the first offset.
	if len(dh.entries) > 0 && dh.entries[0].Offset != dh.entriesOffset+1 {
		panic(fmt.Sprintf("Unexpected entriesOffset value."))
	}
}

// Read some entries from the directory inode. Return an error if zero entries
// are returned. When the end of the directory is hit, this error will be
// io.EOF.
//
// SHARED_LOCKS_REQUIRED(in.Mu)
func readEntries(
	ctx context.Context,
	in *inode.DirInode,
	tok string,
	firstEntryOffset fuse.DirOffset) (
	entries []fuseutil.Dirent, newTok string, err error) {
	// Loop until we get a non-empty result, we finish, or an error occurs.
	for {
		entries, newTok, err = in.ReadEntries(ctx, tok)

		// Propagate errors.
		if err != nil {
			return
		}

		// If some entries were returned, we are done.
		if len(entries) > 0 {
			return
		}

		// If we're at the end of the directory, we're done.
		if newTok == "" {
			err = io.EOF
			return
		}

		// Otherwise, go around and ask for more entries.
		tok = newTok
	}
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

// Handle a request to read from the directory.
//
// Because the GCS API for listing objects has no notion of a stable offset
// like the posix telldir/seekdir API does, there is no way for us to
// efficiently support seeking backwards. Therefore we return EINVAL when an
// offset for an entry that we no longer have buffered is encountered.
//
// Special case: we assume that a zero offset indicates that rewinddir has been
// called (since fuse gives us no way to intercept and know for sure), and
// start the listing process over again.
//
// EXCLUSIVE_LOCKS_REQUIRED(dh.Mu)
// LOCKS_EXCLUDED(dh.in.Mu)
func (dh *dirHandle) ReadDir(
	ctx context.Context,
	req *fuse.ReadDirRequest) (resp *fuse.ReadDirResponse, err error) {
	resp = &fuse.ReadDirResponse{}

	dh.in.Mu.RLock()
	defer dh.in.Mu.RUnlock()

	// If the request is for offset zero, we assume that either this is the first
	// call or rewinddir has been called. Reset state.
	if req.Offset == 0 {
		dh.entries = nil
		dh.entriesOffset = 0
		dh.tok = ""
	}

	// Is the offset from before what we have buffered? If not, this represents a
	// seekdir we cannot support, as discussed in the method comments above.
	if req.Offset < dh.entriesOffset {
		err = fuse.EINVAL
		return
	}

	// Is the offset past the end of what we have buffered? If so, this must be
	// an invalid seekdir according to posix.
	index := int(req.Offset - dh.entriesOffset)
	if index > len(dh.entries) {
		err = fuse.EINVAL
		return
	}

	// Do we need to read more entries?
	if index == len(dh.entries) {
		var newEntries []fuseutil.Dirent
		var newTok string

		newEntries, newTok, err = readEntries(
			ctx,
			dh.in,
			dh.tok,
			dh.entriesOffset+fuse.DirOffset(len(dh.entries)))

		// For EOF we simply return an empty listing.
		if err == io.EOF {
			err = nil
			return
		}

		// Propagate other errors.
		if err != nil {
			err = fmt.Errorf("readEntries: %v", err)
			return
		}

		// Update state.
		dh.entriesOffset += fuse.DirOffset(len(dh.entries))
		dh.entries = newEntries
		dh.tok = newTok

		// Now we want to read from the front of dh.entries.
		index = 0
	}

	// Now we copy out entries until we run out of entries or space.
	for i := index; i < len(dh.entries); i++ {
		resp.Data = fuseutil.AppendDirent(resp.Data, dh.entries[i])
		if len(resp.Data) > req.Size {
			resp.Data = resp.Data[:req.Size]
			break
		}
	}

	return
}
<commit_msg>Preserve the offset invariant.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fs

import (
	"fmt"
	"io"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/gcloud/syncutil"
	"github.com/jacobsa/gcsfuse/fs/inode"
	"golang.org/x/net/context"
)

// State required for reading from directories.
type dirHandle struct {
	/////////////////////////
	// Constant data
	/////////////////////////

	in *inode.DirInode

	/////////////////////////
	// Mutable state
	/////////////////////////

	Mu syncutil.InvariantMutex

	// Entries that we have buffered from a previous call to in.ReadEntries.
	//
	// INVARIANT: For each i in range, entries[i+1].Offset == entries[i].Offset + 1
	//
	// GUARDED_BY(Mu)
	entries []fuseutil.Dirent

	// The logical offset at which entries[0] lies.
	//
	// INVARIANT: If len(entries) > 0, entriesOffset + 1 == entries[0].Offset
	//
	// GUARDED_BY(Mu)
	entriesOffset fuse.DirOffset

	// The continuation token to supply in the next call to in.ReadEntries.
	//
	// GUARDED_BY(Mu)
	tok string
}

// Create a directory handle that obtains listings from the supplied inode.
func newDirHandle(in *inode.DirInode) (dh *dirHandle) {
	// Set up the basic struct.
	dh = &dirHandle{
		in: in,
	}

	// Set up invariant checking.
	dh.Mu = syncutil.NewInvariantMutex(dh.checkInvariants)

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func (dh *dirHandle) checkInvariants() {
	// Check that the offsets increment by one each time.
	for i := 0; i < len(dh.entries)-1; i++ {
		if dh.entries[i].Offset+1 != dh.entries[i+1].Offset {
			panic(
				fmt.Sprintf(
					"Unexpected offset sequence: %v, %v",
					dh.entries[i].Offset,
					dh.entries[i+1].Offset))
		}
	}

	// Check the first offset.
	if len(dh.entries) > 0 && dh.entries[0].Offset != dh.entriesOffset+1 {
		panic(fmt.Sprintf("Unexpected entriesOffset value."))
	}
}

// Read some entries from the directory inode. Return an error if zero entries
// are returned. When the end of the directory is hit, this error will be
// io.EOF.
//
// SHARED_LOCKS_REQUIRED(in.Mu)
func readEntries(
	ctx context.Context,
	in *inode.DirInode,
	tok string,
	firstEntryOffset fuse.DirOffset) (
	entries []fuseutil.Dirent, newTok string, err error) {
	// Fix up the offset of any entries returned.
	defer func() {
		for i := 0; i < len(entries); i++ {
			entries[i].Offset = firstEntryOffset + fuse.DirOffset(i)
		}
	}()

	// Loop until we get a non-empty result, we finish, or an error occurs.
	for {
		entries, newTok, err = in.ReadEntries(ctx, tok)

		// Propagate errors.
		if err != nil {
			return
		}

		// If some entries were returned, we are done.
		if len(entries) > 0 {
			return
		}

		// If we're at the end of the directory, we're done.
		if newTok == "" {
			err = io.EOF
			return
		}

		// Otherwise, go around and ask for more entries.
		tok = newTok
	}
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

// Handle a request to read from the directory.
//
// Because the GCS API for listing objects has no notion of a stable offset
// like the posix telldir/seekdir API does, there is no way for us to
// efficiently support seeking backwards. Therefore we return EINVAL when an
// offset for an entry that we no longer have buffered is encountered.
//
// Special case: we assume that a zero offset indicates that rewinddir has been
// called (since fuse gives us no way to intercept and know for sure), and
// start the listing process over again.
//
// EXCLUSIVE_LOCKS_REQUIRED(dh.Mu)
// LOCKS_EXCLUDED(dh.in.Mu)
func (dh *dirHandle) ReadDir(
	ctx context.Context,
	req *fuse.ReadDirRequest) (resp *fuse.ReadDirResponse, err error) {
	resp = &fuse.ReadDirResponse{}

	dh.in.Mu.RLock()
	defer dh.in.Mu.RUnlock()

	// If the request is for offset zero, we assume that either this is the first
	// call or rewinddir has been called. Reset state.
	if req.Offset == 0 {
		dh.entries = nil
		dh.entriesOffset = 0
		dh.tok = ""
	}

	// Is the offset from before what we have buffered? If not, this represents a
	// seekdir we cannot support, as discussed in the method comments above.
	if req.Offset < dh.entriesOffset {
		err = fuse.EINVAL
		return
	}

	// Is the offset past the end of what we have buffered? If so, this must be
	// an invalid seekdir according to posix.
	index := int(req.Offset - dh.entriesOffset)
	if index > len(dh.entries) {
		err = fuse.EINVAL
		return
	}

	// Do we need to read more entries?
	if index == len(dh.entries) {
		var newEntries []fuseutil.Dirent
		var newTok string

		newEntries, newTok, err = readEntries(
			ctx,
			dh.in,
			dh.tok,
			dh.entriesOffset+fuse.DirOffset(len(dh.entries)))

		// For EOF we simply return an empty listing.
		if err == io.EOF {
			err = nil
			return
		}

		// Propagate other errors.
		if err != nil {
			err = fmt.Errorf("readEntries: %v", err)
			return
		}

		// Update state.
		dh.entriesOffset += fuse.DirOffset(len(dh.entries))
		dh.entries = newEntries
		dh.tok = newTok

		// Now we want to read from the front of dh.entries.
		index = 0
	}

	// Now we copy out entries until we run out of entries or space.
	for i := index; i < len(dh.entries); i++ {
		resp.Data = fuseutil.AppendDirent(resp.Data, dh.entries[i])
		if len(resp.Data) > req.Size {
			resp.Data = resp.Data[:req.Size]
			break
		}
	}

	return
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package inode

import (
	"fmt"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"github.com/jacobsa/gcsfuse/gcsproxy"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

// TODO(jacobsa): Add a Destroy method here that calls ObjectProxy.Destroy, and
// make sure it's called when the inode is forgotten. Also, make sure package
// fuse has support for actually calling Forget.
type FileInode struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	bucket gcs.Bucket

	/////////////////////////
	// Constant data
	/////////////////////////

	id fuse.InodeID

	/////////////////////////
	// Mutable state
	/////////////////////////

	// A mutex that must be held when calling certain methods. See documentation
	// for each method.
	mu syncutil.InvariantMutex

	// A proxy for the backing object in GCS.
	//
	// INVARIANT: proxy.CheckInvariants() does not panic
	//
	// GUARDED_BY(mu)
	proxy *gcsproxy.ObjectProxy
}

var _ Inode = &FileInode{}

// Create a file inode for the given object in GCS.
//
// REQUIRES: o != nil
// REQUIRES: o.Generation > 0
// REQUIRES: len(o.Name) > 0
// REQUIRES: o.Name[len(o.Name)-1] != '/'
func NewFileInode(
	ctx context.Context,
	bucket gcs.Bucket,
	id fuse.InodeID,
	o *storage.Object) (f *FileInode, err error) {
	// Set up the basic struct.
	f = &FileInode{
		bucket: bucket,
		id:     id,
	}

	// Set up the proxy.
	f.proxy, err = gcsproxy.NewObjectProxy(ctx, bucket, o)
	if err != nil {
		err = fmt.Errorf("NewObjectProxy: %v", err)
		return
	}

	// Set up invariant checking.
	f.mu = syncutil.NewInvariantMutex(f.checkInvariants)

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func (f *FileInode) checkInvariants() {
	// Make sure the name is legal.
	name := f.proxy.Name()
	if len(name) == 0 || name[len(name)-1] == '/' {
		panic("Illegal file name: " + name)
	}

	// INVARIANT: proxy.CheckInvariants() does not panic
	f.proxy.CheckInvariants()
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

func (f *FileInode) Lock() {
	f.mu.Lock()
}

func (f *FileInode) Unlock() {
	f.mu.Unlock()
}

func (f *FileInode) ID() fuse.InodeID {
	return f.id
}

// EXCLUSIVE_LOCKS_REQUIRED(f.mu)
func (f *FileInode) Name() string {
	return f.proxy.Name()
}

// Return the generation number from which this inode was branched. This is
// used as a precondition in object write requests.
//
// TODO(jacobsa): Make sure to add a test for opening a file with O_CREAT then
// opening it again for reading, and sharing data across the two descriptors.
// This should fail if we have screwed up the fuse lookup process with regards
// to the zero generation.
//
// EXCLUSIVE_LOCKS_REQUIRED(f.mu)
func (f *FileInode) SourceGeneration() int64 {
	return f.proxy.SourceGeneration()
}

// EXCLUSIVE_LOCKS_REQUIRED(f.mu)
func (f *FileInode) Attributes(
	ctx context.Context) (attrs fuse.InodeAttributes, err error) {
	// Stat the object.
	size, _, err := f.proxy.Stat(ctx)
	if err != nil {
		err = fmt.Errorf("Stat: %v", err)
		return
	}

	// Fill out the struct.
	//
	// TODO(jacobsa): Add a test for nlink == 0 when clobbered and update the
	// code here.
	//
	// TODO(jacobsa): Make ObjectProxy.Stat return a struct containing mtime as
	// well as size and clobbered. (Get mtime from the local file when around,
	// otherwise the source object.) Then include Mtime here. But first make sure
	// there is a failing test.
	attrs = fuse.InodeAttributes{
		Size: uint64(size),
		Mode: 0700,
	}

	return
}

// Serve a read request for this file.
//
// EXCLUSIVE_LOCKS_REQUIRED(f.mu)
func (f *FileInode) ReadFile(
	ctx context.Context,
	req *fuse.ReadFileRequest) (resp *fuse.ReadFileResponse, err error)
<commit_msg>Implemented FileInode.ReadFile.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package inode

import (
	"fmt"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"github.com/jacobsa/gcsfuse/gcsproxy"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

// TODO(jacobsa): Add a Destroy method here that calls ObjectProxy.Destroy, and
// make sure it's called when the inode is forgotten. Also, make sure package
// fuse has support for actually calling Forget.
type FileInode struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	bucket gcs.Bucket

	/////////////////////////
	// Constant data
	/////////////////////////

	id fuse.InodeID

	/////////////////////////
	// Mutable state
	/////////////////////////

	// A mutex that must be held when calling certain methods. See documentation
	// for each method.
	mu syncutil.InvariantMutex

	// A proxy for the backing object in GCS.
	//
	// INVARIANT: proxy.CheckInvariants() does not panic
	//
	// GUARDED_BY(mu)
	proxy *gcsproxy.ObjectProxy
}

var _ Inode = &FileInode{}

// Create a file inode for the given object in GCS.
//
// REQUIRES: o != nil
// REQUIRES: o.Generation > 0
// REQUIRES: len(o.Name) > 0
// REQUIRES: o.Name[len(o.Name)-1] != '/'
func NewFileInode(
	ctx context.Context,
	bucket gcs.Bucket,
	id fuse.InodeID,
	o *storage.Object) (f *FileInode, err error) {
	// Set up the basic struct.
	f = &FileInode{
		bucket: bucket,
		id:     id,
	}

	// Set up the proxy.
	f.proxy, err = gcsproxy.NewObjectProxy(ctx, bucket, o)
	if err != nil {
		err = fmt.Errorf("NewObjectProxy: %v", err)
		return
	}

	// Set up invariant checking.
	f.mu = syncutil.NewInvariantMutex(f.checkInvariants)

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func (f *FileInode) checkInvariants() {
	// Make sure the name is legal.
	name := f.proxy.Name()
	if len(name) == 0 || name[len(name)-1] == '/' {
		panic("Illegal file name: " + name)
	}

	// INVARIANT: proxy.CheckInvariants() does not panic
	f.proxy.CheckInvariants()
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

func (f *FileInode) Lock() {
	f.mu.Lock()
}

func (f *FileInode) Unlock() {
	f.mu.Unlock()
}

func (f *FileInode) ID() fuse.InodeID {
	return f.id
}

// EXCLUSIVE_LOCKS_REQUIRED(f.mu)
func (f *FileInode) Name() string {
	return f.proxy.Name()
}

// Return the generation number from which this inode was branched. This is
// used as a precondition in object write requests.
//
// TODO(jacobsa): Make sure to add a test for opening a file with O_CREAT then
// opening it again for reading, and sharing data across the two descriptors.
// This should fail if we have screwed up the fuse lookup process with regards
// to the zero generation.
//
// EXCLUSIVE_LOCKS_REQUIRED(f.mu)
func (f *FileInode) SourceGeneration() int64 {
	return f.proxy.SourceGeneration()
}

// EXCLUSIVE_LOCKS_REQUIRED(f.mu)
func (f *FileInode) Attributes(
	ctx context.Context) (attrs fuse.InodeAttributes, err error) {
	// Stat the object.
	size, _, err := f.proxy.Stat(ctx)
	if err != nil {
		err = fmt.Errorf("Stat: %v", err)
		return
	}

	// Fill out the struct.
	//
	// TODO(jacobsa): Add a test for nlink == 0 when clobbered and update the
	// code here.
	//
	// TODO(jacobsa): Make ObjectProxy.Stat return a struct containing mtime as
	// well as size and clobbered. (Get mtime from the local file when around,
	// otherwise the source object.) Then include Mtime here. But first make sure
	// there is a failing test.
	attrs = fuse.InodeAttributes{
		Size: uint64(size),
		Mode: 0700,
	}

	return
}

// Serve a read request for this file.
//
// EXCLUSIVE_LOCKS_REQUIRED(f.mu)
func (f *FileInode) ReadFile(
	ctx context.Context,
	req *fuse.ReadFileRequest) (resp *fuse.ReadFileResponse, err error) {
	resp = &fuse.ReadFileResponse{}

	// Read from the proxy.
	//
	// TODO(jacobsa): Make sure we have a failing test for io.EOF here.
	buf := make([]byte, req.Size)
	n, err := f.proxy.ReadAt(ctx, buf, req.Offset)
	if err != nil {
		err = fmt.Errorf("ReadAt: %v", err)
		return
	}

	// Fill in the response.
	//
	// TODO(jacobsa): Make sure a test fails if we don't reslice here.
	resp.Data = buf[:n]

	return
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package inode

import (
	"fmt"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"github.com/jacobsa/gcsfuse/gcsproxy"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

// TODO(jacobsa): Add a Destroy method here that calls ObjectProxy.Destroy, and
// make sure it's called when the inode is forgotten. Also, make sure package
// fuse has support for actually calling Forget.
type FileInode struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	bucket gcs.Bucket

	/////////////////////////
	// Constant data
	/////////////////////////

	id fuse.InodeID

	/////////////////////////
	// Mutable state
	/////////////////////////

	// A mutex that must be held when calling certain methods. See documentation
	// for each method.
	mu syncutil.InvariantMutex

	// A proxy for the backing object in GCS.
	//
	// INVARIANT: proxy.CheckInvariants() does not panic
	//
	// GUARDED_BY(mu)
	proxy *gcsproxy.ObjectProxy
}

var _ Inode = &FileInode{}

// Create a file inode for the given object in GCS.
//
// REQUIRES: o != nil
// REQUIRES: o.Generation > 0
// REQUIRES: len(o.Name) > 0
// REQUIRES: o.Name[len(o.Name)-1] != '/'
func NewFileInode(
	ctx context.Context,
	bucket gcs.Bucket,
	id fuse.InodeID,
	o *storage.Object) (f *FileInode, err error) {
	// Set up the basic struct.
	f = &FileInode{
		bucket: bucket,
		id:     id,
	}

	// Set up the proxy.
	f.proxy, err = gcsproxy.NewObjectProxy(ctx, bucket, o)
	if err != nil {
		err = fmt.Errorf("NewObjectProxy: %v", err)
		return
	}

	// Set up invariant checking.
	f.mu = syncutil.NewInvariantMutex(f.checkInvariants)

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func (f *FileInode) checkInvariants() {
	// Make sure the name is legal.
	name := f.proxy.Name()
	if len(name) == 0 || name[len(name)-1] == '/' {
		panic("Illegal file name: " + name)
	}

	// INVARIANT: proxy.CheckInvariants() does not panic
	f.proxy.CheckInvariants()
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

func (f *FileInode) Lock() {
	f.mu.Lock()
}

func (f *FileInode) Unlock() {
	f.mu.Unlock()
}

func (f *FileInode) ID() fuse.InodeID {
	return f.id
}

// SHARED_LOCKS_REQUIRED(f.mu)
func (f *FileInode) Name() string {
	return f.proxy.Name()
}

// Return the generation number from which this inode was branched. This is
// used as a precondition in object write requests.
//
// TODO(jacobsa): Make sure to add a test for opening a file with O_CREAT then
// opening it again for reading, and sharing data across the two descriptors.
// This should fail if we have screwed up the fuse lookup process with regards
// to the zero generation.
//
// SHARED_LOCKS_REQUIRED(f.mu)
func (f *FileInode) SourceGeneration() int64 {
	return f.proxy.SourceGeneration()
}

// SHARED_LOCKS_REQUIRED(f.mu)
func (f *FileInode) Attributes(
	ctx context.Context) (attrs fuse.InodeAttributes, err error) {
	// Stat the object.
	size, _, err := f.proxy.Stat(ctx)
	if err != nil {
		err = fmt.Errorf("Stat: %v", err)
		return
	}

	// Fill out the struct.
	//
	// TODO(jacobsa): Add a test for nlink == 0 when clobbered and update the
	// code here.
	//
	// TODO(jacobsa): Make ObjectProxy.Stat return a struct containing mtime as
	// well as size and clobbered. (Get mtime from the local file when around,
	// otherwise the source object.) Then include Mtime here. But first make sure
	// there is a failing test.
	attrs = fuse.InodeAttributes{
		Size: uint64(size),
		Mode: 0700,
	}

	return
}
<commit_msg>Use EXCLUSIVE_LOCKS_REQUIRED.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package inode

import (
	"fmt"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"github.com/jacobsa/gcsfuse/gcsproxy"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

// TODO(jacobsa): Add a Destroy method here that calls ObjectProxy.Destroy, and
// make sure it's called when the inode is forgotten. Also, make sure package
// fuse has support for actually calling Forget.
type FileInode struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	bucket gcs.Bucket

	/////////////////////////
	// Constant data
	/////////////////////////

	id fuse.InodeID

	/////////////////////////
	// Mutable state
	/////////////////////////

	// A mutex that must be held when calling certain methods. See documentation
	// for each method.
	mu syncutil.InvariantMutex

	// A proxy for the backing object in GCS.
	//
	// INVARIANT: proxy.CheckInvariants() does not panic
	//
	// GUARDED_BY(mu)
	proxy *gcsproxy.ObjectProxy
}

var _ Inode = &FileInode{}

// Create a file inode for the given object in GCS.
//
// REQUIRES: o != nil
// REQUIRES: o.Generation > 0
// REQUIRES: len(o.Name) > 0
// REQUIRES: o.Name[len(o.Name)-1] != '/'
func NewFileInode(
	ctx context.Context,
	bucket gcs.Bucket,
	id fuse.InodeID,
	o *storage.Object) (f *FileInode, err error) {
	// Set up the basic struct.
	f = &FileInode{
		bucket: bucket,
		id:     id,
	}

	// Set up the proxy.
	f.proxy, err = gcsproxy.NewObjectProxy(ctx, bucket, o)
	if err != nil {
		err = fmt.Errorf("NewObjectProxy: %v", err)
		return
	}

	// Set up invariant checking.
	f.mu = syncutil.NewInvariantMutex(f.checkInvariants)

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func (f *FileInode) checkInvariants() {
	// Make sure the name is legal.
	name := f.proxy.Name()
	if len(name) == 0 || name[len(name)-1] == '/' {
		panic("Illegal file name: " + name)
	}

	// INVARIANT: proxy.CheckInvariants() does not panic
	f.proxy.CheckInvariants()
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

func (f *FileInode) Lock() {
	f.mu.Lock()
}

func (f *FileInode) Unlock() {
	f.mu.Unlock()
}

func (f *FileInode) ID() fuse.InodeID {
	return f.id
}

// EXCLUSIVE_LOCKS_REQUIRED(f.mu)
func (f *FileInode) Name() string {
	return f.proxy.Name()
}

// Return the generation number from which this inode was branched. This is
// used as a precondition in object write requests.
//
// TODO(jacobsa): Make sure to add a test for opening a file with O_CREAT then
// opening it again for reading, and sharing data across the two descriptors.
// This should fail if we have screwed up the fuse lookup process with regards
// to the zero generation.
//
// EXCLUSIVE_LOCKS_REQUIRED(f.mu)
func (f *FileInode) SourceGeneration() int64 {
	return f.proxy.SourceGeneration()
}

// EXCLUSIVE_LOCKS_REQUIRED(f.mu)
func (f *FileInode) Attributes(
	ctx context.Context) (attrs fuse.InodeAttributes, err error) {
	// Stat the object.
	size, _, err := f.proxy.Stat(ctx)
	if err != nil {
		err = fmt.Errorf("Stat: %v", err)
		return
	}

	// Fill out the struct.
	//
	// TODO(jacobsa): Add a test for nlink == 0 when clobbered and update the
	// code here.
	//
	// TODO(jacobsa): Make ObjectProxy.Stat return a struct containing mtime as
	// well as size and clobbered. (Get mtime from the local file when around,
	// otherwise the source object.) Then include Mtime here. But first make sure
	// there is a failing test.
	attrs = fuse.InodeAttributes{
		Size: uint64(size),
		Mode: 0700,
	}

	return
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"log"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsSpotInstanceRequest() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsSpotInstanceRequestCreate,
		Read:   resourceAwsSpotInstanceRequestRead,
		Delete: resourceAwsSpotInstanceRequestDelete,
		Update: resourceAwsSpotInstanceRequestUpdate,

		Schema: func() map[string]*schema.Schema {
			// The Spot Instance Request Schema is based on the AWS Instance schema.
			s := resourceAwsInstance().Schema

			// Everything on a spot instance is ForceNew except tags
			for k, v := range s {
				if k == "tags" {
					continue
				}
				v.ForceNew = true
			}

			s["spot_price"] = &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			}
			s["spot_type"] = &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Default:  "persistent",
			}
			s["wait_for_fulfillment"] = &schema.Schema{
				Type:     schema.TypeBool,
				Optional: true,
				Default:  false,
			}
			s["spot_bid_status"] = &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			}
			s["spot_request_state"] = &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			}
			s["spot_instance_id"] = &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			}
			s["block_duration_minutes"] = &schema.Schema{
				Type:     schema.TypeInt,
				Optional: true,
				ForceNew: true,
			}

			return s
		}(),
	}
}

func resourceAwsSpotInstanceRequestCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	instanceOpts, err := buildAwsInstanceOpts(d, meta)
	if err != nil {
		return err
	}

	spotOpts := &ec2.RequestSpotInstancesInput{
		SpotPrice: aws.String(d.Get("spot_price").(string)),
		Type:      aws.String(d.Get("spot_type").(string)),

		// Though the AWS API supports creating spot instance requests for multiple
		// instances, for TF purposes we fix this to one instance per request.
		// Users can get equivalent behavior out of TF's "count" meta-parameter.
		InstanceCount: aws.Int64(1),

		LaunchSpecification: &ec2.RequestSpotLaunchSpecification{
			BlockDeviceMappings: instanceOpts.BlockDeviceMappings,
			EbsOptimized:        instanceOpts.EBSOptimized,
			Monitoring:          instanceOpts.Monitoring,
			IamInstanceProfile:  instanceOpts.IAMInstanceProfile,
			ImageId:             instanceOpts.ImageID,
			InstanceType:        instanceOpts.InstanceType,
			KeyName:             instanceOpts.KeyName,
			Placement:           instanceOpts.SpotPlacement,
			SecurityGroupIds:    instanceOpts.SecurityGroupIDs,
			SecurityGroups:      instanceOpts.SecurityGroups,
			SubnetId:            instanceOpts.SubnetID,
			UserData:            instanceOpts.UserData64,
		},
	}

	if v, ok := d.GetOk("block_duration_minutes"); ok {
		spotOpts.BlockDurationMinutes = aws.Int64(int64(v.(int)))
	}

	// If the instance is configured with a Network Interface (a subnet, has
	// public IP, etc), then the instanceOpts.SecurityGroupIds and SubnetId will
	// be nil
	if len(instanceOpts.NetworkInterfaces) > 0 {
		spotOpts.LaunchSpecification.SecurityGroupIds = instanceOpts.NetworkInterfaces[0].Groups
		spotOpts.LaunchSpecification.SubnetId = instanceOpts.NetworkInterfaces[0].SubnetId
	}

	// Make the spot instance request
	log.Printf("[DEBUG] Requesting spot bid opts: %s", spotOpts)

	var resp *ec2.RequestSpotInstancesOutput
	err = resource.Retry(15*time.Second, func() *resource.RetryError {
		var err error
		resp, err = conn.RequestSpotInstances(spotOpts)
		// IAM instance profiles can take ~10 seconds to propagate in AWS:
		// http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#launch-instance-with-role-console
		if isAWSErr(err, "InvalidParameterValue", "Invalid IAM Instance Profile") {
			log.Printf("[DEBUG] Invalid IAM Instance Profile referenced, retrying...")
			return resource.RetryableError(err)
		}
		// IAM roles can also take time to propagate in AWS:
		if isAWSErr(err, "InvalidParameterValue", " has no associated IAM Roles") {
			log.Printf("[DEBUG] IAM Instance Profile appears to have no IAM roles, retrying...")
			return resource.RetryableError(err)
		}
		return resource.NonRetryableError(err)
	})

	if err != nil {
		return fmt.Errorf("Error requesting spot instances: %s", err)
	}
	if len(resp.SpotInstanceRequests) != 1 {
		return fmt.Errorf(
			"Expected response with length 1, got: %s", resp)
	}

	sir := *resp.SpotInstanceRequests[0]
	d.SetId(*sir.SpotInstanceRequestId)

	if d.Get("wait_for_fulfillment").(bool) {
		spotStateConf := &resource.StateChangeConf{
			// http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-bid-status.html
			Pending:    []string{"start", "pending-evaluation", "pending-fulfillment"},
			Target:     []string{"fulfilled"},
			Refresh:    SpotInstanceStateRefreshFunc(conn, sir),
			Timeout:    10 * time.Minute,
			Delay:      10 * time.Second,
			MinTimeout: 3 * time.Second,
		}

		log.Printf("[DEBUG] waiting for spot bid to resolve... this may take several minutes.")
		_, err = spotStateConf.WaitForState()

		if err != nil {
			return fmt.Errorf("Error while waiting for spot request (%s) to resolve: %s", sir, err)
		}
	}

	return resourceAwsSpotInstanceRequestUpdate(d, meta)
}

// Update spot state, etc
func resourceAwsSpotInstanceRequestRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	req := &ec2.DescribeSpotInstanceRequestsInput{
		SpotInstanceRequestIds: []*string{aws.String(d.Id())},
	}
	resp, err := conn.DescribeSpotInstanceRequests(req)

	if err != nil {
		// If the spot request was not found, return nil so that we can show
		// that it is gone.
		if ec2err, ok := err.(awserr.Error); ok && ec2err.Code() == "InvalidSpotInstanceRequestID.NotFound" {
			d.SetId("")
			return nil
		}

		// Some other error, report it
		return err
	}

	// If nothing was found, then return no state
	if len(resp.SpotInstanceRequests) == 0 {
		d.SetId("")
		return nil
	}

	request := resp.SpotInstanceRequests[0]

	// if the request is cancelled, then it is gone
	if *request.State == "cancelled" {
		d.SetId("")
		return nil
	}

	d.Set("spot_bid_status", *request.Status.Code)
	// Instance ID is not set if the request is still pending
	if request.InstanceId != nil {
		d.Set("spot_instance_id", *request.InstanceId)
		// Read the instance data, setting up connection information
		if err := readInstance(d, meta); err != nil {
			return fmt.Errorf("[ERR] Error reading Spot Instance Data: %s", err)
		}
	}

	d.Set("spot_request_state", request.State)
	d.Set("block_duration_minutes", request.BlockDurationMinutes)
	d.Set("tags", tagsToMap(request.Tags))

	return nil
}

func readInstance(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	resp, err := conn.DescribeInstances(&ec2.DescribeInstancesInput{
		InstanceIds: []*string{aws.String(d.Get("spot_instance_id").(string))},
	})
	if err != nil {
		// If the instance was not found, return nil so that we can show
		// that the instance is gone.
		if ec2err, ok := err.(awserr.Error); ok && ec2err.Code() == "InvalidInstanceID.NotFound" {
			return fmt.Errorf("no instance found")
		}

		// Some other error, report it
		return err
	}

	// If nothing was found, then return no state
	if len(resp.Reservations) == 0 {
		return fmt.Errorf("no instances found")
	}

	instance := resp.Reservations[0].Instances[0]

	// Set these fields for connection information
	if instance != nil {
		d.Set("public_dns", instance.PublicDnsName)
		d.Set("public_ip", instance.PublicIpAddress)
		d.Set("private_dns", instance.PrivateDnsName)
		d.Set("private_ip", instance.PrivateIpAddress)

		// set connection information
		if instance.PublicIpAddress != nil {
			d.SetConnInfo(map[string]string{
				"type": "ssh",
				"host": *instance.PublicIpAddress,
			})
		} else if instance.PrivateIpAddress != nil {
			d.SetConnInfo(map[string]string{
				"type": "ssh",
				"host": *instance.PrivateIpAddress,
			})
		}
		if err := readBlockDevices(d, instance, conn); err != nil {
			return err
		}

		var ipv6Addresses []string
		if len(instance.NetworkInterfaces) > 0 {
			for _, ni := range instance.NetworkInterfaces {
				if *ni.Attachment.DeviceIndex == 0 {
					d.Set("subnet_id", ni.SubnetId)
					d.Set("network_interface_id", ni.NetworkInterfaceId)
					d.Set("associate_public_ip_address", ni.Association != nil)
					d.Set("ipv6_address_count", len(ni.Ipv6Addresses))

					for _, address := range ni.Ipv6Addresses {
						ipv6Addresses = append(ipv6Addresses, *address.Ipv6Address)
					}
				}
			}
		} else {
			d.Set("subnet_id", instance.SubnetId)
			d.Set("network_interface_id", "")
		}

		if err := d.Set("ipv6_addresses", ipv6Addresses); err != nil {
			log.Printf("[WARN] Error setting ipv6_addresses for AWS Spot Instance (%s): %s", d.Id(), err)
		}
	}

	return nil
}

func resourceAwsSpotInstanceRequestUpdate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	d.Partial(true)
	if err := setTags(conn, d); err != nil {
		return err
	} else {
		d.SetPartial("tags")
	}

	d.Partial(false)

	return resourceAwsSpotInstanceRequestRead(d, meta)
}

func resourceAwsSpotInstanceRequestDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	log.Printf("[INFO] Cancelling spot request: %s", d.Id())
	_, err := conn.CancelSpotInstanceRequests(&ec2.CancelSpotInstanceRequestsInput{
		SpotInstanceRequestIds: []*string{aws.String(d.Id())},
	})

	if err != nil {
		return fmt.Errorf("Error cancelling spot request (%s): %s", d.Id(), err)
	}

	if instanceId := d.Get("spot_instance_id").(string); instanceId != "" {
		log.Printf("[INFO] Terminating instance: %s", instanceId)
		if err := awsTerminateInstance(conn, instanceId); err != nil {
			return fmt.Errorf("Error terminating spot instance: %s", err)
		}
	}

	return nil
}

// SpotInstanceStateRefreshFunc returns a resource.StateRefreshFunc that is used to watch
// an EC2 spot instance request
func SpotInstanceStateRefreshFunc(
	conn *ec2.EC2, sir ec2.SpotInstanceRequest) resource.StateRefreshFunc {

	return func() (interface{}, string, error) {
		resp, err := conn.DescribeSpotInstanceRequests(&ec2.DescribeSpotInstanceRequestsInput{
			SpotInstanceRequestIds: []*string{sir.SpotInstanceRequestId},
		})

		if err != nil {
			if ec2err, ok := err.(awserr.Error); ok && ec2err.Code() == "InvalidSpotInstanceRequestID.NotFound" {
				// Set this to nil as if we didn't find anything.
				resp = nil
			} else {
				log.Printf("Error on StateRefresh: %s", err)
				return nil, "", err
			}
		}

		if resp == nil || len(resp.SpotInstanceRequests) == 0 {
			// Sometimes AWS just has consistency issues and doesn't see
			// our request yet. Return an empty state.
			return nil, "", nil
		}

		req := resp.SpotInstanceRequests[0]
		return req, *req.Status.Code, nil
	}
}
<commit_msg>provider/aws: aws_spot_instance_request not forcenew on volume_tags (#14046)<commit_after>package aws

import (
	"fmt"
	"log"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsSpotInstanceRequest() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsSpotInstanceRequestCreate,
		Read:   resourceAwsSpotInstanceRequestRead,
		Delete: resourceAwsSpotInstanceRequestDelete,
		Update: resourceAwsSpotInstanceRequestUpdate,

		Schema: func() map[string]*schema.Schema {
			// The Spot Instance Request Schema is based on the AWS Instance schema.
			s := resourceAwsInstance().Schema

			// Everything on a spot instance is ForceNew except tags
			for k, v := range s {
				if k == "tags" || k == "volume_tags" {
					continue
				}
				v.ForceNew = true
			}

			s["spot_price"] = &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			}
			s["spot_type"] = &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Default:  "persistent",
			}
			s["wait_for_fulfillment"] = &schema.Schema{
				Type:     schema.TypeBool,
				Optional: true,
				Default:  false,
			}
			s["spot_bid_status"] = &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			}
			s["spot_request_state"] = &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			}
			s["spot_instance_id"] = &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			}
			s["block_duration_minutes"] = &schema.Schema{
				Type:     schema.TypeInt,
				Optional: true,
				ForceNew: true,
			}

			return s
		}(),
	}
}

func resourceAwsSpotInstanceRequestCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	instanceOpts, err := buildAwsInstanceOpts(d, meta)
	if err != nil {
		return err
	}

	spotOpts := &ec2.RequestSpotInstancesInput{
		SpotPrice: aws.String(d.Get("spot_price").(string)),
		Type:      aws.String(d.Get("spot_type").(string)),

		// Though the AWS API supports creating spot instance requests for multiple
		// instances, for TF purposes we fix this to one instance per request.
		// Users can get equivalent behavior out of TF's "count" meta-parameter.
		InstanceCount: aws.Int64(1),

		LaunchSpecification: &ec2.RequestSpotLaunchSpecification{
			BlockDeviceMappings: instanceOpts.BlockDeviceMappings,
			EbsOptimized:        instanceOpts.EBSOptimized,
			Monitoring:          instanceOpts.Monitoring,
			IamInstanceProfile:  instanceOpts.IAMInstanceProfile,
			ImageId:             instanceOpts.ImageID,
			InstanceType:        instanceOpts.InstanceType,
			KeyName:             instanceOpts.KeyName,
			Placement:           instanceOpts.SpotPlacement,
			SecurityGroupIds:    instanceOpts.SecurityGroupIDs,
			SecurityGroups:      instanceOpts.SecurityGroups,
			SubnetId:            instanceOpts.SubnetID,
			UserData:            instanceOpts.UserData64,
		},
	}

	if v, ok := d.GetOk("block_duration_minutes"); ok {
		spotOpts.BlockDurationMinutes = aws.Int64(int64(v.(int)))
	}

	// If the instance is configured with a Network Interface (a subnet, has
	// public IP, etc), then the instanceOpts.SecurityGroupIds and SubnetId will
	// be nil
	if len(instanceOpts.NetworkInterfaces) > 0 {
		spotOpts.LaunchSpecification.SecurityGroupIds = instanceOpts.NetworkInterfaces[0].Groups
		spotOpts.LaunchSpecification.SubnetId = instanceOpts.NetworkInterfaces[0].SubnetId
	}

	// Make the spot instance request
	log.Printf("[DEBUG] Requesting spot bid opts: %s", spotOpts)

	var resp *ec2.RequestSpotInstancesOutput
	err = resource.Retry(15*time.Second, func() *resource.RetryError {
		var err error
		resp, err = conn.RequestSpotInstances(spotOpts)
		// IAM instance profiles can take ~10 seconds to propagate in AWS:
		// http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#launch-instance-with-role-console
		if isAWSErr(err, "InvalidParameterValue", "Invalid IAM Instance Profile") {
			log.Printf("[DEBUG] Invalid IAM Instance Profile referenced, retrying...")
			return resource.RetryableError(err)
		}
		// IAM roles can also take time to propagate in AWS:
		if isAWSErr(err, "InvalidParameterValue", " has no associated IAM Roles") {
			log.Printf("[DEBUG] IAM Instance Profile appears to have no IAM roles, retrying...")
			return resource.RetryableError(err)
		}
		return resource.NonRetryableError(err)
	})

	if err != nil {
		return fmt.Errorf("Error requesting spot instances: %s", err)
	}
	if len(resp.SpotInstanceRequests) != 1 {
		return fmt.Errorf(
			"Expected response with length 1, got: %s", resp)
	}

	sir := *resp.SpotInstanceRequests[0]
	d.SetId(*sir.SpotInstanceRequestId)

	if d.Get("wait_for_fulfillment").(bool) {
		spotStateConf := &resource.StateChangeConf{
			// http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-bid-status.html
			Pending:    []string{"start", "pending-evaluation", "pending-fulfillment"},
			Target:     []string{"fulfilled"},
			Refresh:    SpotInstanceStateRefreshFunc(conn, sir),
			Timeout:    10 * time.Minute,
			Delay:      10 * time.Second,
			MinTimeout: 3 * time.Second,
		}

		log.Printf("[DEBUG] waiting for spot bid to resolve... this may take several minutes.")
		_, err = spotStateConf.WaitForState()

		if err != nil {
			return fmt.Errorf("Error while waiting for spot request (%s) to resolve: %s", sir, err)
		}
	}

	return resourceAwsSpotInstanceRequestUpdate(d, meta)
}

// Update spot state, etc
func resourceAwsSpotInstanceRequestRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	req := &ec2.DescribeSpotInstanceRequestsInput{
		SpotInstanceRequestIds: []*string{aws.String(d.Id())},
	}
	resp, err := conn.DescribeSpotInstanceRequests(req)

	if err != nil {
		// If the spot request was not found, return nil so that we can show
		// that it is gone.
		if ec2err, ok := err.(awserr.Error); ok && ec2err.Code() == "InvalidSpotInstanceRequestID.NotFound" {
			d.SetId("")
			return nil
		}

		// Some other error, report it
		return err
	}

	// If nothing was found, then return no state
	if len(resp.SpotInstanceRequests) == 0 {
		d.SetId("")
		return nil
	}

	request := resp.SpotInstanceRequests[0]

	// if the request is cancelled, then it is gone
	if *request.State == "cancelled" {
		d.SetId("")
		return nil
	}

	d.Set("spot_bid_status", *request.Status.Code)
	// Instance ID is not set if the request is still pending
	if request.InstanceId != nil {
		d.Set("spot_instance_id", *request.InstanceId)
		// Read the instance data, setting up connection information
		if err := readInstance(d, meta); err != nil {
			return fmt.Errorf("[ERR] Error reading Spot Instance Data: %s", err)
		}
	}

	d.Set("spot_request_state", request.State)
	d.Set("block_duration_minutes", request.BlockDurationMinutes)
	d.Set("tags", tagsToMap(request.Tags))

	return nil
}

func readInstance(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	resp, err := conn.DescribeInstances(&ec2.DescribeInstancesInput{
		InstanceIds: []*string{aws.String(d.Get("spot_instance_id").(string))},
	})
	if err != nil {
		// If the instance was not found, return nil so that we can show
		// that the instance is gone.
		if ec2err, ok := err.(awserr.Error); ok && ec2err.Code() == "InvalidInstanceID.NotFound" {
			return fmt.Errorf("no instance found")
		}

		// Some other error, report it
		return err
	}

	// If nothing was found, then return no state
	if len(resp.Reservations) == 0 {
		return fmt.Errorf("no instances found")
	}

	instance := resp.Reservations[0].Instances[0]

	// Set these fields for connection information
	if instance != nil {
		d.Set("public_dns", instance.PublicDnsName)
		d.Set("public_ip", instance.PublicIpAddress)
		d.Set("private_dns", instance.PrivateDnsName)
		d.Set("private_ip", instance.PrivateIpAddress)

		// set connection information
		if instance.PublicIpAddress != nil {
			d.SetConnInfo(map[string]string{
				"type": "ssh",
				"host": *instance.PublicIpAddress,
			})
		} else if instance.PrivateIpAddress != nil {
			d.SetConnInfo(map[string]string{
				"type": "ssh",
				"host": *instance.PrivateIpAddress,
			})
		}
		if err := readBlockDevices(d, instance, conn); err != nil {
			return err
		}

		var ipv6Addresses []string
		if len(instance.NetworkInterfaces) > 0 {
			for _, ni := range instance.NetworkInterfaces {
				if *ni.Attachment.DeviceIndex == 0 {
					d.Set("subnet_id", ni.SubnetId)
					d.Set("network_interface_id", ni.NetworkInterfaceId)
					d.Set("associate_public_ip_address", ni.Association != nil)
					d.Set("ipv6_address_count", len(ni.Ipv6Addresses))

					for _, address := range ni.Ipv6Addresses {
						ipv6Addresses = append(ipv6Addresses, *address.Ipv6Address)
					}
				}
			}
		} else {
			d.Set("subnet_id", instance.SubnetId)
			d.Set("network_interface_id", "")
		}

		if err := d.Set("ipv6_addresses", ipv6Addresses); err != nil {
			log.Printf("[WARN] Error setting ipv6_addresses for AWS Spot Instance (%s): %s", d.Id(), err)
		}
	}

	return nil
}

func resourceAwsSpotInstanceRequestUpdate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	d.Partial(true)
	if err := setTags(conn, d); err != nil {
		return err
	} else {
		d.SetPartial("tags")
	}

	d.Partial(false)

	return resourceAwsSpotInstanceRequestRead(d, meta)
}

func resourceAwsSpotInstanceRequestDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	log.Printf("[INFO] Cancelling spot request: %s", d.Id())
	_, err := conn.CancelSpotInstanceRequests(&ec2.CancelSpotInstanceRequestsInput{
		SpotInstanceRequestIds: []*string{aws.String(d.Id())},
	})

	if err != nil {
		return fmt.Errorf("Error cancelling spot request (%s): %s", d.Id(), err)
	}

	if instanceId := d.Get("spot_instance_id").(string); instanceId != "" {
		log.Printf("[INFO] Terminating instance: %s", instanceId)
		if err := awsTerminateInstance(conn, instanceId); err != nil {
			return fmt.Errorf("Error terminating spot instance: %s", err)
		}
	}

	return nil
}

// SpotInstanceStateRefreshFunc returns a resource.StateRefreshFunc that is used to watch
// an EC2 spot instance request
func SpotInstanceStateRefreshFunc(
	conn *ec2.EC2, sir ec2.SpotInstanceRequest) resource.StateRefreshFunc {

	return func() (interface{}, string, error) {
		resp, err := conn.DescribeSpotInstanceRequests(&ec2.DescribeSpotInstanceRequestsInput{
			SpotInstanceRequestIds: []*string{sir.SpotInstanceRequestId},
		})

		if err != nil {
			if ec2err, ok := err.(awserr.Error); ok && ec2err.Code() == "InvalidSpotInstanceRequestID.NotFound" {
				// Set this to nil as if we didn't find anything.
				resp = nil
			} else {
				log.Printf("Error on StateRefresh: %s", err)
				return nil, "", err
			}
		}

		if resp == nil || len(resp.SpotInstanceRequests) == 0 {
			// Sometimes AWS just has consistency issues and doesn't see
			// our request yet. Return an empty state.
			return nil, "", nil
		}

		req := resp.SpotInstanceRequests[0]
		return req, *req.Status.Code, nil
	}
}
<|endoftext|>