<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package math provides basic constants and mathematical functions.
package math

// Mathematical constants.
// Reference: http://oeis.org/Axxxxxx
const (
	E   = 2.71828182845904523536028747135266249775724709369995957496696763 // A001113
	Pi  = 3.14159265358979323846264338327950288419716939937510582097494459 // A000796
	Phi = 1.61803398874989484820458683436563811772030917980576286213544862 // A001622

	Sqrt2   = 1.41421356237309504880168872420969807856967187537694807317667974 // A002193
	SqrtE   = 1.64872127070012814684865078781416357165377610071014801157507931 // A019774
	SqrtPi  = 1.77245385090551602729816748334114518279754945612238712821380779 // A002161
	SqrtPhi = 1.27201964951406896425242246173749149171560804184009624861664038 // A139339

	Ln2    = 0.693147180559945309417232121458176568075500134360255254120680009 // A002162
	Log2E  = 1 / Ln2
	Ln10   = 2.30258509299404568401799145468436420760110148862877297603332790 // A002392
	Log10E = 1 / Ln10
)

// Floating-point limit values.
// Max is the largest finite value representable by the type.
// SmallestNonzero is the smallest positive, non-zero value representable by the type.
const (
	MaxFloat32             = 3.40282346638528859811704183484516925440e+38  /* 2**127 * (2**24 - 1) / 2**23 */
	SmallestNonzeroFloat32 = 1.401298464324817070923729583289916131280e-45 /* 1 / 2**(127 - 1 + 23) */

	MaxFloat64             = 1.797693134862315708145274237317043567981e+308 /* 2**1023 * (2**53 - 1) / 2**52 */
	SmallestNonzeroFloat64 = 4.940656458412465441765687928682213723651e-324 /* 1 / 2**(1023 - 1 + 52) */
)

// Integer limit values.
const (
	MaxInt8   = 1<<7 - 1
	MinInt8   = -1 << 7
	MaxInt16  = 1<<15 - 1
	MinInt16  = -1 << 15
	MaxInt32  = 1<<31 - 1
	MinInt32  = -1 << 31
	MaxInt64  = 1<<63 - 1
	MinInt64  = -1 << 63
	MaxUint8  = 1<<8 - 1
	MaxUint16 = 1<<16 - 1
	MaxUint32 = 1<<32 - 1
	MaxUint64 = 1<<64 - 1
)
<commit_msg>math: slightly more readable comments<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package math provides basic constants and mathematical functions.
package math

// Mathematical constants.
// Reference: http://oeis.org/Axxxxxx
const (
	E   = 2.71828182845904523536028747135266249775724709369995957496696763 // A001113
	Pi  = 3.14159265358979323846264338327950288419716939937510582097494459 // A000796
	Phi = 1.61803398874989484820458683436563811772030917980576286213544862 // A001622

	Sqrt2   = 1.41421356237309504880168872420969807856967187537694807317667974 // A002193
	SqrtE   = 1.64872127070012814684865078781416357165377610071014801157507931 // A019774
	SqrtPi  = 1.77245385090551602729816748334114518279754945612238712821380779 // A002161
	SqrtPhi = 1.27201964951406896425242246173749149171560804184009624861664038 // A139339

	Ln2    = 0.693147180559945309417232121458176568075500134360255254120680009 // A002162
	Log2E  = 1 / Ln2
	Ln10   = 2.30258509299404568401799145468436420760110148862877297603332790 // A002392
	Log10E = 1 / Ln10
)

// Floating-point limit values.
// Max is the largest finite value representable by the type.
// SmallestNonzero is the smallest positive, non-zero value representable by the type.
const (
	MaxFloat32             = 3.40282346638528859811704183484516925440e+38  // 2**127 * (2**24 - 1) / 2**23
	SmallestNonzeroFloat32 = 1.401298464324817070923729583289916131280e-45 // 1 / 2**(127 - 1 + 23)

	MaxFloat64             = 1.797693134862315708145274237317043567981e+308 // 2**1023 * (2**53 - 1) / 2**52
	SmallestNonzeroFloat64 = 4.940656458412465441765687928682213723651e-324 // 1 / 2**(1023 - 1 + 52)
)

// Integer limit values.
const (
	MaxInt8   = 1<<7 - 1
	MinInt8   = -1 << 7
	MaxInt16  = 1<<15 - 1
	MinInt16  = -1 << 15
	MaxInt32  = 1<<31 - 1
	MinInt32  = -1 << 31
	MaxInt64  = 1<<63 - 1
	MinInt64  = -1 << 63
	MaxUint8  = 1<<8 - 1
	MaxUint16 = 1<<16 - 1
	MaxUint32 = 1<<32 - 1
	MaxUint64 = 1<<64 - 1
)
<|endoftext|>


<commit_before>/*
fixityworker.go receives GenericFile records from the fixity queue.
It downloads the generic files from S3 preservation storage,
calculates the files' SHA256 checksums and writes the results back
to Fluctus. None of the data downloaded from S3 is saved to disk;
it's simply streamed through the SHA256 hash writer and then discarded.
*/
package workers

import (
	"encoding/json"
	"fmt"
	"github.com/APTrust/bagman/bagman"
	"github.com/bitly/go-nsq"
	"strings"
	"time"
)

type FixityChecker struct {
	FixityChannel  chan *bagman.FixityResult
	ResultsChannel chan *bagman.FixityResult
	ProcUtil       *bagman.ProcessUtil
}

func NewFixityChecker(procUtil *bagman.ProcessUtil) (*FixityChecker) {
	fixityChecker := &FixityChecker{
		ProcUtil: procUtil,
	}
	workerBufferSize := procUtil.Config.BagDeleteWorker.Workers * 10
	fixityChecker.FixityChannel = make(chan *bagman.FixityResult, workerBufferSize)
	fixityChecker.ResultsChannel = make(chan *bagman.FixityResult, workerBufferSize)
	for i := 0; i < procUtil.Config.BagDeleteWorker.Workers; i++ {
		go fixityChecker.logResult()
		go fixityChecker.checkFile()
	}
	return fixityChecker
}

// MessageHandler handles messages from the queue, putting each
// item into the pipleline.
func (fixityChecker *FixityChecker) HandleMessage(message *nsq.Message) error {
	message.DisableAutoResponse()
	var genericFile bagman.GenericFile
	err := json.Unmarshal(message.Body, &genericFile)
	if err != nil {
		detailedError := fmt.Errorf(
			"Could not unmarshal JSON data from nsq: %v. JSON: %s",
			err, string(message.Body))
		fixityChecker.ProcUtil.MessageLog.Error(detailedError.Error())
		message.Finish()
		return detailedError
	}

	if genericFile.GetChecksum("sha256") == nil {
		detailedError := fmt.Errorf(
			"Cannot check GenericFile %s: it has no SHA256 checksum!",
			genericFile.Identifier)
		fixityChecker.ProcUtil.MessageLog.Error(detailedError.Error())
		message.Finish()
		return detailedError
	}

	result := bagman.NewFixityResult(&genericFile)
	result.NsqMessage = message
	fixityChecker.FixityChannel <- result
	fixityChecker.ProcUtil.MessageLog.Info("Put %s into fixity channel", genericFile.Identifier)
	return nil
}

func (fixityChecker *FixityChecker) logResult() {
	for result := range fixityChecker.ResultsChannel {
		fixityChecker.ProcUtil.MessageLog.Debug("Fedora digest = '%s' ... S3 digest = '%s'",
			result.FedoraSha256(), result.Sha256)
		if result.S3FileExists == false {
			fixityChecker.ProcUtil.MessageLog.Error("GenericFile '%s' with URL '%s' does not exist in S3",
				result.GenericFile.Identifier, result.GenericFile.URI)
			result.NsqMessage.Finish()
			fixityChecker.ProcUtil.IncrementFailed()
			fixityChecker.logStats()
			continue
		}
		// Check failure cases...
		if result.GotDigestFromPreservationFile() == false && result.Retry == true {
			if result.NsqMessage.Attempts >= uint16(fixityChecker.ProcUtil.Config.FixityWorker.MaxAttempts) {
				fixityChecker.ProcUtil.MessageLog.Error(
					"Attempt to calculate checksum for file %s at S3 URL %s has failed too many times. " +
						"This item will not be requeued.",
					result.GenericFile.Identifier,
					result.GenericFile.URI)
				result.NsqMessage.Finish()
				fixityChecker.ProcUtil.IncrementFailed()
				fixityChecker.logStats()
				// -----------------------------------------
				// TODO: SEND TO TROUBLE QUEUE!
				// -----------------------------------------
			} else {
				fixityChecker.ProcUtil.MessageLog.Error(
					"Requeueing %s because fetch from S3 failed, or read from S3 " +
						"datastream failed while calculating checksum. This item is being requeued.",
					result.GenericFile.Identifier)
				result.NsqMessage.Requeue(1 * time.Minute)
				fixityChecker.ProcUtil.IncrementFailed()
				fixityChecker.logStats()
			}
			continue
		}

		// If we got this far, we have enough info to compare checksums.
		eventSaved := fixityChecker.savePremisEvent(result)
		if eventSaved == false {
			fixityChecker.ProcUtil.MessageLog.Error(
				"Requeueing %s because attempt to save event to Fluctus failed",
				result.GenericFile.Identifier)
			result.NsqMessage.Requeue(1 * time.Minute)
			fixityChecker.ProcUtil.IncrementFailed()
		} else {
			fixityChecker.ProcUtil.MessageLog.Info("Finished with %s", result.GenericFile.Identifier)
			fixityChecker.ProcUtil.IncrementSucceeded()
			result.NsqMessage.Finish()
		}
		fixityChecker.logStats()
	}
}

func (fixityChecker *FixityChecker) savePremisEvent(fixityResult *bagman.FixityResult) (bool) {
	premisEvent, err := fixityResult.BuildPremisEvent()
	if err != nil {
		fixityChecker.ProcUtil.MessageLog.Error("Error building PremisEvent for %s: %v",
			fixityResult.GenericFile.Identifier, err)
		return false
	} else {
		_, err := fixityChecker.ProcUtil.FluctusClient.PremisEventSave(
			fixityResult.GenericFile.Identifier,
			"GenericFile",
			premisEvent)
		if err != nil {
			fixityChecker.ProcUtil.MessageLog.Error(
				"Error saving PremisEvent for %s to Fluctus: %v",
				fixityResult.GenericFile.Identifier, err)
			return false
		} else {
			fixityChecker.ProcUtil.MessageLog.Info("Saved PremisEvent for %s",
				fixityResult.GenericFile.Identifier)
		}
	}
	return true
}

func (fixityChecker *FixityChecker) logStats() {
	fixityChecker.ProcUtil.MessageLog.Info("**STATS** Succeeded: %d, Failed: %d",
		fixityChecker.ProcUtil.Succeeded(), fixityChecker.ProcUtil.Failed())
}

// Fetch the file and calculate its SHA256 digest. This may take hours if the
// file is 250GB. Touch the NSQ message on both sides of the operation to
// prevent it from timing out.
func (fixityChecker *FixityChecker) checkFile() {
	for result := range fixityChecker.FixityChannel {
		fixityChecker.ProcUtil.MessageLog.Info("Checking %s", result.GenericFile.Identifier)
		result.NsqMessage.Touch()
		err := fixityChecker.ProcUtil.S3Client.FetchAndCalculateSha256(result)
		// Log usage errors. These shouldn't happen.
		if err != nil && strings.Index(err.Error(), "cannot by nil") > 0 {
			fixityChecker.ProcUtil.MessageLog.Error(err.Error())
		}
		result.NsqMessage.Touch()
		fixityChecker.ResultsChannel <- result
	}
}
<commit_msg>If fixture check fails, send to failure queue.<commit_after>/*
fixityworker.go receives GenericFile records from the fixity queue.
It downloads the generic files from S3 preservation storage,
calculates the files' SHA256 checksums and writes the results back
to Fluctus. None of the data downloaded from S3 is saved to disk;
it's simply streamed through the SHA256 hash writer and then discarded.
*/
package workers

import (
	"encoding/json"
	"fmt"
	"github.com/APTrust/bagman/bagman"
	"github.com/bitly/go-nsq"
	"strings"
	"time"
)

type FixityChecker struct {
	FixityChannel  chan *bagman.FixityResult
	ResultsChannel chan *bagman.FixityResult
	ProcUtil       *bagman.ProcessUtil
}

func NewFixityChecker(procUtil *bagman.ProcessUtil) (*FixityChecker) {
	fixityChecker := &FixityChecker{
		ProcUtil: procUtil,
	}
	workerBufferSize := procUtil.Config.BagDeleteWorker.Workers * 10
	fixityChecker.FixityChannel = make(chan *bagman.FixityResult, workerBufferSize)
	fixityChecker.ResultsChannel = make(chan *bagman.FixityResult, workerBufferSize)
	for i := 0; i < procUtil.Config.BagDeleteWorker.Workers; i++ {
		go fixityChecker.logResult()
		go fixityChecker.checkFile()
	}
	return fixityChecker
}

// MessageHandler handles messages from the queue, putting each
// item into the pipleline.
func (fixityChecker *FixityChecker) HandleMessage(message *nsq.Message) error {
	message.DisableAutoResponse()
	var genericFile bagman.GenericFile
	err := json.Unmarshal(message.Body, &genericFile)
	if err != nil {
		detailedError := fmt.Errorf(
			"Could not unmarshal JSON data from nsq: %v. JSON: %s",
			err, string(message.Body))
		fixityChecker.ProcUtil.MessageLog.Error(detailedError.Error())
		message.Finish()
		return detailedError
	}

	if genericFile.GetChecksum("sha256") == nil {
		detailedError := fmt.Errorf(
			"Cannot check GenericFile %s: it has no SHA256 checksum!",
			genericFile.Identifier)
		fixityChecker.ProcUtil.MessageLog.Error(detailedError.Error())
		message.Finish()
		return detailedError
	}

	result := bagman.NewFixityResult(&genericFile)
	result.NsqMessage = message
	fixityChecker.FixityChannel <- result
	fixityChecker.ProcUtil.MessageLog.Info("Put %s into fixity channel", genericFile.Identifier)
	return nil
}

func (fixityChecker *FixityChecker) logResult() {
	for result := range fixityChecker.ResultsChannel {
		fixityChecker.ProcUtil.MessageLog.Debug("Fedora digest = '%s' ... S3 digest = '%s'",
			result.FedoraSha256(), result.Sha256)
		if result.S3FileExists == false {
			fixityChecker.ProcUtil.MessageLog.Error("GenericFile '%s' with URL '%s' does not exist in S3",
				result.GenericFile.Identifier, result.GenericFile.URI)
			result.NsqMessage.Finish()
			fixityChecker.ProcUtil.IncrementFailed()
			fixityChecker.logStats()
			continue
		}
		// Check failure cases...
		if result.GotDigestFromPreservationFile() == false && result.Retry == true {
			if result.NsqMessage.Attempts >= uint16(fixityChecker.ProcUtil.Config.FixityWorker.MaxAttempts) {
				fixityChecker.ProcUtil.MessageLog.Error(
					"Attempt to calculate checksum for file %s at S3 URL %s has failed too many times. " +
						"This item will not be requeued.",
					result.GenericFile.Identifier,
					result.GenericFile.URI)
				// Too many failures. Send to trouble queue.
				err := bagman.Enqueue(fixityChecker.ProcUtil.Config.NsqdHttpAddress,
					fixityChecker.ProcUtil.Config.FailedFixityWorker.NsqTopic, result)
				if err != nil {
					fixityChecker.ProcUtil.MessageLog.Error("Could not send '%s' to trouble queue: %v",
						result.GenericFile.Identifier, err)
				}
				result.NsqMessage.Finish()
				fixityChecker.ProcUtil.IncrementFailed()
				fixityChecker.logStats()
			} else {
				fixityChecker.ProcUtil.MessageLog.Error(
					"Requeueing %s because fetch from S3 failed, or read from S3 " +
						"datastream failed while calculating checksum. This item is being requeued.",
					result.GenericFile.Identifier)
				result.NsqMessage.Requeue(1 * time.Minute)
				fixityChecker.ProcUtil.IncrementFailed()
				fixityChecker.logStats()
			}
			continue
		}

		// If we got this far, we have enough info to compare checksums.
		eventSaved := fixityChecker.savePremisEvent(result)
		if eventSaved == false {
			fixityChecker.ProcUtil.MessageLog.Error(
				"Requeueing %s because attempt to save event to Fluctus failed",
				result.GenericFile.Identifier)
			result.NsqMessage.Requeue(1 * time.Minute)
			fixityChecker.ProcUtil.IncrementFailed()
		} else {
			fixityChecker.ProcUtil.MessageLog.Info("Finished with %s", result.GenericFile.Identifier)
			fixityChecker.ProcUtil.IncrementSucceeded()
			result.NsqMessage.Finish()
		}
		fixityChecker.logStats()
	}
}

func (fixityChecker *FixityChecker) savePremisEvent(fixityResult *bagman.FixityResult) (bool) {
	premisEvent, err := fixityResult.BuildPremisEvent()
	if err != nil {
		fixityChecker.ProcUtil.MessageLog.Error("Error building PremisEvent for %s: %v",
			fixityResult.GenericFile.Identifier, err)
		return false
	} else {
		_, err := fixityChecker.ProcUtil.FluctusClient.PremisEventSave(
			fixityResult.GenericFile.Identifier,
			"GenericFile",
			premisEvent)
		if err != nil {
			fixityChecker.ProcUtil.MessageLog.Error(
				"Error saving PremisEvent for %s to Fluctus: %v",
				fixityResult.GenericFile.Identifier, err)
			return false
		} else {
			fixityChecker.ProcUtil.MessageLog.Info("Saved PremisEvent for %s",
				fixityResult.GenericFile.Identifier)
		}
	}
	return true
}

func (fixityChecker *FixityChecker) logStats() {
	fixityChecker.ProcUtil.MessageLog.Info("**STATS** Succeeded: %d, Failed: %d",
		fixityChecker.ProcUtil.Succeeded(), fixityChecker.ProcUtil.Failed())
}

// Fetch the file and calculate its SHA256 digest. This may take hours if the
// file is 250GB. Touch the NSQ message on both sides of the operation to
// prevent it from timing out.
func (fixityChecker *FixityChecker) checkFile() {
	for result := range fixityChecker.FixityChannel {
		fixityChecker.ProcUtil.MessageLog.Info("Checking %s", result.GenericFile.Identifier)
		result.NsqMessage.Touch()
		err := fixityChecker.ProcUtil.S3Client.FetchAndCalculateSha256(result)
		// Log usage errors. These shouldn't happen.
		if err != nil && strings.Index(err.Error(), "cannot by nil") > 0 {
			fixityChecker.ProcUtil.MessageLog.Error(err.Error())
		}
		result.NsqMessage.Touch()
		fixityChecker.ResultsChannel <- result
	}
}
<|endoftext|>


<commit_before>package google

import (
	"bytes"
	"crypto"
	"crypto/rand"
	"crypto/rsa"
	"crypto/sha256"
	"crypto/x509"
	"encoding/base64"
	"encoding/pem"
	"errors"
	"fmt"
	"log"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"

	"sort"

	"github.com/hashicorp/errwrap"
	"github.com/hashicorp/terraform-plugin-sdk/helper/pathorcontents"
	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
	"github.com/hashicorp/terraform-plugin-sdk/helper/validation"
	"golang.org/x/oauth2/google"
	"golang.org/x/oauth2/jwt"
)

const gcsBaseUrl = "https://storage.googleapis.com"
const googleCredentialsEnvVar = "GOOGLE_APPLICATION_CREDENTIALS"

func dataSourceGoogleSignedUrl() *schema.Resource {
	return &schema.Resource{
		Read: dataSourceGoogleSignedUrlRead,

		Schema: map[string]*schema.Schema{
			"bucket": {
				Type:     schema.TypeString,
				Required: true,
			},
			"content_md5": {
				Type:     schema.TypeString,
				Optional: true,
				Default:  "",
			},
			"content_type": {
				Type:     schema.TypeString,
				Optional: true,
				Default:  "",
			},
			"credentials": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"duration": {
				Type:     schema.TypeString,
				Optional: true,
				Default:  "1h",
			},
			"extension_headers": {
				Type:         schema.TypeMap,
				Optional:     true,
				Elem:         &schema.Schema{Type: schema.TypeString},
				ValidateFunc: validateExtensionHeaders,
			},
			"http_method": {
				Type:         schema.TypeString,
				Optional:     true,
				Default:      "GET",
				ValidateFunc: validation.StringInSlice([]string{"GET", "HEAD", "PUT", "DELETE"}, true),
			},
			"path": {
				Type:     schema.TypeString,
				Required: true,
			},
			"signed_url": {
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func validateExtensionHeaders(v interface{}, k string) (ws []string, errors []error) {
	hdrMap := v.(map[string]interface{})
	for k := range hdrMap {
		if !strings.HasPrefix(strings.ToLower(k), "x-goog-") {
			errors = append(errors, fmt.Errorf(
				"extension_header (%s) not valid, header name must begin with 'x-goog-'", k))
		}
	}
	return
}

func dataSourceGoogleSignedUrlRead(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	// Build UrlData object from data source attributes
	urlData := &UrlData{}

	// HTTP Method
	if method, ok := d.GetOk("http_method"); ok {
		urlData.HttpMethod = method.(string)
	}

	// convert duration to an expiration datetime (unix time in seconds)
	durationString := "1h"
	if v, ok := d.GetOk("duration"); ok {
		durationString = v.(string)
	}
	duration, err := time.ParseDuration(durationString)
	if err != nil {
		return errwrap.Wrapf("could not parse duration: {{err}}", err)
	}
	expires := time.Now().Unix() + int64(duration.Seconds())
	urlData.Expires = int(expires)

	// content_md5 is optional
	if v, ok := d.GetOk("content_md5"); ok {
		urlData.ContentMd5 = v.(string)
	}

	// content_type is optional
	if v, ok := d.GetOk("content_type"); ok {
		urlData.ContentType = v.(string)
	}

	// extension_headers (x-goog-* HTTP headers) are optional
	if v, ok := d.GetOk("extension_headers"); ok {
		hdrMap := v.(map[string]interface{})

		if len(hdrMap) > 0 {
			urlData.HttpHeaders = make(map[string]string, len(hdrMap))
			for k, v := range hdrMap {
				urlData.HttpHeaders[k] = v.(string)
			}
		}
	}

	urlData.Path = fmt.Sprintf("/%s/%s", d.Get("bucket").(string), d.Get("path").(string))

	// Load JWT Config from Google Credentials
	jwtConfig, err := loadJwtConfig(d, config)
	if err != nil {
		return err
	}
	urlData.JwtConfig = jwtConfig

	// Construct URL
	signedUrl, err := urlData.SignedUrl()
	if err != nil {
		return err
	}

	// Success
	d.Set("signed_url", signedUrl)

	encodedSig, err := urlData.EncodedSignature()
	if err != nil {
		return err
	}
	d.SetId(encodedSig)

	return nil
}

// loadJwtConfig looks for credentials json in the following places,
// in order of preference:
//   1. `credentials` attribute of the datasource
//   2. `credentials` attribute in the provider definition.
//   3. A JSON file whose path is specified by the
//      GOOGLE_APPLICATION_CREDENTIALS environment variable.
func loadJwtConfig(d *schema.ResourceData, meta interface{}) (*jwt.Config, error) {
	config := meta.(*Config)

	credentials := ""
	if v, ok := d.GetOk("credentials"); ok {
		log.Println("[DEBUG] using data source credentials to sign URL")
		credentials = v.(string)

	} else if config.Credentials != "" {
		log.Println("[DEBUG] using provider credentials to sign URL")
		credentials = config.Credentials

	} else if filename := os.Getenv(googleCredentialsEnvVar); filename != "" {
		log.Println("[DEBUG] using env GOOGLE_APPLICATION_CREDENTIALS credentials to sign URL")
		credentials = filename

	}

	if strings.TrimSpace(credentials) != "" {
		contents, _, err := pathorcontents.Read(credentials)
		if err != nil {
			return nil, errwrap.Wrapf("Error loading credentials: {{err}}", err)
		}

		cfg, err := google.JWTConfigFromJSON([]byte(contents), "")
		if err != nil {
			return nil, errwrap.Wrapf("Error parsing credentials: {{err}}", err)
		}
		return cfg, nil
	}

	return nil, errors.New("Credentials not found in datasource, provider configuration or GOOGLE_APPLICATION_CREDENTIALS environment variable.")
}

// parsePrivateKey converts the binary contents of a private key file
// to an *rsa.PrivateKey. It detects whether the private key is in a
// PEM container or not. If so, it extracts the the private key
// from PEM container before conversion. It only supports PEM
// containers with no passphrase.
// copied from golang.org/x/oauth2/internal
func parsePrivateKey(key []byte) (*rsa.PrivateKey, error) {
	block, _ := pem.Decode(key)
	if block != nil {
		key = block.Bytes
	}
	parsedKey, err := x509.ParsePKCS8PrivateKey(key)
	if err != nil {
		parsedKey, err = x509.ParsePKCS1PrivateKey(key)
		if err != nil {
			return nil, errwrap.Wrapf("private key should be a PEM or plain PKSC1 or PKCS8; parse error: {{err}}", err)
		}
	}
	parsed, ok := parsedKey.(*rsa.PrivateKey)
	if !ok {
		return nil, errors.New("private key is invalid")
	}
	return parsed, nil
}

// UrlData stores the values required to create a Signed Url
type UrlData struct {
	JwtConfig   *jwt.Config
	ContentMd5  string
	ContentType string
	HttpMethod  string
	Expires     int
	HttpHeaders map[string]string
	Path        string
}

// SigningString creates a string representation of the UrlData in a form ready for signing:
// see https://cloud.google.com/storage/docs/access-control/create-signed-urls-program
// Example output:
// -------------------
// GET
//
//
// 1388534400
// bucket/objectname
// -------------------
func (u *UrlData) SigningString() []byte {
	var buf bytes.Buffer

	// HTTP Verb
	buf.WriteString(u.HttpMethod)
	buf.WriteString("\n")

	// Content MD5 (optional, always add new line)
	buf.WriteString(u.ContentMd5)
	buf.WriteString("\n")

	// Content Type (optional, always add new line)
	buf.WriteString(u.ContentType)
	buf.WriteString("\n")

	// Expiration
	buf.WriteString(strconv.Itoa(u.Expires))
	buf.WriteString("\n")

	// Extra HTTP headers (optional)
	// Must be sorted in lexigraphical order
	var keys []string
	for k := range u.HttpHeaders {
		keys = append(keys, strings.ToLower(k))
	}
	sort.Strings(keys)
	// Write sorted headers to signing string buffer
	for _, k := range keys {
		buf.WriteString(fmt.Sprintf("%s:%s\n", k, u.HttpHeaders[k]))
	}

	// Storage Object path (includes bucketname)
	buf.WriteString(u.Path)

	return buf.Bytes()
}

func (u *UrlData) Signature() ([]byte, error) {
	// Sign url data
	signature, err := SignString(u.SigningString(), u.JwtConfig)
	if err != nil {
		return nil, err

	}

	return signature, nil
}

// EncodedSignature returns the Signature() after base64 encoding and url escaping
func (u *UrlData) EncodedSignature() (string, error) {
	signature, err := u.Signature()
	if err != nil {
		return "", err
	}

	// base64 encode signature
	encoded := base64.StdEncoding.EncodeToString(signature)
	// encoded signature may include /, = characters that need escaping
	encoded = url.QueryEscape(encoded)

	return encoded, nil
}

// SignedUrl constructs the final signed URL a client can use to retrieve storage object
func (u *UrlData) SignedUrl() (string, error) {

	encodedSig, err := u.EncodedSignature()
	if err != nil {
		return "", err
	}

	// build url
	// https://cloud.google.com/storage/docs/access-control/create-signed-urls-program
	var urlBuffer bytes.Buffer
	urlBuffer.WriteString(gcsBaseUrl)
	urlBuffer.WriteString(u.Path)
	urlBuffer.WriteString("?GoogleAccessId=")
	urlBuffer.WriteString(u.JwtConfig.Email)
	urlBuffer.WriteString("&Expires=")
	urlBuffer.WriteString(strconv.Itoa(u.Expires))
	urlBuffer.WriteString("&Signature=")
	urlBuffer.WriteString(encodedSig)

	return urlBuffer.String(), nil
}

// SignString calculates the SHA256 signature of the input string
func SignString(toSign []byte, cfg *jwt.Config) ([]byte, error) {
	// Parse private key
	pk, err := parsePrivateKey(cfg.PrivateKey)
	if err != nil {
		return nil, errwrap.Wrapf("failed to sign string, could not parse key: {{err}}", err)
	}

	// Hash string
	hasher := sha256.New()
	if _, err := hasher.Write(toSign); err != nil {
		return nil, errwrap.Wrapf("failed to calculate sha256: {{err}}", err)
	}

	// Sign string
	signed, err := rsa.SignPKCS1v15(rand.Reader, pk, crypto.SHA256, hasher.Sum(nil))
	if err != nil {
		return nil, errwrap.Wrapf("failed to sign string, an error occurred: {{err}}", err)
	}

	return signed, nil
}
<commit_msg>Change credentials to sensitive (#3462)<commit_after>package google

import (
	"bytes"
	"crypto"
	"crypto/rand"
	"crypto/rsa"
	"crypto/sha256"
	"crypto/x509"
	"encoding/base64"
	"encoding/pem"
	"errors"
	"fmt"
	"log"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"

	"sort"

	"github.com/hashicorp/errwrap"
	"github.com/hashicorp/terraform-plugin-sdk/helper/pathorcontents"
	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
	"github.com/hashicorp/terraform-plugin-sdk/helper/validation"
	"golang.org/x/oauth2/google"
	"golang.org/x/oauth2/jwt"
)

const gcsBaseUrl = "https://storage.googleapis.com"
const googleCredentialsEnvVar = "GOOGLE_APPLICATION_CREDENTIALS"

func dataSourceGoogleSignedUrl() *schema.Resource {
	return &schema.Resource{
		Read: dataSourceGoogleSignedUrlRead,

		Schema: map[string]*schema.Schema{
			"bucket": {
				Type:     schema.TypeString,
				Required: true,
			},
			"content_md5": {
				Type:     schema.TypeString,
				Optional: true,
				Default:  "",
			},
			"content_type": {
				Type:     schema.TypeString,
				Optional: true,
				Default:  "",
			},
			"credentials": {
				Type:      schema.TypeString,
				Sensitive: true,
				Optional:  true,
			},
			"duration": {
				Type:     schema.TypeString,
				Optional: true,
				Default:  "1h",
			},
			"extension_headers": {
				Type:         schema.TypeMap,
				Optional:     true,
				Elem:         &schema.Schema{Type: schema.TypeString},
				ValidateFunc: validateExtensionHeaders,
			},
			"http_method": {
				Type:         schema.TypeString,
				Optional:     true,
				Default:      "GET",
				ValidateFunc: validation.StringInSlice([]string{"GET", "HEAD", "PUT", "DELETE"}, true),
			},
			"path": {
				Type:     schema.TypeString,
				Required: true,
			},
			"signed_url": {
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func validateExtensionHeaders(v interface{}, k string) (ws []string, errors []error) {
	hdrMap := v.(map[string]interface{})
	for k := range hdrMap {
		if !strings.HasPrefix(strings.ToLower(k), "x-goog-") {
			errors = append(errors, fmt.Errorf(
				"extension_header (%s) not valid, header name must begin with 'x-goog-'", k))
		}
	}
	return
}

func dataSourceGoogleSignedUrlRead(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	// Build UrlData object from data source attributes
	urlData := &UrlData{}

	// HTTP Method
	if method, ok := d.GetOk("http_method"); ok {
		urlData.HttpMethod = method.(string)
	}

	// convert duration to an expiration datetime (unix time in seconds)
	durationString := "1h"
	if v, ok := d.GetOk("duration"); ok {
		durationString = v.(string)
	}
	duration, err := time.ParseDuration(durationString)
	if err != nil {
		return errwrap.Wrapf("could not parse duration: {{err}}", err)
	}
	expires := time.Now().Unix() + int64(duration.Seconds())
	urlData.Expires = int(expires)

	// content_md5 is optional
	if v, ok := d.GetOk("content_md5"); ok {
		urlData.ContentMd5 = v.(string)
	}

	// content_type is optional
	if v, ok := d.GetOk("content_type"); ok {
		urlData.ContentType = v.(string)
	}

	// extension_headers (x-goog-* HTTP headers) are optional
	if v, ok := d.GetOk("extension_headers"); ok {
		hdrMap := v.(map[string]interface{})

		if len(hdrMap) > 0 {
			urlData.HttpHeaders = make(map[string]string, len(hdrMap))
			for k, v := range hdrMap {
				urlData.HttpHeaders[k] = v.(string)
			}
		}
	}

	urlData.Path = fmt.Sprintf("/%s/%s", d.Get("bucket").(string), d.Get("path").(string))

	// Load JWT Config from Google Credentials
	jwtConfig, err := loadJwtConfig(d, config)
	if err != nil {
		return err
	}
	urlData.JwtConfig = jwtConfig

	// Construct URL
	signedUrl, err := urlData.SignedUrl()
	if err != nil {
		return err
	}

	// Success
	d.Set("signed_url", signedUrl)

	encodedSig, err := urlData.EncodedSignature()
	if err != nil {
		return err
	}
	d.SetId(encodedSig)

	return nil
}

// loadJwtConfig looks for credentials json in the following places,
// in order of preference:
//   1. `credentials` attribute of the datasource
//   2. `credentials` attribute in the provider definition.
//   3. A JSON file whose path is specified by the
//      GOOGLE_APPLICATION_CREDENTIALS environment variable.
func loadJwtConfig(d *schema.ResourceData, meta interface{}) (*jwt.Config, error) {
	config := meta.(*Config)

	credentials := ""
	if v, ok := d.GetOk("credentials"); ok {
		log.Println("[DEBUG] using data source credentials to sign URL")
		credentials = v.(string)

	} else if config.Credentials != "" {
		log.Println("[DEBUG] using provider credentials to sign URL")
		credentials = config.Credentials

	} else if filename := os.Getenv(googleCredentialsEnvVar); filename != "" {
		log.Println("[DEBUG] using env GOOGLE_APPLICATION_CREDENTIALS credentials to sign URL")
		credentials = filename

	}

	if strings.TrimSpace(credentials) != "" {
		contents, _, err := pathorcontents.Read(credentials)
		if err != nil {
			return nil, errwrap.Wrapf("Error loading credentials: {{err}}", err)
		}

		cfg, err := google.JWTConfigFromJSON([]byte(contents), "")
		if err != nil {
			return nil, errwrap.Wrapf("Error parsing credentials: {{err}}", err)
		}
		return cfg, nil
	}

	return nil, errors.New("Credentials not found in datasource, provider configuration or GOOGLE_APPLICATION_CREDENTIALS environment variable.")
}

// parsePrivateKey converts the binary contents of a private key file
// to an *rsa.PrivateKey. It detects whether the private key is in a
// PEM container or not. If so, it extracts the the private key
// from PEM container before conversion. It only supports PEM
// containers with no passphrase.
// copied from golang.org/x/oauth2/internal
func parsePrivateKey(key []byte) (*rsa.PrivateKey, error) {
	block, _ := pem.Decode(key)
	if block != nil {
		key = block.Bytes
	}
	parsedKey, err := x509.ParsePKCS8PrivateKey(key)
	if err != nil {
		parsedKey, err = x509.ParsePKCS1PrivateKey(key)
		if err != nil {
			return nil, errwrap.Wrapf("private key should be a PEM or plain PKSC1 or PKCS8; parse error: {{err}}", err)
		}
	}
	parsed, ok := parsedKey.(*rsa.PrivateKey)
	if !ok {
		return nil, errors.New("private key is invalid")
	}
	return parsed, nil
}

// UrlData stores the values required to create a Signed Url
type UrlData struct {
	JwtConfig   *jwt.Config
	ContentMd5  string
	ContentType string
	HttpMethod  string
	Expires     int
	HttpHeaders map[string]string
	Path        string
}

// SigningString creates a string representation of the UrlData in a form ready for signing:
// see https://cloud.google.com/storage/docs/access-control/create-signed-urls-program
// Example output:
// -------------------
// GET
//
//
// 1388534400
// bucket/objectname
// -------------------
func (u *UrlData) SigningString() []byte {
	var buf bytes.Buffer

	// HTTP Verb
	buf.WriteString(u.HttpMethod)
	buf.WriteString("\n")

	// Content MD5 (optional, always add new line)
	buf.WriteString(u.ContentMd5)
	buf.WriteString("\n")

	// Content Type (optional, always add new line)
	buf.WriteString(u.ContentType)
	buf.WriteString("\n")

	// Expiration
	buf.WriteString(strconv.Itoa(u.Expires))
	buf.WriteString("\n")

	// Extra HTTP headers (optional)
	// Must be sorted in lexigraphical order
	var keys []string
	for k := range u.HttpHeaders {
		keys = append(keys, strings.ToLower(k))
	}
	sort.Strings(keys)
	// Write sorted headers to signing string buffer
	for _, k := range keys {
		buf.WriteString(fmt.Sprintf("%s:%s\n", k, u.HttpHeaders[k]))
	}

	// Storage Object path (includes bucketname)
	buf.WriteString(u.Path)

	return buf.Bytes()
}

func (u *UrlData) Signature() ([]byte, error) {
	// Sign url data
	signature, err := SignString(u.SigningString(), u.JwtConfig)
	if err != nil {
		return nil, err

	}

	return signature, nil
}

// EncodedSignature returns the Signature() after base64 encoding and url escaping
func (u *UrlData) EncodedSignature() (string, error) {
	signature, err := u.Signature()
	if err != nil {
		return "", err
	}

	// base64 encode signature
	encoded := base64.StdEncoding.EncodeToString(signature)
	// encoded signature may include /, = characters that need escaping
	encoded = url.QueryEscape(encoded)

	return encoded, nil
}

// SignedUrl constructs the final signed URL a client can use to retrieve storage object
func (u *UrlData) SignedUrl() (string, error) {

	encodedSig, err := u.EncodedSignature()
	if err != nil {
		return "", err
	}

	// build url
	// https://cloud.google.com/storage/docs/access-control/create-signed-urls-program
	var urlBuffer bytes.Buffer
	urlBuffer.WriteString(gcsBaseUrl)
	urlBuffer.WriteString(u.Path)
	urlBuffer.WriteString("?GoogleAccessId=")
	urlBuffer.WriteString(u.JwtConfig.Email)
	urlBuffer.WriteString("&Expires=")
	urlBuffer.WriteString(strconv.Itoa(u.Expires))
	urlBuffer.WriteString("&Signature=")
	urlBuffer.WriteString(encodedSig)

	return urlBuffer.String(), nil
}

// SignString calculates the SHA256 signature of the input string
func SignString(toSign []byte, cfg *jwt.Config) ([]byte, error) {
	// Parse private key
	pk, err := parsePrivateKey(cfg.PrivateKey)
	if err != nil {
		return nil, errwrap.Wrapf("failed to sign string, could not parse key: {{err}}", err)
	}

	// Hash string
	hasher := sha256.New()
	if _, err := hasher.Write(toSign); err != nil {
		return nil, errwrap.Wrapf("failed to calculate sha256: {{err}}", err)
	}

	// Sign string
	signed, err := rsa.SignPKCS1v15(rand.Reader, pk, crypto.SHA256, hasher.Sum(nil))
	if err != nil {
		return nil, errwrap.Wrapf("failed to sign string, an error occurred: {{err}}", err)
	}

	return signed, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package git

import (
	"path"
	"strings"

	"github.com/go-git/go-git/v5/plumbing/filemode"
	"github.com/go-git/go-git/v5/plumbing/object"
)

// GetTreeEntryByPath get the tree entries according the sub dir
func (t *Tree) GetTreeEntryByPath(relpath string) (*TreeEntry, error) {
	if len(relpath) == 0 {
		return &TreeEntry{
			ID: t.ID,
			//Type: ObjectTree,
			gogitTreeEntry: &object.TreeEntry{
				Name: "",
				Mode: filemode.Dir,
				Hash: t.ID,
			},
		}, nil
	}

	relpath = path.Clean(relpath)
	parts := strings.Split(relpath, "/")
	var err error
	tree := t
	for i, name := range parts {
		if i == len(parts)-1 {
			entries, err := tree.ListEntries()
			if err != nil {
				return nil, err
			}
			for _, v := range entries {
				if v.Name() == name {
					return v, nil
				}
			}
		} else {
			tree, err = tree.SubTree(name)
			if err != nil {
				return nil, err
			}
		}
	}
	return nil, ErrNotExist{"", relpath}
}

// GetBlobByPath get the blob object according the path
func (t *Tree) GetBlobByPath(relpath string) (*Blob, error) {
	entry, err := t.GetTreeEntryByPath(relpath)
	if err != nil {
		return nil, err
	}

	if !entry.IsDir() && !entry.IsSubModule() {
		return entry.Blob(), nil
	}

	return nil, ErrNotExist{"", relpath}
}
<commit_msg>Fix 500 on README in submodule (#13006)<commit_after>// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package git

import (
	"path"
	"strings"

	"github.com/go-git/go-git/v5/plumbing"
	"github.com/go-git/go-git/v5/plumbing/filemode"
	"github.com/go-git/go-git/v5/plumbing/object"
)

// GetTreeEntryByPath get the tree entries according the sub dir
func (t *Tree) GetTreeEntryByPath(relpath string) (*TreeEntry, error) {
	if len(relpath) == 0 {
		return &TreeEntry{
			ID: t.ID,
			//Type: ObjectTree,
			gogitTreeEntry: &object.TreeEntry{
				Name: "",
				Mode: filemode.Dir,
				Hash: t.ID,
			},
		}, nil
	}

	relpath = path.Clean(relpath)
	parts := strings.Split(relpath, "/")
	var err error
	tree := t
	for i, name := range parts {
		if i == len(parts)-1 {
			entries, err := tree.ListEntries()
			if err != nil {
				if err == plumbing.ErrObjectNotFound {
					return nil, ErrNotExist{
						RelPath: relpath,
					}
				}
				return nil, err
			}
			for _, v := range entries {
				if v.Name() == name {
					return v, nil
				}
			}
		} else {
			tree, err = tree.SubTree(name)
			if err != nil {
				if err == plumbing.ErrObjectNotFound {
					return nil, ErrNotExist{
						RelPath: relpath,
					}
				}
				return nil, err
			}
		}
	}
	return nil, ErrNotExist{"", relpath}
}

// GetBlobByPath get the blob object according the path
func (t *Tree) GetBlobByPath(relpath string) (*Blob, error) {
	entry, err := t.GetTreeEntryByPath(relpath)
	if err != nil {
		return nil, err
	}

	if !entry.IsDir() && !entry.IsSubModule() {
		return entry.Blob(), nil
	}

	return nil, ErrNotExist{"", relpath}
}
<|endoftext|>


<commit_before><commit_msg>Get full details<commit_after><|endoftext|>


<commit_before><commit_msg>use logfmt instead of text handler<commit_after><|endoftext|>


<commit_before>package bytepool

import (
  "io"
)

type Item struct {
  pool *Pool
  length int
  position int
  bytes []byte
}

func newItem(capacity int, pool *Pool) *Item {
  return &Item {
    pool: pool,
    bytes: make([]byte, capacity),
  }
}

func (item *Item) Write(b []byte) int {
  if item.Full() { return 0 }
  n := copy(item.bytes[item.length:], b)
  item.length += n
  return n
}

func (item *Item) WriteByte(b byte) bool {
  if item.Full() { return false }
  item.bytes[item.length] = b
  item.length += 1
  return true
}

func (item *Item) WriteString(s string) int {
  if item.Full() { return 0 }
  n := copy(item.bytes[item.length:], s)
  item.length += n
  return n
}

func (item *Item) ReadFrom(reader io.Reader) (int64, error) {
  var read int64
  for {
    r, err := reader.Read(item.bytes[item.length:])
    read += int64(r)
    item.length += r
    if err == io.EOF || item.Full() { return read, nil }
    if err != nil { return read, err }
  }
}

func (item *Item) Read(p []byte) (int, error) {
  if item.Drained() { return 0, io.EOF }
  n := copy(p, item.bytes[item.position:item.length])
  item.position += n
  if item.Drained() { return n, io.EOF }
  return n, nil
}

func (item *Item) Bytes() []byte {
  return item.bytes[0:item.length]
}

func (item *Item) String() string {
  return string(item.Bytes())
}

func (item *Item) Len() int {
  return item.length;
}

func (item *Item) Position(position int) {
  item.length = position
}

func (item *Item) Full() bool {
  return item.length == cap(item.bytes)
}

func (item *Item) Drained() bool {
  return item.length == item.position
}

func (item *Item) Close() error{
  item.length = 0
  item.position = 0
  if item.pool != nil {
    item.pool.list <- item
  }
  return nil
}
<commit_msg>renamed position to read for [hopeful] clarity<commit_after>package bytepool

import (
  "io"
)

type Item struct {
  pool *Pool
  length int
  read int
  bytes []byte
}

func newItem(capacity int, pool *Pool) *Item {
  return &Item {
    pool: pool,
    bytes: make([]byte, capacity),
  }
}

func (item *Item) Write(b []byte) int {
  if item.Full() { return 0 }
  n := copy(item.bytes[item.length:], b)
  item.length += n
  return n
}

func (item *Item) WriteByte(b byte) bool {
  if item.Full() { return false }
  item.bytes[item.length] = b
  item.length += 1
  return true
}

func (item *Item) WriteString(s string) int {
  if item.Full() { return 0 }
  n := copy(item.bytes[item.length:], s)
  item.length += n
  return n
}

func (item *Item) ReadFrom(reader io.Reader) (int64, error) {
  var read int64
  for {
    r, err := reader.Read(item.bytes[item.length:])
    read += int64(r)
    item.length += r
    if err == io.EOF || item.Full() { return read, nil }
    if err != nil { return read, err }
  }
}

func (item *Item) Read(p []byte) (int, error) {
  if item.Drained() { return 0, io.EOF }
  n := copy(p, item.bytes[item.read:item.length])
  item.read += n
  if item.Drained() { return n, io.EOF }
  return n, nil
}

func (item *Item) Bytes() []byte {
  return item.bytes[0:item.length]
}

func (item *Item) String() string {
  return string(item.Bytes())
}

func (item *Item) Len() int {
  return item.length;
}

func (item *Item) Position(position int) {
  item.length = position
}

func (item *Item) Full() bool {
  return item.length == cap(item.bytes)
}

func (item *Item) Drained() bool {
  return item.length == item.read
}

func (item *Item) Close() error{
  item.length = 0
  item.read = 0
  if item.pool != nil {
    item.pool.list <- item
  }
  return nil
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes";
	"flag";
	"fmt";
	"go/ast";
	"go/parser";
	"go/printer";
	"go/scanner";
	"io";
	"os";
	pathutil "path";
	"strings";
)

var (
	// main operation modes
	list		= flag.Bool("l", false, "list files whose formatting differs from gofmt's");
	write		= flag.Bool("w", false, "write result to (source) file instead of stdout");
	rewriteRule	= flag.String("r", "", "rewrite rule (e.g., 'α[β:len(α)] -> α[β:]')");

	// debugging support
	comments	= flag.Bool("comments", true, "print comments");
	trace		= flag.Bool("trace", false, "print parse trace");

	// layout control
	tabwidth	= flag.Int("tabwidth", 8, "tab width");
	usespaces	= flag.Bool("spaces", false, "align with spaces instead of tabs");
)

var exitCode = 0
var rewrite func(*ast.File) *ast.File

func report(err os.Error) {
	scanner.PrintError(os.Stderr, err);
	exitCode = 2;
}

func usage() {
	fmt.Fprintf(os.Stderr, "usage: gofmt [flags] [path ...]\n");
	flag.PrintDefaults();
	os.Exit(2);
}

func parserMode() uint {
	mode := uint(0);
	if *comments {
		mode |= parser.ParseComments
	}
	if *trace {
		mode |= parser.Trace
	}
	return mode;
}

func printerMode() uint {
	mode := uint(0);
	if *usespaces {
		mode |= printer.UseSpaces
	}
	return mode;
}

func isGoFile(d *os.Dir) bool {
	// ignore non-Go files
	return d.IsRegular() && !strings.HasPrefix(d.Name, ".") && strings.HasSuffix(d.Name, ".go")
}

func processFile(filename string) os.Error {
	src, err := io.ReadFile(filename);
	if err != nil {
		return err
	}

	file, err := parser.ParseFile(filename, src, parserMode());
	if err != nil {
		return err
	}

	if rewrite != nil {
		file = rewrite(file)
	}

	var res bytes.Buffer;
	_, err = (&printer.Config{printerMode(), *tabwidth, nil}).Fprint(&res, file);
	if err != nil {
		return err
	}

	if bytes.Compare(src, res.Bytes()) != 0 {
		// formatting has changed
		if *list {
			fmt.Fprintln(os.Stdout, filename)
		}
		if *write {
			err = io.WriteFile(filename, res.Bytes(), 0);
			if err != nil {
				return err
			}
		}
	}

	if !*list && !*write {
		_, err = os.Stdout.Write(res.Bytes())
	}

	return err;
}

type fileVisitor chan os.Error

func (v fileVisitor) VisitDir(path string, d *os.Dir) bool {
	return true
}

func (v fileVisitor) VisitFile(path string, d *os.Dir) {
	if isGoFile(d) {
		v <- nil;	// synchronize error handler
		if err := processFile(path); err != nil {
			v <- err
		}
	}
}

func walkDir(path string) {
	// start an error handler
	v := make(fileVisitor);
	go func() {
		for err := range v {
			if err != nil {
				report(err)
			}
		}
	}();
	// walk the tree
	pathutil.Walk(path, v, v);
	close(v);
}

func main() {
	flag.Usage = usage;
	flag.Parse();
	if *tabwidth < 0 {
		fmt.Fprintf(os.Stderr, "negative tabwidth %d\n", *tabwidth);
		os.Exit(2);
	}

	initRewrite();

	if flag.NArg() == 0 {
		if err := processFile("/dev/stdin"); err != nil {
			report(err)
		}
	}

	for i := 0; i < flag.NArg(); i++ {
		path := flag.Arg(i);
		switch dir, err := os.Stat(path); {
		case err != nil:
			report(err)
		case dir.IsRegular():
			if err := processFile(path); err != nil {
				report(err)
			}
		case dir.IsDirectory():
			walkDir(path)
		}
	}

	os.Exit(exitCode);
}
<commit_msg>gofmt: use os.Stdin instead of opening /dev/stdin<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes";
	"flag";
	"fmt";
	"go/ast";
	"go/parser";
	"go/printer";
	"go/scanner";
	"io";
	"os";
	pathutil "path";
	"strings";
)

var (
	// main operation modes
	list		= flag.Bool("l", false, "list files whose formatting differs from gofmt's");
	write		= flag.Bool("w", false, "write result to (source) file instead of stdout");
	rewriteRule	= flag.String("r", "", "rewrite rule (e.g., 'α[β:len(α)] -> α[β:]')");

	// debugging support
	comments	= flag.Bool("comments", true, "print comments");
	trace		= flag.Bool("trace", false, "print parse trace");

	// layout control
	tabwidth	= flag.Int("tabwidth", 8, "tab width");
	usespaces	= flag.Bool("spaces", false, "align with spaces instead of tabs");
)

var exitCode = 0
var rewrite func(*ast.File) *ast.File

func report(err os.Error) {
	scanner.PrintError(os.Stderr, err);
	exitCode = 2;
}

func usage() {
	fmt.Fprintf(os.Stderr, "usage: gofmt [flags] [path ...]\n");
	flag.PrintDefaults();
	os.Exit(2);
}

func parserMode() uint {
	mode := uint(0);
	if *comments {
		mode |= parser.ParseComments
	}
	if *trace {
		mode |= parser.Trace
	}
	return mode;
}

func printerMode() uint {
	mode := uint(0);
	if *usespaces {
		mode |= printer.UseSpaces
	}
	return mode;
}

func isGoFile(d *os.Dir) bool {
	// ignore non-Go files
	return d.IsRegular() && !strings.HasPrefix(d.Name, ".") && strings.HasSuffix(d.Name, ".go")
}

func processFile(f *os.File) os.Error {
	src, err := io.ReadAll(f);
	if err != nil {
		return err
	}

	file, err := parser.ParseFile(f.Name(), src, parserMode());
	if err != nil {
		return err
	}

	if rewrite != nil {
		file = rewrite(file)
	}

	var res bytes.Buffer;
	_, err = (&printer.Config{printerMode(), *tabwidth, nil}).Fprint(&res, file);
	if err != nil {
		return err
	}

	if bytes.Compare(src, res.Bytes()) != 0 {
		// formatting has changed
		if *list {
			fmt.Fprintln(os.Stdout, f.Name())
		}
		if *write {
			err = io.WriteFile(f.Name(), res.Bytes(), 0);
			if err != nil {
				return err
			}
		}
	}

	if !*list && !*write {
		_, err = os.Stdout.Write(res.Bytes())
	}

	return err;
}

func processFileByName(filename string) (err os.Error) {
	file, err := os.Open(filename, os.O_RDONLY, 0);
	if err != nil {
		return
	}
	defer file.Close();
	return processFile(file);
}

type fileVisitor chan os.Error

func (v fileVisitor) VisitDir(path string, d *os.Dir) bool {
	return true
}

func (v fileVisitor) VisitFile(path string, d *os.Dir) {
	if isGoFile(d) {
		v <- nil;	// synchronize error handler
		if err := processFileByName(path); err != nil {
			v <- err
		}
	}
}

func walkDir(path string) {
	// start an error handler
	v := make(fileVisitor);
	go func() {
		for err := range v {
			if err != nil {
				report(err)
			}
		}
	}();
	// walk the tree
	pathutil.Walk(path, v, v);
	close(v);
}

func main() {
	flag.Usage = usage;
	flag.Parse();
	if *tabwidth < 0 {
		fmt.Fprintf(os.Stderr, "negative tabwidth %d\n", *tabwidth);
		os.Exit(2);
	}

	initRewrite();

	if flag.NArg() == 0 {
		if err := processFile(os.Stdin); err != nil {
			report(err)
		}
	}

	for i := 0; i < flag.NArg(); i++ {
		path := flag.Arg(i);
		switch dir, err := os.Stat(path); {
		case err != nil:
			report(err)
		case dir.IsRegular():
			if err := processFileByName(path); err != nil {
				report(err)
			}
		case dir.IsDirectory():
			walkDir(path)
		}
	}

	os.Exit(exitCode);
}
<|endoftext|>


<commit_before><commit_msg>Cleanup<commit_after><|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"lib"
	"net"
	"os"
	"os/signal"
	"strings"
	"syscall"
)

var UNIX_SOCK_PATH = "/tmp/sudo-agent.sock"
var agent lib.DarwinConf

// udp message structure
type Message struct {
	Tag    string
	Method string
	Args   []string
}

func onInterrupt(conn *net.UnixConn) {
	conn.Close()
	os.Remove(UNIX_SOCK_PATH)
	os.Exit(0)
}

func onReceive(msg Message) {
	fmt.Printf("<- %+v\n", msg)

	tag := msg.Tag

	// tag verify
	if tag != os.Args[1] {
		fmt.Printf("unexpected verify tag: \"%v\"\n", tag)
		return
	}

	method := msg.Method
	args := msg.Args

	switch method {
	case "setGlobal":
		agent.SetGlobal(args[0])
	case "setPAC":
		agent.SetPAC(args[0])
	case "restoreGlobal":
		agent.RestoreGlobal(args[0])
	case "restorePAC":
		agent.RestorePAC(args[0])
	case "kill":
		agent.Kill()
	}
}

// $ ./sudo-agent <verify_tag>
func main() {
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)

	// get unix addr
	addr, err := net.ResolveUnixAddr("unixgram", UNIX_SOCK_PATH)
	if err != nil {
		fmt.Println(err)
		return
	}

	// listen unix socket
	conn, err := net.ListenUnixgram("unixgram", addr)
	if err != nil {
		fmt.Println(err)
		return
	}

	go func() {
		<-sigs // handle SIGINT and SIGTERM
		onInterrupt(conn)
	}()

	buffer := make([]byte, 512)
	for {
		// receive
		size, _, err := conn.ReadFrom(buffer)
		if err != nil {
			fmt.Println(err)
			continue
		}

		// to string
		str := string(buffer[0:size])
		fmt.Printf("%v bytes recieved: %v\n", size, str)

		// parse json
		var msg Message
		dec := json.NewDecoder(strings.NewReader(str))
		err1 := dec.Decode(&msg)
		if err1 != nil {
			fmt.Printf("cannot parse (\"%v\") due to: %v", str, err1)
			continue
		}
		onReceive(msg)
	}
}
<commit_msg>chore(thirdparty): use UDP socket instead of unix socket<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"lib"
	"net"
	"os"
	"os/signal"
	"strings"
	"syscall"
)

var agent lib.DarwinConf

// udp message structure
type Message struct {
	Tag    string
	Method string
	Args   []string
}

func onInterrupt(conn *net.UnixConn) {
	conn.Close()
	os.Exit(0)
}

func onReceive(msg Message) {
	fmt.Printf("<- %+v\n", msg)

	tag := msg.Tag

	// tag verify
	if tag != os.Args[1] {
		fmt.Printf("unexpected verify tag: \"%v\"\n", tag)
		return
	}

	method := msg.Method
	args := msg.Args

	switch method {
	case "setGlobal":
		agent.SetGlobal(args[0])
	case "setPAC":
		agent.SetPAC(args[0])
	case "restoreGlobal":
		agent.RestoreGlobal(args[0])
	case "restorePAC":
		agent.RestorePAC(args[0])
	case "kill":
		agent.Kill()
	}
}

// $ ./sudo-agent <verify_tag>
func main() {
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)

	// get unix addr
	addr, err := net.ResolveUDPAddr("udp4", "127.0.0.1:4758")
	if err != nil {
		fmt.Println(err)
		return
	}

	// listen unix socket
	conn, err := net.ListenUDP("udp4", addr)
	if err != nil {
		fmt.Println(err)
		return
	}

	go func() {
		<-sigs // handle SIGINT and SIGTERM
		onInterrupt(conn)
	}()

	buffer := make([]byte, 512)
	for {
		// receive
		size, _, err := conn.ReadFrom(buffer)
		if err != nil {
			fmt.Println(err)
			continue
		}

		// to string
		str := string(buffer[0:size])
		fmt.Printf("%v bytes recieved: %v\n", size, str)

		// parse json
		var msg Message
		dec := json.NewDecoder(strings.NewReader(str))
		err1 := dec.Decode(&msg)
		if err1 != nil {
			fmt.Printf("cannot parse (\"%v\") due to: %v", str, err1)
			continue
		}
		onReceive(msg)
	}
}
<|endoftext|>


<commit_before>package plugins

import (
	"github.com/Seklfreak/Robyul2/helpers"
	"github.com/bwmarrin/discordgo"
)

type About struct{}

func (a *About) Commands() []string {
	return []string{
		"about",
		"info",
	}
}

func (a *About) Init(session *discordgo.Session) {

}

func (a *About) Action(command string, content string, msg *discordgo.Message, session *discordgo.Session) {
	m := "**Hey! I'm Robyul.**\nI'm built using Go, open-source and a fork of Shiro, formerly called Karen, which you can find here: <https://github.com/SubliminalHQ/shiro>.\nYou can find out more about me here: <https://robyul.chat/>.\nSuggestions and discussions are always welcome on the Discord for me: <https://discord.gg/s5qZvUV>."

	helpers.SendMessage(msg.ChannelID, m)
}
<commit_msg>[about] improves about<commit_after>package plugins

import (
	"strings"

	"fmt"

	"github.com/Seklfreak/Robyul2/helpers"
	"github.com/bwmarrin/discordgo"
)

type About struct{}

func (a *About) Commands() []string {
	return []string{
		"about",
		"info",
	}
}

func (a *About) Init(session *discordgo.Session) {

}

func (a *About) Action(command string, content string, msg *discordgo.Message, session *discordgo.Session) {
	channel, err := helpers.GetChannel(msg.ChannelID)
	helpers.Relax(err)

	embed := &discordgo.MessageEmbed{
		URL:         "https://robyul.chat/",
		Title:       "Hello, I'm Robyul!",
		Description: "I'm built using Go, open-source and a fork of [Shiro](https://github.com/SubliminalHQ/shiro), formerly called Karen.",
		Color:       0,
		Author: &discordgo.MessageEmbedAuthor{
			URL:     "https://robyul.chat/",
			Name:    "Robyul - The KPop Discord Bot",
			IconURL: session.State.User.AvatarURL("64"),
		},
		Fields: []*discordgo.MessageEmbedField{
			{
				Name:   ":earth_asia: Website",
				Value:  "https://robyul.chat/",
				Inline: false,
			},
			{
				Name:   ":tools: Source",
				Value:  "https://github.com/Seklfreak/Robyul2",
				Inline: false,
			},
			{
				Name:   ":speech_left: Support Discord",
				Value:  "https://discord.gg/s5qZvUV",
				Inline: false,
			},
			{
				Name:   ":railway_track: Trello",
				Value:  "https://trello.robyul.chat/",
				Inline: false,
			},
			{
				Name:   ":vertical_traffic_light: Status",
				Value:  "https://status.robyul.chat/",
				Inline: false,
			},
			{
				Name:   ":cop: Team",
				Value:  strings.Replace(helpers.GetStaffUsernamesText(), "or", "and", -1),
				Inline: false,
			},
			{
				Name: ":bulb: Tip",
				Value: fmt.Sprintf("Make suggestions for Robyul: `%ssuggest <your suggestion>`!",
					helpers.GetPrefixForServer(channel.GuildID)),
				Inline: false,
			},
		},
	}

	helpers.SendEmbed(msg.ChannelID, embed)
}
<|endoftext|>


<commit_before>package pgproc

import (
	"database/sql"
	"errors"
	"fmt"
	"github.com/lib/pq"
	"reflect"
	"strings"
	"time"
)

type PgProc struct {
	db *sql.DB
}

type returnType struct {
	scalar         bool
	setof          bool
	scalarType     string
	compositeNames pq.StringArray
	compositeTypes pq.StringArray
}

var (
	DateMinusInfinity = time.Date(0, 1, 1, 0, 0, 0, 0, time.UTC)
	DateInfinity      = time.Date(9999, 1, 1, 0, 0, 0, 0, time.UTC)
)

// NewPgProc creates a new connection to a PostgreSQL database
func NewPgProc(conninfo string) (*PgProc, error) {
	var pgproc = PgProc{}
	var err error
	pgproc.db, err = sql.Open("postgres", conninfo)
	if err != nil {
		return nil, err
	}
	pq.EnableInfinityTs(DateMinusInfinity, DateInfinity)
	return &pgproc, nil
}

// Call calls a PostgreSQL procedure and stores the result
func (p *PgProc) Call(result interface{}, schema string, proc string, params ...interface{}) error {

	if proc[0] == '_' {
		return errors.New("function not callable")
	}

	rt, err := p.getReturnType(schema, proc, len(params))
	if err != nil {
		return err
	}
	query := fmt.Sprintf("SELECT * FROM %s.%s(%s)",
		pq.QuoteIdentifier(schema),
		pq.QuoteIdentifier(proc),
		paramsString(len(params)))

	if rt.scalar {
		if !rt.setof {
			row := p.db.QueryRow(query, params...)
			err = row.Scan(result)
		} else {
			rows, _ := p.db.Query(query, params...)
			defer rows.Close()
			c := reflect.ValueOf(result) // the channel we have to send to
			// val is a zero element of the same type of the channel type
			val := reflect.Zero(reflect.TypeOf(result).Elem()).Interface()
			for rows.Next() {
				if err := rows.Scan(&val); err != nil {
					return err
				}
				c.Send(reflect.ValueOf(val))
			}
		}
	} else {
		if !rt.setof {
			row := p.db.QueryRow(query, params...)
			err = ScanCompositeRow(row, rt, result)
		} else {
			rows, _ := p.db.Query(query, params...)
			defer rows.Close()
			for rows.Next() {
				if err := ScanCompositeRows(rows, rt, result); err != nil {
					return err
				}
			}
		}
	}
	return err
}

func ScanCompositeRow(row *sql.Row, rt *returnType, result interface{}) error {
	v := reflect.ValueOf(result).Elem()
	var vs []interface{}
	for _, name := range rt.compositeNames {
		f := v.FieldByName(strings.Title(name))
		if !f.IsValid() {
			fieldName, found := getFieldByTag(result, name)
			if !found {
				return errors.New("Error field " + name + " not found")
			}
			f = v.FieldByName(fieldName)
		}
		field := f.Addr().Interface()
		vs = append(vs, field)
	}

	err := row.Scan(vs...)
	return err
}

func ScanCompositeRows(rows *sql.Rows, rt *returnType, result interface{}) error {
	c := reflect.ValueOf(result) // the channel we have to send to
	v := reflect.New(reflect.TypeOf(result).Elem()).Elem()
	var vs []interface{}

	for _, name := range rt.compositeNames {
		field := v.FieldByName(strings.Title(name)).Addr().Interface()
		vs = append(vs, field)
	}

	err := rows.Scan(vs...)
	c.Send(v)
	return err
}

//
// Local static functions
//

// paramsString returns a string $1,$2,...,$len
func paramsString(len int) string {
	if len == 0 {
		return ""
	}
	result := "$1"
	for i := 2; i <= len; i++ {
		result += fmt.Sprintf(",$%d", i)
	}
	return result
}

// getReturnType gives the type returned by a postgreSQL procedure
func (p *PgProc) getReturnType(schema string, proc string, nargs int) (*returnType, error) {
	rt, err := p.getScalarReturnType(schema, proc, nargs)
	if err == sql.ErrNoRows {
		return p.getCompositeReturnType(schema, proc, nargs)
	} else {
		return rt, nil
	}
}

// getScalarReturnType gives the scalar type returned by a postgreSQL procedure
// or returns a ErrNoRows error if the return type is not scalar
func (p *PgProc) getScalarReturnType(schema string, proc string, nargs int) (*returnType, error) {
	query := `
SELECT
  pg_type_ret.typname,
  proretset
FROM pg_proc
INNER JOIN pg_type pg_type_ret ON pg_type_ret.oid = pg_proc.prorettype
INNER JOIN pg_namespace pg_namespace_ret ON pg_namespace_ret.oid = pg_type_ret.typnamespace
INNER JOIN pg_namespace pg_namespace_proc ON pg_namespace_proc.oid = pg_proc.pronamespace
WHERE
  pg_namespace_proc.nspname = $1 AND
  proname = $2 AND
  pronargs = $3 AND
  typtype IN ('b', 'p', 'e')`

	row := p.db.QueryRow(query, schema, proc, nargs)
	var (
		name  string
		setof bool
	)
	err := row.Scan(&name, &setof)
	if err == sql.ErrNoRows {
		return nil, err
	} else {
		return &returnType{scalar: true, setof: setof, scalarType: name}, nil
	}
}

// getCompositeReturnType gives the compiste type returned by a postgreSQL procedure
func (p *PgProc) getCompositeReturnType(schema string, proc string, nargs int) (*returnType, error) {
	query := `
SELECT
  (SELECT array_agg(attname ORDER BY attnum) FROM pg_attribute
   WHERE attrelid = pg_type_ret.typrelid AND attnum > 0),
  (SELECT array_agg(typname ORDER BY attnum) FROM pg_attribute
   INNER JOIN pg_type ON pg_attribute.atttypid = pg_type.oid
   WHERE attrelid = pg_type_ret.typrelid AND attnum > 0),
  proretset
FROM pg_proc
INNER JOIN pg_type pg_type_ret ON pg_type_ret.oid = pg_proc.prorettype
INNER JOIN pg_namespace pg_namespace_proc ON pg_namespace_proc.oid = pg_proc.pronamespace
WHERE
  pg_namespace_proc.nspname = $1 AND
  proname = $2 AND
  pronargs = $3 AND
  pg_type_ret.typtype IN ('c')`

	row := p.db.QueryRow(query, schema, proc, nargs)
	var (
		names pq.StringArray
		types pq.StringArray
		setof bool
	)
	err := row.Scan(&names, &types, &setof)
	if err == sql.ErrNoRows {
		return nil, sql.ErrNoRows
	} else {
		return &returnType{scalar: false, setof: setof, compositeNames: names, compositeTypes: types}, nil
	}

}

// TODO: Optimize with map
func getFieldByTag(v interface{}, tag string) (string, bool) {
	t := reflect.TypeOf(v).Elem()
	for i := 0; i < t.NumField(); i++ {
		f := t.Field(i)
		foundTag := f.Tag.Get("pgproc")
		if foundTag == tag {
			return f.Name, true
		}
	}
	return "", false
}
<commit_msg>call proc with void result<commit_after>package pgproc

import (
	"database/sql"
	"errors"
	"fmt"
	"github.com/lib/pq"
	"reflect"
	"strings"
	"time"
)

type PgProc struct {
	db *sql.DB
}

type returnType struct {
	scalar         bool
	setof          bool
	scalarType     string
	compositeNames pq.StringArray
	compositeTypes pq.StringArray
}

var (
	DateMinusInfinity = time.Date(0, 1, 1, 0, 0, 0, 0, time.UTC)
	DateInfinity      = time.Date(9999, 1, 1, 0, 0, 0, 0, time.UTC)
)

// NewPgProc creates a new connection to a PostgreSQL database
func NewPgProc(conninfo string) (*PgProc, error) {
	var pgproc = PgProc{}
	var err error
	pgproc.db, err = sql.Open("postgres", conninfo)
	if err != nil {
		return nil, err
	}
	pq.EnableInfinityTs(DateMinusInfinity, DateInfinity)
	return &pgproc, nil
}

// Call calls a PostgreSQL procedure and stores the result
func (p *PgProc) Call(result interface{}, schema string, proc string, params ...interface{}) error {

	if proc[0] == '_' {
		return errors.New("function not callable")
	}

	rt, err := p.getReturnType(schema, proc, len(params))
	if err != nil {
		return err
	}
	query := fmt.Sprintf("SELECT * FROM %s.%s(%s)",
		pq.QuoteIdentifier(schema),
		pq.QuoteIdentifier(proc),
		paramsString(len(params)))

	if rt.scalar {
		if !rt.setof {
			if result != nil {
				row := p.db.QueryRow(query, params...)
				err = row.Scan(result)
			} else {
				_, err = p.db.Exec(query, params...)
			}
		} else {
			rows, _ := p.db.Query(query, params...)
			defer rows.Close()
			c := reflect.ValueOf(result) // the channel we have to send to
			// val is a zero element of the same type of the channel type
			val := reflect.Zero(reflect.TypeOf(result).Elem()).Interface()
			for rows.Next() {
				if err := rows.Scan(&val); err != nil {
					return err
				}
				c.Send(reflect.ValueOf(val))
			}
		}
	} else {
		if !rt.setof {
			row := p.db.QueryRow(query, params...)
			err = ScanCompositeRow(row, rt, result)
		} else {
			rows, _ := p.db.Query(query, params...)
			defer rows.Close()
			for rows.Next() {
				if err := ScanCompositeRows(rows, rt, result); err != nil {
					return err
				}
			}
		}
	}
	return err
}

func ScanCompositeRow(row *sql.Row, rt *returnType, result interface{}) error {
	v := reflect.ValueOf(result).Elem()
	var vs []interface{}
	for _, name := range rt.compositeNames {
		f := v.FieldByName(strings.Title(name))
		if !f.IsValid() {
			fieldName, found := getFieldByTag(result, name)
			if !found {
				return errors.New("Error field " + name + " not found")
			}
			f = v.FieldByName(fieldName)
		}
		field := f.Addr().Interface()
		vs = append(vs, field)
	}

	err := row.Scan(vs...)
	return err
}

func ScanCompositeRows(rows *sql.Rows, rt *returnType, result interface{}) error {
	c := reflect.ValueOf(result) // the channel we have to send to
	v := reflect.New(reflect.TypeOf(result).Elem()).Elem()
	var vs []interface{}

	for _, name := range rt.compositeNames {
		field := v.FieldByName(strings.Title(name)).Addr().Interface()
		vs = append(vs, field)
	}

	err := rows.Scan(vs...)
	c.Send(v)
	return err
}

//
// Local static functions
//

// paramsString returns a string $1,$2,...,$len
func paramsString(len int) string {
	if len == 0 {
		return ""
	}
	result := "$1"
	for i := 2; i <= len; i++ {
		result += fmt.Sprintf(",$%d", i)
	}
	return result
}

// getReturnType gives the type returned by a postgreSQL procedure
func (p *PgProc) getReturnType(schema string, proc string, nargs int) (*returnType, error) {
	rt, err := p.getScalarReturnType(schema, proc, nargs)
	if err == sql.ErrNoRows {
		return p.getCompositeReturnType(schema, proc, nargs)
	} else {
		return rt, nil
	}
}

// getScalarReturnType gives the scalar type returned by a postgreSQL procedure
// or returns a ErrNoRows error if the return type is not scalar
func (p *PgProc) getScalarReturnType(schema string, proc string, nargs int) (*returnType, error) {
	query := `
SELECT
  pg_type_ret.typname,
  proretset
FROM pg_proc
INNER JOIN pg_type pg_type_ret ON pg_type_ret.oid = pg_proc.prorettype
INNER JOIN pg_namespace pg_namespace_ret ON pg_namespace_ret.oid = pg_type_ret.typnamespace
INNER JOIN pg_namespace pg_namespace_proc ON pg_namespace_proc.oid = pg_proc.pronamespace
WHERE
  pg_namespace_proc.nspname = $1 AND
  proname = $2 AND
  pronargs = $3 AND
  typtype IN ('b', 'p', 'e')`

	row := p.db.QueryRow(query, schema, proc, nargs)
	var (
		name  string
		setof bool
	)
	err := row.Scan(&name, &setof)
	if err == sql.ErrNoRows {
		return nil, err
	} else {
		return &returnType{scalar: true, setof: setof, scalarType: name}, nil
	}
}

// getCompositeReturnType gives the compiste type returned by a postgreSQL procedure
func (p *PgProc) getCompositeReturnType(schema string, proc string, nargs int) (*returnType, error) {
	query := `
SELECT
  (SELECT array_agg(attname ORDER BY attnum) FROM pg_attribute
   WHERE attrelid = pg_type_ret.typrelid AND attnum > 0),
  (SELECT array_agg(typname ORDER BY attnum) FROM pg_attribute
   INNER JOIN pg_type ON pg_attribute.atttypid = pg_type.oid
   WHERE attrelid = pg_type_ret.typrelid AND attnum > 0),
  proretset
FROM pg_proc
INNER JOIN pg_type pg_type_ret ON pg_type_ret.oid = pg_proc.prorettype
INNER JOIN pg_namespace pg_namespace_proc ON pg_namespace_proc.oid = pg_proc.pronamespace
WHERE
  pg_namespace_proc.nspname = $1 AND
  proname = $2 AND
  pronargs = $3 AND
  pg_type_ret.typtype IN ('c')`

	row := p.db.QueryRow(query, schema, proc, nargs)
	var (
		names pq.StringArray
		types pq.StringArray
		setof bool
	)
	err := row.Scan(&names, &types, &setof)
	if err == sql.ErrNoRows {
		return nil, sql.ErrNoRows
	} else {
		return &returnType{scalar: false, setof: setof, compositeNames: names, compositeTypes: types}, nil
	}

}

// TODO: Optimize with map
func getFieldByTag(v interface{}, tag string) (string, bool) {
	t := reflect.TypeOf(v).Elem()
	for i := 0; i < t.NumField(); i++ {
		f := t.Field(i)
		foundTag := f.Tag.Get("pgproc")
		if foundTag == tag {
			return f.Name, true
		}
	}
	return "", false
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package pkiutil

import (
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"io/ioutil"
	"os"
	"testing"

	certutil "k8s.io/client-go/util/cert"
)

func TestNewCertificateAuthority(t *testing.T) {
	cert, key, err := NewCertificateAuthority()

	if cert == nil {
		t.Errorf(
			"failed NewCertificateAuthority, cert == nil",
		)
	}
	if key == nil {
		t.Errorf(
			"failed NewCertificateAuthority, key == nil",
		)
	}
	if err != nil {
		t.Errorf(
			"failed NewCertificateAuthority with an error: %v",
			err,
		)
	}
}

func TestNewCertAndKey(t *testing.T) {
	var tests = []struct {
		caKeySize int
		expected  bool
	}{
		{
			// RSA key too small
			caKeySize: 128,
			expected:  false,
		},
		{
			// Should succeed
			caKeySize: 2048,
			expected:  true,
		},
	}

	for _, rt := range tests {
		caKey, err := rsa.GenerateKey(rand.Reader, rt.caKeySize)
		if err != nil {
			t.Fatalf("Couldn't create rsa Private Key")
		}
		caCert := &x509.Certificate{}
		config := certutil.Config{
			CommonName:   "test",
			Organization: []string{"test"},
			Usages:       []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
		}
		_, _, actual := NewCertAndKey(caCert, caKey, config)
		if (actual == nil) != rt.expected {
			t.Errorf(
				"failed NewCertAndKey:\n\texpected: %t\n\t  actual: %t",
				rt.expected,
				(actual == nil),
			)
		}
	}
}

func TestWriteCertAndKey(t *testing.T) {
	tmpdir, err := ioutil.TempDir("", "")
	if err != nil {
		t.Fatalf("Couldn't create tmpdir")
	}
	defer os.Remove(tmpdir)

	caKey, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		t.Fatalf("Couldn't create rsa Private Key")
	}
	caCert := &x509.Certificate{}
	actual := WriteCertAndKey(tmpdir, "foo", caCert, caKey)
	if actual != nil {
		t.Errorf(
			"failed WriteCertAndKey with an error: %v",
			actual,
		)
	}
}
<commit_msg>kubeadm: added tests cert/pkiutil pkg<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package pkiutil

import (
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"io/ioutil"
	"os"
	"testing"

	certutil "k8s.io/client-go/util/cert"
)

func TestNewCertificateAuthority(t *testing.T) {
	cert, key, err := NewCertificateAuthority()

	if cert == nil {
		t.Errorf(
			"failed NewCertificateAuthority, cert == nil",
		)
	}
	if key == nil {
		t.Errorf(
			"failed NewCertificateAuthority, key == nil",
		)
	}
	if err != nil {
		t.Errorf(
			"failed NewCertificateAuthority with an error: %v",
			err,
		)
	}
}

func TestNewCertAndKey(t *testing.T) {
	var tests = []struct {
		caKeySize int
		expected  bool
	}{
		{
			// RSA key too small
			caKeySize: 128,
			expected:  false,
		},
		{
			// Should succeed
			caKeySize: 2048,
			expected:  true,
		},
	}

	for _, rt := range tests {
		caKey, err := rsa.GenerateKey(rand.Reader, rt.caKeySize)
		if err != nil {
			t.Fatalf("Couldn't create rsa Private Key")
		}
		caCert := &x509.Certificate{}
		config := certutil.Config{
			CommonName:   "test",
			Organization: []string{"test"},
			Usages:       []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
		}
		_, _, actual := NewCertAndKey(caCert, caKey, config)
		if (actual == nil) != rt.expected {
			t.Errorf(
				"failed NewCertAndKey:\n\texpected: %t\n\t  actual: %t",
				rt.expected,
				(actual == nil),
			)
		}
	}
}

func TestWriteCertAndKey(t *testing.T) {
	tmpdir, err := ioutil.TempDir("", "")
	if err != nil {
		t.Fatalf("Couldn't create tmpdir")
	}
	defer os.RemoveAll(tmpdir)

	caKey, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		t.Fatalf("Couldn't create rsa Private Key")
	}
	caCert := &x509.Certificate{}
	actual := WriteCertAndKey(tmpdir, "foo", caCert, caKey)
	if actual != nil {
		t.Errorf(
			"failed WriteCertAndKey with an error: %v",
			actual,
		)
	}
}

func TestCertOrKeyExist(t *testing.T) {
	tmpdir, err := ioutil.TempDir("", "")
	if err != nil {
		t.Fatalf("Couldn't create tmpdir")
	}
	defer os.RemoveAll(tmpdir)

	caKey, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		t.Fatalf("Couldn't create rsa Private Key")
	}
	caCert := &x509.Certificate{}
	actual := WriteCertAndKey(tmpdir, "foo", caCert, caKey)
	if actual != nil {
		t.Errorf(
			"failed WriteCertAndKey with an error: %v",
			actual,
		)
	}

	var tests = []struct {
		path     string
		name     string
		expected bool
	}{
		{
			path:     "",
			name:     "",
			expected: false,
		},
		{
			path:     tmpdir,
			name:     "foo",
			expected: true,
		},
	}
	for _, rt := range tests {
		actual := CertOrKeyExist(rt.path, rt.name)
		if actual != rt.expected {
			t.Errorf(
				"failed CertOrKeyExist:\n\texpected: %t\n\t  actual: %t",
				rt.expected,
				actual,
			)
		}
	}
}

func TestTryLoadCertAndKeyFromDisk(t *testing.T) {
	tmpdir, err := ioutil.TempDir("", "")
	if err != nil {
		t.Fatalf("Couldn't create tmpdir")
	}
	defer os.RemoveAll(tmpdir)

	caCert, caKey, err := NewCertificateAuthority()
	if err != nil {
		t.Errorf(
			"failed to create cert and key with an error: %v",
			err,
		)
	}
	err = WriteCertAndKey(tmpdir, "foo", caCert, caKey)
	if err != nil {
		t.Errorf(
			"failed to write cert and key with an error: %v",
			err,
		)
	}

	var tests = []struct {
		path     string
		name     string
		expected bool
	}{
		{
			path:     "",
			name:     "",
			expected: false,
		},
		{
			path:     tmpdir,
			name:     "foo",
			expected: true,
		},
	}
	for _, rt := range tests {
		_, _, actual := TryLoadCertAndKeyFromDisk(rt.path, rt.name)
		if (actual == nil) != rt.expected {
			t.Errorf(
				"failed TryLoadCertAndKeyFromDisk:\n\texpected: %t\n\t  actual: %t",
				rt.expected,
				(actual == nil),
			)
		}
	}
}
<|endoftext|>


<commit_before>package renter

import (
	"io"
	"os"
	"sync"
	"time"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/modules/renter/hostdb"
	"github.com/NebulousLabs/Sia/types"
)

const (
	// repairThreads is the number of repairs that can run concurrently.
	repairThreads = 10
)

// When a file contract is within 'renewThreshold' blocks of expiring, the renter
// will attempt to renew the contract.
var renewThreshold = func() types.BlockHeight {
	switch build.Release {
	case "testing":
		return 10
	case "dev":
		return 200
	default:
		return 2000
	}
}()

// repair attempts to repair a file chunk by uploading its pieces to more
// hosts.
func (f *file) repair(chunkIndex uint64, missingPieces []uint64, r io.ReaderAt, hosts []hostdb.Uploader) error {
	// read chunk data and encode
	chunk := make([]byte, f.chunkSize())
	_, err := r.ReadAt(chunk, int64(chunkIndex*f.chunkSize()))
	if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
		return err
	}
	pieces, err := f.erasureCode.Encode(chunk)
	if err != nil {
		return err
	}
	// encrypt pieces
	for i := range pieces {
		key := deriveKey(f.masterKey, chunkIndex, uint64(i))
		pieces[i], err = key.EncryptBytes(pieces[i])
		if err != nil {
			return err
		}
	}

	// upload one piece per host
	numPieces := len(missingPieces)
	if len(hosts) < numPieces {
		numPieces = len(hosts)
	}
	var wg sync.WaitGroup
	wg.Add(numPieces)
	for i := 0; i < numPieces; i++ {
		// each goroutine gets a different host, index, and piece, so there
		// are no data race concerns
		pIndex := missingPieces[i]
		go func(host hostdb.Uploader, pieceIndex uint64, piece []byte) {
			defer wg.Done()

			// upload data to host
			offset, err := host.Upload(piece)
			if err != nil {
				return
			}

			// create contract entry, if necessary
			f.mu.Lock()
			defer f.mu.Unlock()
			contract, ok := f.contracts[host.ContractID()]
			if !ok {
				contract = fileContract{
					ID:          host.ContractID(),
					IP:          host.Address(),
					WindowStart: host.EndHeight(),
				}
			}

			// update contract
			contract.Pieces = append(contract.Pieces, pieceData{
				Chunk:  chunkIndex,
				Piece:  pieceIndex,
				Offset: offset,
			})
			f.contracts[host.ContractID()] = contract
		}(hosts[i], pIndex, pieces[pIndex])
	}
	wg.Wait()

	return nil
}

// incompleteChunks returns a map of chunks containing pieces that have not
// been uploaded.
func (f *file) incompleteChunks() map[uint64][]uint64 {
	f.mu.RLock()
	defer f.mu.RUnlock()

	present := make([][]bool, f.numChunks())
	for i := range present {
		present[i] = make([]bool, f.erasureCode.NumPieces())
	}
	for _, fc := range f.contracts {
		for _, p := range fc.Pieces {
			present[p.Chunk][p.Piece] = true
		}
	}

	incomplete := make(map[uint64][]uint64)
	for chunkIndex, pieceBools := range present {
		for pieceIndex, ok := range pieceBools {
			if !ok {
				incomplete[uint64(chunkIndex)] = append(incomplete[uint64(chunkIndex)], uint64(pieceIndex))
			}
		}
	}
	return incomplete
}

// chunkHosts returns the hosts storing the given chunk.
func (f *file) chunkHosts(chunk uint64) []modules.NetAddress {
	f.mu.RLock()
	defer f.mu.RUnlock()

	var old []modules.NetAddress
	for _, fc := range f.contracts {
		for _, p := range fc.Pieces {
			if p.Chunk == chunk {
				old = append(old, fc.IP)
				break
			}
		}
	}
	return old
}

// expiringContracts returns the contracts that will expire soon.
// TODO: what if contract has fully expired?
func (f *file) expiringContracts(height types.BlockHeight) []fileContract {
	f.mu.RLock()
	defer f.mu.RUnlock()

	var expiring []fileContract
	for _, fc := range f.contracts {
		if height >= fc.WindowStart-renewThreshold {
			expiring = append(expiring, fc)
		}
	}
	return expiring
}

// offlineChunks returns the chunks belonging to "offline" hosts -- hosts that
// do not meet uptime requirements. Importantly, only chunks missing more than
// half their redundancy are returned.
func (f *file) offlineChunks(hdb hostDB) map[uint64][]uint64 {
	f.mu.RLock()
	defer f.mu.RUnlock()

	// helper function for determining if a host is offline. A host is
	// considered offline if it is in AllHosts but not ActiveHosts. For good
	// order notation, we covert AllHosts and ActiveHosts to a single map. The
	// map lookup will return true if the host was in ActiveHosts.
	//
	// TODO: once the hostdb is persistent, checking for existence in
	// ActiveHosts should be sufficient.
	hostSet := make(map[modules.NetAddress]bool)
	for _, host := range hdb.AllHosts() {
		hostSet[host.NetAddress] = false
	}
	for _, host := range hdb.ActiveHosts() {
		hostSet[host.NetAddress] = true
	}
	isOffline := func(addr modules.NetAddress) bool {
		active, exists := hostSet[addr]
		return exists && !active
	}

	// mark all pieces belonging to offline hosts.
	offline := make(map[uint64][]uint64)
	for _, fc := range f.contracts {
		if isOffline(fc.IP) {
			for _, p := range fc.Pieces {
				offline[p.Chunk] = append(offline[p.Chunk], p.Piece)
			}
		}
	}
	// filter out chunks missing less than half of their redundancy
	filtered := make(map[uint64][]uint64)
	for chunk, pieces := range offline {
		if len(pieces) > f.erasureCode.NumPieces()/2 {
			filtered[chunk] = pieces
		}
	}
	return filtered
}

// threadedRepairLoop improves the health of files tracked by the renter by
// reuploading their missing pieces. Multiple repair attempts may be necessary
// before the file reaches full redundancy.
func (r *Renter) threadedRepairLoop() {
	// Files are repaired concurrently. A repair worker must acquire a 'token'
	// in order to run, and returns the token to the pool when it has
	// finished.
	tokenPool := make(chan struct{}, repairThreads)
	// fill the tokenPool with tokens
	for i := 0; i < repairThreads; i++ {
		tokenPool <- struct{}{}
	}

	for {
		time.Sleep(5 * time.Second)

		if !r.wallet.Unlocked() {
			continue
		}

		// make copy of repair set under lock
		repairing := make(map[string]trackedFile)
		id := r.mu.RLock()
		for name, meta := range r.tracking {
			repairing[name] = meta
		}
		r.mu.RUnlock(id)

		var wg sync.WaitGroup
		wg.Add(len(repairing))
		for name, meta := range repairing {
			go func(name string, meta trackedFile) {
				defer wg.Done()
				t := <-tokenPool                 // acquire token
				r.threadedRepairFile(name, meta) // repair
				tokenPool <- t                   // return token
			}(name, meta)
		}
		// wait for all repairs to complete before looping; otherwise we risk
		// spawning multiple repair threads for the same file.
		wg.Wait()
	}
}

// threadedRepairFile repairs and saves an individual file.
func (r *Renter) threadedRepairFile(name string, meta trackedFile) {
	// helper function
	logAndRemove := func(fmt string, args ...interface{}) {
		r.log.Printf(fmt, args...)
		id := r.mu.Lock()
		delete(r.tracking, name)
		r.mu.Unlock(id)
	}

	id := r.mu.RLock()
	f, ok := r.files[name]
	r.mu.RUnlock(id)
	if !ok {
		logAndRemove("removing %v from repair set: no longer tracking that file", name)
		return
	}

	// check for expiration
	height := r.cs.Height()
	if !meta.Renew && meta.EndHeight < height {
		logAndRemove("removing %v from repair set: storage period has ended", name)
		return
	}

	// determine if there is any work to do
	incChunks := f.incompleteChunks()
	offlineChunks := f.offlineChunks(r.hostDB)
	expContracts := f.expiringContracts(height)
	if len(incChunks) == 0 && len(offlineChunks) == 0 && (!meta.Renew || len(expContracts) == 0) {
		return
	}

	// open file handle
	handle, err := os.Open(meta.RepairPath)
	if err != nil {
		logAndRemove("removing %v from repair set: %v", name, err)
		return
	}
	defer handle.Close()

	// repair incomplete chunks
	if len(incChunks) != 0 {
		r.log.Printf("repairing %v chunks of %v", len(incChunks), f.name)
		var duration types.BlockHeight
		if meta.Renew {
			duration = defaultDuration
		} else {
			duration = meta.EndHeight - height
		}
		r.repairChunks(f, handle, incChunks, duration)
	}

	// repair offline chunks
	if len(offlineChunks) != 0 {
		r.log.Printf("reuploading %v offline chunks of %v", len(offlineChunks), f.name)
		var duration types.BlockHeight
		if meta.Renew {
			duration = defaultDuration
		} else {
			duration = meta.EndHeight - height
		}
		r.repairChunks(f, handle, offlineChunks, duration)
	}

	// renew expiring contracts
	if meta.Renew && len(expContracts) != 0 {
		r.log.Printf("renewing %v contracts of %v", len(expContracts), f.name)
		newHeight := height + defaultDuration
		r.renewContracts(f, expContracts, newHeight)
	}

	// save the repaired file data
	f.mu.RLock()
	err = r.saveFile(f)
	f.mu.RUnlock()
	if err != nil {
		// definitely bad, but we probably shouldn't delete from the
		// repair set if this happens
		r.log.Printf("failed to save repaired file %v: %v", name, err)
	}
}

// repairChunks uploads missing chunks of f to new hosts.
func (r *Renter) repairChunks(f *file, handle io.ReaderAt, chunks map[uint64][]uint64, duration types.BlockHeight) {
	// create host pool
	contractSize := (f.pieceSize + crypto.TwofishOverhead) * f.numChunks() // each host gets one piece of each chunk
	pool, err := r.hostDB.NewPool(contractSize, duration)
	if err != nil {
		r.log.Printf("failed to repair %v: %v", f.name, err)
		return
	}
	defer pool.Close() // heh

	for chunk, pieces := range chunks {
		// Determine host set. We want one host for each missing piece, and no
		// repeats of other hosts of this chunk.
		hosts := pool.UniqueHosts(len(pieces), f.chunkHosts(chunk))
		if len(hosts) == 0 {
			r.log.Printf("aborting repair of %v: not enough hosts", f.name)
			return
		}
		// upload to new hosts
		err = f.repair(chunk, pieces, handle, hosts)
		if err != nil {
			r.log.Printf("aborting repair of %v: %v", f.name, err)
			return
		}
	}
}

// renewContracts renews each of the supplied contracts, replacing their entry
// in f with the new contract.
func (r *Renter) renewContracts(f *file, contracts []fileContract, newHeight types.BlockHeight) {
	for _, c := range contracts {
		newID, err := r.hostDB.Renew(c.ID, newHeight)
		if err != nil {
			r.log.Printf("failed to renew contract %v: %v", c.ID, err)
			continue
		}
		f.mu.Lock()
		f.contracts[newID] = fileContract{
			ID:          newID,
			IP:          c.IP,
			Pieces:      c.Pieces,
			WindowStart: newHeight,
		}
		// need to delete the old contract; otherwise f.expiringContracts
		// will continue to return it
		delete(f.contracts, c.ID)
		f.mu.Unlock()
	}
}
<commit_msg>create pool with correct contractSize<commit_after>package renter

import (
	"io"
	"os"
	"sync"
	"time"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/modules/renter/hostdb"
	"github.com/NebulousLabs/Sia/types"
)

const (
	// repairThreads is the number of repairs that can run concurrently.
	repairThreads = 10
)

// When a file contract is within 'renewThreshold' blocks of expiring, the renter
// will attempt to renew the contract.
var renewThreshold = func() types.BlockHeight {
	switch build.Release {
	case "testing":
		return 10
	case "dev":
		return 200
	default:
		return 2000
	}
}()

// repair attempts to repair a file chunk by uploading its pieces to more
// hosts.
func (f *file) repair(chunkIndex uint64, missingPieces []uint64, r io.ReaderAt, hosts []hostdb.Uploader) error {
	// read chunk data and encode
	chunk := make([]byte, f.chunkSize())
	_, err := r.ReadAt(chunk, int64(chunkIndex*f.chunkSize()))
	if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
		return err
	}
	pieces, err := f.erasureCode.Encode(chunk)
	if err != nil {
		return err
	}
	// encrypt pieces
	for i := range pieces {
		key := deriveKey(f.masterKey, chunkIndex, uint64(i))
		pieces[i], err = key.EncryptBytes(pieces[i])
		if err != nil {
			return err
		}
	}

	// upload one piece per host
	numPieces := len(missingPieces)
	if len(hosts) < numPieces {
		numPieces = len(hosts)
	}
	var wg sync.WaitGroup
	wg.Add(numPieces)
	for i := 0; i < numPieces; i++ {
		// each goroutine gets a different host, index, and piece, so there
		// are no data race concerns
		pIndex := missingPieces[i]
		go func(host hostdb.Uploader, pieceIndex uint64, piece []byte) {
			defer wg.Done()

			// upload data to host
			offset, err := host.Upload(piece)
			if err != nil {
				return
			}

			// create contract entry, if necessary
			f.mu.Lock()
			defer f.mu.Unlock()
			contract, ok := f.contracts[host.ContractID()]
			if !ok {
				contract = fileContract{
					ID:          host.ContractID(),
					IP:          host.Address(),
					WindowStart: host.EndHeight(),
				}
			}

			// update contract
			contract.Pieces = append(contract.Pieces, pieceData{
				Chunk:  chunkIndex,
				Piece:  pieceIndex,
				Offset: offset,
			})
			f.contracts[host.ContractID()] = contract
		}(hosts[i], pIndex, pieces[pIndex])
	}
	wg.Wait()

	return nil
}

// incompleteChunks returns a map of chunks containing pieces that have not
// been uploaded.
func (f *file) incompleteChunks() map[uint64][]uint64 {
	f.mu.RLock()
	defer f.mu.RUnlock()

	present := make([][]bool, f.numChunks())
	for i := range present {
		present[i] = make([]bool, f.erasureCode.NumPieces())
	}
	for _, fc := range f.contracts {
		for _, p := range fc.Pieces {
			present[p.Chunk][p.Piece] = true
		}
	}

	incomplete := make(map[uint64][]uint64)
	for chunkIndex, pieceBools := range present {
		for pieceIndex, ok := range pieceBools {
			if !ok {
				incomplete[uint64(chunkIndex)] = append(incomplete[uint64(chunkIndex)], uint64(pieceIndex))
			}
		}
	}
	return incomplete
}

// chunkHosts returns the hosts storing the given chunk.
func (f *file) chunkHosts(chunk uint64) []modules.NetAddress {
	f.mu.RLock()
	defer f.mu.RUnlock()

	var old []modules.NetAddress
	for _, fc := range f.contracts {
		for _, p := range fc.Pieces {
			if p.Chunk == chunk {
				old = append(old, fc.IP)
				break
			}
		}
	}
	return old
}

// expiringContracts returns the contracts that will expire soon.
// TODO: what if contract has fully expired?
func (f *file) expiringContracts(height types.BlockHeight) []fileContract {
	f.mu.RLock()
	defer f.mu.RUnlock()

	var expiring []fileContract
	for _, fc := range f.contracts {
		if height >= fc.WindowStart-renewThreshold {
			expiring = append(expiring, fc)
		}
	}
	return expiring
}

// offlineChunks returns the chunks belonging to "offline" hosts -- hosts that
// do not meet uptime requirements. Importantly, only chunks missing more than
// half their redundancy are returned.
func (f *file) offlineChunks(hdb hostDB) map[uint64][]uint64 {
	f.mu.RLock()
	defer f.mu.RUnlock()

	// helper function for determining if a host is offline. A host is
	// considered offline if it is in AllHosts but not ActiveHosts. For good
	// order notation, we covert AllHosts and ActiveHosts to a single map. The
	// map lookup will return true if the host was in ActiveHosts.
	//
	// TODO: once the hostdb is persistent, checking for existence in
	// ActiveHosts should be sufficient.
	hostSet := make(map[modules.NetAddress]bool)
	for _, host := range hdb.AllHosts() {
		hostSet[host.NetAddress] = false
	}
	for _, host := range hdb.ActiveHosts() {
		hostSet[host.NetAddress] = true
	}
	isOffline := func(addr modules.NetAddress) bool {
		active, exists := hostSet[addr]
		return exists && !active
	}

	// mark all pieces belonging to offline hosts.
	offline := make(map[uint64][]uint64)
	for _, fc := range f.contracts {
		if isOffline(fc.IP) {
			for _, p := range fc.Pieces {
				offline[p.Chunk] = append(offline[p.Chunk], p.Piece)
			}
		}
	}
	// filter out chunks missing less than half of their redundancy
	filtered := make(map[uint64][]uint64)
	for chunk, pieces := range offline {
		if len(pieces) > f.erasureCode.NumPieces()/2 {
			filtered[chunk] = pieces
		}
	}
	return filtered
}

// threadedRepairLoop improves the health of files tracked by the renter by
// reuploading their missing pieces. Multiple repair attempts may be necessary
// before the file reaches full redundancy.
func (r *Renter) threadedRepairLoop() {
	// Files are repaired concurrently. A repair worker must acquire a 'token'
	// in order to run, and returns the token to the pool when it has
	// finished.
	tokenPool := make(chan struct{}, repairThreads)
	// fill the tokenPool with tokens
	for i := 0; i < repairThreads; i++ {
		tokenPool <- struct{}{}
	}

	for {
		time.Sleep(5 * time.Second)

		if !r.wallet.Unlocked() {
			continue
		}

		// make copy of repair set under lock
		repairing := make(map[string]trackedFile)
		id := r.mu.RLock()
		for name, meta := range r.tracking {
			repairing[name] = meta
		}
		r.mu.RUnlock(id)

		var wg sync.WaitGroup
		wg.Add(len(repairing))
		for name, meta := range repairing {
			go func(name string, meta trackedFile) {
				defer wg.Done()
				t := <-tokenPool                 // acquire token
				r.threadedRepairFile(name, meta) // repair
				tokenPool <- t                   // return token
			}(name, meta)
		}
		// wait for all repairs to complete before looping; otherwise we risk
		// spawning multiple repair threads for the same file.
		wg.Wait()
	}
}

// threadedRepairFile repairs and saves an individual file.
func (r *Renter) threadedRepairFile(name string, meta trackedFile) {
	// helper function
	logAndRemove := func(fmt string, args ...interface{}) {
		r.log.Printf(fmt, args...)
		id := r.mu.Lock()
		delete(r.tracking, name)
		r.mu.Unlock(id)
	}

	id := r.mu.RLock()
	f, ok := r.files[name]
	r.mu.RUnlock(id)
	if !ok {
		logAndRemove("removing %v from repair set: no longer tracking that file", name)
		return
	}

	// check for expiration
	height := r.cs.Height()
	if !meta.Renew && meta.EndHeight < height {
		logAndRemove("removing %v from repair set: storage period has ended", name)
		return
	}

	// determine if there is any work to do
	incChunks := f.incompleteChunks()
	offlineChunks := f.offlineChunks(r.hostDB)
	expContracts := f.expiringContracts(height)
	if len(incChunks) == 0 && len(offlineChunks) == 0 && (!meta.Renew || len(expContracts) == 0) {
		return
	}

	// open file handle
	handle, err := os.Open(meta.RepairPath)
	if err != nil {
		logAndRemove("removing %v from repair set: %v", name, err)
		return
	}
	defer handle.Close()

	// repair incomplete chunks
	if len(incChunks) != 0 {
		r.log.Printf("repairing %v chunks of %v", len(incChunks), f.name)
		var duration types.BlockHeight
		if meta.Renew {
			duration = defaultDuration
		} else {
			duration = meta.EndHeight - height
		}
		r.repairChunks(f, handle, incChunks, duration)
	}

	// repair offline chunks
	if len(offlineChunks) != 0 {
		r.log.Printf("reuploading %v offline chunks of %v", len(offlineChunks), f.name)
		var duration types.BlockHeight
		if meta.Renew {
			duration = defaultDuration
		} else {
			duration = meta.EndHeight - height
		}
		r.repairChunks(f, handle, offlineChunks, duration)
	}

	// renew expiring contracts
	if meta.Renew && len(expContracts) != 0 {
		r.log.Printf("renewing %v contracts of %v", len(expContracts), f.name)
		newHeight := height + defaultDuration
		r.renewContracts(f, expContracts, newHeight)
	}

	// save the repaired file data
	f.mu.RLock()
	err = r.saveFile(f)
	f.mu.RUnlock()
	if err != nil {
		// definitely bad, but we probably shouldn't delete from the
		// repair set if this happens
		r.log.Printf("failed to save repaired file %v: %v", name, err)
	}
}

// repairChunks uploads missing chunks of f to new hosts.
func (r *Renter) repairChunks(f *file, handle io.ReaderAt, chunks map[uint64][]uint64, duration types.BlockHeight) {
	// create host pool
	contractSize := (f.pieceSize + crypto.TwofishOverhead) * uint64(len(chunks)) // each host gets one piece of each chunk
	pool, err := r.hostDB.NewPool(contractSize, duration)
	if err != nil {
		r.log.Printf("failed to repair %v: %v", f.name, err)
		return
	}
	defer pool.Close() // heh

	for chunk, pieces := range chunks {
		// Determine host set. We want one host for each missing piece, and no
		// repeats of other hosts of this chunk.
		hosts := pool.UniqueHosts(len(pieces), f.chunkHosts(chunk))
		if len(hosts) == 0 {
			r.log.Printf("aborting repair of %v: not enough hosts", f.name)
			return
		}
		// upload to new hosts
		err = f.repair(chunk, pieces, handle, hosts)
		if err != nil {
			r.log.Printf("aborting repair of %v: %v", f.name, err)
			return
		}
	}
}

// renewContracts renews each of the supplied contracts, replacing their entry
// in f with the new contract.
func (r *Renter) renewContracts(f *file, contracts []fileContract, newHeight types.BlockHeight) {
	for _, c := range contracts {
		newID, err := r.hostDB.Renew(c.ID, newHeight)
		if err != nil {
			r.log.Printf("failed to renew contract %v: %v", c.ID, err)
			continue
		}
		f.mu.Lock()
		f.contracts[newID] = fileContract{
			ID:          newID,
			IP:          c.IP,
			Pieces:      c.Pieces,
			WindowStart: newHeight,
		}
		// need to delete the old contract; otherwise f.expiringContracts
		// will continue to return it
		delete(f.contracts, c.ID)
		f.mu.Unlock()
	}
}
<|endoftext|>