<commit_before>package client

import (
	"net/http"
	"os"

	bosherr "github.com/cloudfoundry/bosh-utils/errors"
	boshlog "github.com/cloudfoundry/bosh-utils/logger"

	"bosh-google-cpi/google/config"

	"golang.org/x/oauth2"
	oauthgoogle "golang.org/x/oauth2/google"
	computebeta "google.golang.org/api/compute/v0.beta"
	"google.golang.org/api/compute/v1"
	"google.golang.org/api/storage/v1"
)

const (
	computeScope = compute.ComputeScope
	storageScope = storage.DevstorageFullControlScope
	metadataHost = "metadata.google.internal"
)

type GoogleClient struct {
	config          config.Config
	computeService  *compute.Service
	computeServiceB *computebeta.Service
	storageService  *storage.Service
	logger          boshlog.Logger
}

func NewGoogleClient(
	config config.Config,
	logger boshlog.Logger,
) (GoogleClient, error) {
	var err error
	var computeClient, storageClient *http.Client
	userAgent := "bosh-google-cpi/0.0.1"

	if config.JSONKey != "" {
		computeJwtConf, err := oauthgoogle.JWTConfigFromJSON([]byte(config.JSONKey), computeScope)
		if err != nil {
			return GoogleClient{}, bosherr.WrapError(err, "Reading Google JSON Key")
		}
		computeClient = computeJwtConf.Client(oauth2.NoContext)

		storageJwtConf, err := oauthgoogle.JWTConfigFromJSON([]byte(config.JSONKey), storageScope)
		if err != nil {
			return GoogleClient{}, bosherr.WrapError(err, "Reading Google JSON Key")
		}
		storageClient = storageJwtConf.Client(oauth2.NoContext)
	} else {
		if v := os.Getenv("GCE_METADATA_HOST"); v == "" {
			os.Setenv("GCE_METADATA_HOST", metadataHost)
		}
		computeClient, err = oauthgoogle.DefaultClient(oauth2.NoContext, computeScope)
		if err != nil {
			return GoogleClient{}, bosherr.WrapError(err, "Creating a Google default client")
		}

		storageClient, err = oauthgoogle.DefaultClient(oauth2.NoContext, storageScope)
		if err != nil {
			return GoogleClient{}, bosherr.WrapError(err, "Creating a Google default client")
		}
	}

	// Custom RoundTripper for retries
	computeRetrier := &RetryTransport{
		Base:       computeClient.Transport,
		MaxRetries: 3,
	}
	computeClient.Transport = computeRetrier
	computeService, err := compute.New(computeClient)
	if err != nil {
		return GoogleClient{}, bosherr.WrapError(err, "Creating a Google Compute Service client")
	}
	computeService.UserAgent = userAgent

	computeServiceB, err := computebeta.New(computeClient)
	if err != nil {
		return GoogleClient{}, bosherr.WrapError(err, "Creating a Google Compute Service client")
	}
	computeServiceB.UserAgent = userAgent

	// Custom RoundTripper for retries
	storageRetrier := &RetryTransport{
		Base:       storageClient.Transport,
		MaxRetries: 3,
	}
	storageClient.Transport = storageRetrier
	storageService, err := storage.New(storageClient)
	if err != nil {
		return GoogleClient{}, bosherr.WrapError(err, "Creating a Google Storage Service client")
	}
	storageService.UserAgent = userAgent

	return GoogleClient{
		config:          config,
		computeService:  computeService,
		computeServiceB: computeServiceB,
		storageService:  storageService,
		logger:          logger,
	}, nil
}

func (c GoogleClient) Project() string {
	return c.config.Project
}

func (c GoogleClient) DefaultRootDiskSizeGb() int {
	return c.config.DefaultRootDiskSizeGb
}

func (c GoogleClient) DefaultRootDiskType() string {
	return c.config.DefaultRootDiskType
}

func (c GoogleClient) ComputeService() *compute.Service {
	return c.computeService
}

func (c GoogleClient) ComputeBetaService() *computebeta.Service {
	return c.computeServiceB
}

func (c GoogleClient) StorageService() *storage.Service {
	return c.storageService
}
<commit_msg>Switches metadata service host to IP address<commit_after>package client

import (
	"net/http"
	"os"

	bosherr "github.com/cloudfoundry/bosh-utils/errors"
	boshlog "github.com/cloudfoundry/bosh-utils/logger"

	"bosh-google-cpi/google/config"

	"golang.org/x/oauth2"
	oauthgoogle "golang.org/x/oauth2/google"
	computebeta "google.golang.org/api/compute/v0.beta"
	"google.golang.org/api/compute/v1"
	"google.golang.org/api/storage/v1"
)

const (
	computeScope = compute.ComputeScope
	storageScope = storage.DevstorageFullControlScope
	// Metadata Host needs to be IP address, rather than FQDN, in case the system
	// is set up to use public DNS servers, which would not resolve correctly.
	metadataHost = "169.254.169.254"
)

type GoogleClient struct {
	config          config.Config
	computeService  *compute.Service
	computeServiceB *computebeta.Service
	storageService  *storage.Service
	logger          boshlog.Logger
}

func NewGoogleClient(
	config config.Config,
	logger boshlog.Logger,
) (GoogleClient, error) {
	var err error
	var computeClient, storageClient *http.Client
	userAgent := "bosh-google-cpi/0.0.1"

	if config.JSONKey != "" {
		computeJwtConf, err := oauthgoogle.JWTConfigFromJSON([]byte(config.JSONKey), computeScope)
		if err != nil {
			return GoogleClient{}, bosherr.WrapError(err, "Reading Google JSON Key")
		}
		computeClient = computeJwtConf.Client(oauth2.NoContext)

		storageJwtConf, err := oauthgoogle.JWTConfigFromJSON([]byte(config.JSONKey), storageScope)
		if err != nil {
			return GoogleClient{}, bosherr.WrapError(err, "Reading Google JSON Key")
		}
		storageClient = storageJwtConf.Client(oauth2.NoContext)
	} else {
		if v := os.Getenv("GCE_METADATA_HOST"); v == "" {
			os.Setenv("GCE_METADATA_HOST", metadataHost)
		}
		computeClient, err = oauthgoogle.DefaultClient(oauth2.NoContext, computeScope)
		if err != nil {
			return GoogleClient{}, bosherr.WrapError(err, "Creating a Google default client")
		}

		storageClient, err = oauthgoogle.DefaultClient(oauth2.NoContext, storageScope)
		if err != nil {
			return GoogleClient{}, bosherr.WrapError(err, "Creating a Google default client")
		}
	}

	// Custom RoundTripper for retries
	computeRetrier := &RetryTransport{
		Base:       computeClient.Transport,
		MaxRetries: 3,
	}
	computeClient.Transport = computeRetrier
	computeService, err := compute.New(computeClient)
	if err != nil {
		return GoogleClient{}, bosherr.WrapError(err, "Creating a Google Compute Service client")
	}
	computeService.UserAgent = userAgent

	computeServiceB, err := computebeta.New(computeClient)
	if err != nil {
		return GoogleClient{}, bosherr.WrapError(err, "Creating a Google Compute Service client")
	}
	computeServiceB.UserAgent = userAgent

	// Custom RoundTripper for retries
	storageRetrier := &RetryTransport{
		Base:       storageClient.Transport,
		MaxRetries: 3,
	}
	storageClient.Transport = storageRetrier
	storageService, err := storage.New(storageClient)
	if err != nil {
		return GoogleClient{}, bosherr.WrapError(err, "Creating a Google Storage Service client")
	}
	storageService.UserAgent = userAgent

	return GoogleClient{
		config:          config,
		computeService:  computeService,
		computeServiceB: computeServiceB,
		storageService:  storageService,
		logger:          logger,
	}, nil
}

func (c GoogleClient) Project() string {
	return c.config.Project
}

func (c GoogleClient) DefaultRootDiskSizeGb() int {
	return c.config.DefaultRootDiskSizeGb
}

func (c GoogleClient) DefaultRootDiskType() string {
	return c.config.DefaultRootDiskType
}

func (c GoogleClient) ComputeService() *compute.Service {
	return c.computeService
}

func (c GoogleClient) ComputeBetaService() *computebeta.Service {
	return c.computeServiceB
}

func (c GoogleClient) StorageService() *storage.Service {
	return c.storageService
}
<|endoftext|>


<commit_before>package persist

import (
	"crypto/md5"
	"crypto/sha1"
	"crypto/sha256"
	"crypto/sha512"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/go-units"
	"github.com/maliceio/malice/libmalice/maldirs"
	"github.com/maliceio/malice/libmalice/malutils"
	"github.com/maliceio/malice/utils/clitable"
	"github.com/rakyll/magicmime"
	// "github.com/dutchcoders/gossdeep"
)

// File is a file object
type File struct {
	Name  string `json:"name"`
	Path  string `json:"path"`
	Valid bool   `json:"valid"`
	Size  string `json:"szie"`
	// Size   int64
	// CRC32  string
	MD5    string `json:"md5"`
	SHA1   string `json:"sha1"`
	SHA256 string `json:"sha256"`
	SHA512 string `json:"sha512"`
	// Ssdeep string `json:"ssdeep"`
	Mime string `json:"mime"`
	Arch string `json:"arch"`
	Data []byte `json:"data"`
}

// Init initializes the File object
func (file *File) Init() {
	if file.Path != "" {
		file.GetName()
		file.GetSize()
		file.getData()
		// file.GetCRC32()
		file.GetMD5()
		file.GetSHA1()
		file.GetSHA256()
		file.GetSHA512()
		file.GetFileMimeType()
		file.CopyToSamples()
		file.Data = nil
	} else {
		log.Fatalf("error occured during file.Init() because file.Path was not set.")
	}
}

// CopyToSamples copys input file to samples folder
func (file *File) CopyToSamples() error {

	// Make .malice directory if it doesn't exist
	if _, err := os.Stat(maldirs.GetSampledsDir()); os.IsNotExist(err) {
		os.MkdirAll(maldirs.GetSampledsDir(), 0777)
	}

	err := malutils.CopyFile(file.Path, path.Join(maldirs.GetSampledsDir(), file.SHA256))
	assert(err)
	return err
}

// GetName returns file name
func (file *File) GetName() (name string, err error) {
	fileHandle, err := os.Open(file.Path)
	if err != nil {
		return
	}
	defer fileHandle.Close()

	stat, err := fileHandle.Stat()
	if err != nil {
		return
	}

	name = stat.Name()

	file.Name = name

	return
}

// GetSize calculates file Size
func (file *File) GetSize() (bytes int64, err error) {
	fileHandle, err := os.Open(file.Path)
	if err != nil {
		return
	}
	defer fileHandle.Close()

	stat, err := fileHandle.Stat()
	if err != nil {
		return
	}

	bytes = stat.Size()

	file.Size = units.HumanSize(float64(bytes))

	return
}

// getData loads file into []bytes
// TODO: This is probably pretty dumb to keep this data in memory
func (file *File) getData() {
	dat, err := ioutil.ReadFile(file.Path)
	assert(err)
	file.Data = dat
}

// // GetCRC32 calculates the Files CRC32
// func (file *File) GetCRC32() (hCRC32Sum string, err error) {
//
// 	var castagnoliTable = crc32.MakeTable(crc32.Castagnoli)
//
// 	crc := crc32.New(castagnoliTable)
// 	_, err = crc.Write(file.Data)
// 	assert(err)
// 	hCRC32Sum = fmt.Sprintf("%x", crc.Sum32())
//
// 	file.CRC32 = hCRC32Sum
//
// 	return
// }

// GetMD5 calculates the Files md5sum
func (file *File) GetMD5() (hMd5Sum string, err error) {

	hmd5 := md5.New()
	_, err = hmd5.Write(file.Data)
	assert(err)
	hMd5Sum = fmt.Sprintf("%x", hmd5.Sum(nil))

	file.MD5 = hMd5Sum

	return
}

// GetSHA1 calculates the Files sha256sum
func (file *File) GetSHA1() (h1Sum string, err error) {

	h1 := sha1.New()
	_, err = h1.Write(file.Data)
	assert(err)
	h1Sum = fmt.Sprintf("%x", h1.Sum(nil))

	file.SHA1 = h1Sum

	return
}

// GetSHA256 calculates the Files sha256sum
func (file *File) GetSHA256() (h256Sum string, err error) {

	h256 := sha256.New()
	_, err = h256.Write(file.Data)
	assert(err)
	h256Sum = fmt.Sprintf("%x", h256.Sum(nil))

	file.SHA256 = h256Sum

	return
}

// GetSHA512 calculates the Files sha256sum
func (file *File) GetSHA512() (h512Sum string, err error) {

	h512 := sha512.New()
	_, err = h512.Write(file.Data)
	assert(err)
	h512Sum = fmt.Sprintf("%x", h512.Sum(nil))

	file.SHA512 = h512Sum

	return
}

// GetFileMimeType returns the mime-type of a file path
func (file *File) GetFileMimeType() (mimetype string, err error) {

	if err := magicmime.Open(magicmime.MAGIC_MIME_TYPE | magicmime.MAGIC_SYMLINK | magicmime.MAGIC_ERROR); err != nil {
		log.Fatal(err)
	}
	defer magicmime.Close()

	mimetype, err = magicmime.TypeByFile(file.Path)
	if err != nil {
		log.Fatalf("error occured during type lookup: %v", err)
	}

	file.Mime = mimetype
	// log.Printf("mime-type: %v", mimetype)
	return
}

// ToJSON converts File object to []byte JSON
func (file *File) ToJSON() []byte {
	fileJSON, err := json.Marshal(file)
	assert(err)
	return fileJSON
}

// PrintFileDetails prints file details
func (file *File) PrintFileDetails() {
	table := clitable.New([]string{"Field", "Value"})
	table.AddRow(map[string]interface{}{"Field": "Name", "Value": file.Name})
	table.AddRow(map[string]interface{}{"Field": "Path", "Value": file.Path})
	table.AddRow(map[string]interface{}{"Field": "Valid", "Value": file.Valid})
	table.AddRow(map[string]interface{}{"Field": "Size", "Value": file.Size})
	// table.AddRow(map[string]interface{}{"Field": "CRC32", "Value": file.CRC32})
	table.AddRow(map[string]interface{}{"Field": "MD5", "Value": file.MD5})
	table.AddRow(map[string]interface{}{"Field": "SHA1", "Value": file.SHA1})
	table.AddRow(map[string]interface{}{"Field": "SHA256", "Value": file.SHA256})
	table.AddRow(map[string]interface{}{"Field": "SHA512", "Value": file.SHA512})
	// table.AddRow(map[string]interface{}{"Field": "Ssdeep", "Value": file.Ssdeep})
	table.AddRow(map[string]interface{}{"Field": "Mime", "Value": file.Mime})
	table.Markdown = true
	table.Print()
	// fmt.Println("Name: ", file.Name)
	// fmt.Println("Path: ", file.Path)
	// fmt.Println("Valid: ", file.Valid)
	// fmt.Println("Size: ", file.Size)
	// fmt.Println("CRC32: ", file.CRC32)
	// fmt.Println("MD5: ", file.MD5)
	// fmt.Println("SHA1: ", file.SHA1)
	// fmt.Println("SHA256: ", file.SHA256)
	// fmt.Println("SHA512: ", file.SHA512)
	// fmt.Println("Ssdeep: ", file.Ssdeep)
	// fmt.Println("Mime: ", file.Mime)
}

// // GetSsdeep calculates the Files ssdeep
// func (file File) GetSsdeep(data []byte) (hssdeep string, err error) {
//
// 	hssdeep, err = ssdeep.HashFilename(file)
// 	if err != nil {
// 		return "", err
// 	}
//
// 	return
// }
//
// // CompareSsdeep returns the percent that two hashes are similar
// func CompareSsdeep(firstHash, secondHash string) (percent int, err error) {
//
// 	percent, err = ssdeep.Compare(firstHash, secondHash)
// 	if err != nil {
// 		return 0, err
// 	}
//
// 	return
// }

func assert(err error) {
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>fix typo<commit_after>package persist

import (
	"crypto/md5"
	"crypto/sha1"
	"crypto/sha256"
	"crypto/sha512"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/go-units"
	"github.com/maliceio/malice/libmalice/maldirs"
	"github.com/maliceio/malice/libmalice/malutils"
	"github.com/maliceio/malice/utils/clitable"
	"github.com/rakyll/magicmime"
	// "github.com/dutchcoders/gossdeep"
)

// File is a file object
type File struct {
	Name  string `json:"name"`
	Path  string `json:"path"`
	Valid bool   `json:"valid"`
	Size  string `json:"size"`
	// Size   int64
	// CRC32  string
	MD5    string `json:"md5"`
	SHA1   string `json:"sha1"`
	SHA256 string `json:"sha256"`
	SHA512 string `json:"sha512"`
	// Ssdeep string `json:"ssdeep"`
	Mime string `json:"mime"`
	Arch string `json:"arch"`
	Data []byte `json:"data"`
}

// Init initializes the File object
func (file *File) Init() {
	if file.Path != "" {
		file.GetName()
		file.GetSize()
		file.getData()
		// file.GetCRC32()
		file.GetMD5()
		file.GetSHA1()
		file.GetSHA256()
		file.GetSHA512()
		file.GetFileMimeType()
		file.CopyToSamples()
		file.Data = nil
	} else {
		log.Fatalf("error occured during file.Init() because file.Path was not set.")
	}
}

// CopyToSamples copys input file to samples folder
func (file *File) CopyToSamples() error {

	// Make .malice directory if it doesn't exist
	if _, err := os.Stat(maldirs.GetSampledsDir()); os.IsNotExist(err) {
		os.MkdirAll(maldirs.GetSampledsDir(), 0777)
	}

	err := malutils.CopyFile(file.Path, path.Join(maldirs.GetSampledsDir(), file.SHA256))
	assert(err)
	return err
}

// GetName returns file name
func (file *File) GetName() (name string, err error) {
	fileHandle, err := os.Open(file.Path)
	if err != nil {
		return
	}
	defer fileHandle.Close()

	stat, err := fileHandle.Stat()
	if err != nil {
		return
	}

	name = stat.Name()

	file.Name = name

	return
}

// GetSize calculates file Size
func (file *File) GetSize() (bytes int64, err error) {
	fileHandle, err := os.Open(file.Path)
	if err != nil {
		return
	}
	defer fileHandle.Close()

	stat, err := fileHandle.Stat()
	if err != nil {
		return
	}

	bytes = stat.Size()

	file.Size = units.HumanSize(float64(bytes))

	return
}

// getData loads file into []bytes
// TODO: This is probably pretty dumb to keep this data in memory
func (file *File) getData() {
	dat, err := ioutil.ReadFile(file.Path)
	assert(err)
	file.Data = dat
}

// // GetCRC32 calculates the Files CRC32
// func (file *File) GetCRC32() (hCRC32Sum string, err error) {
//
// 	var castagnoliTable = crc32.MakeTable(crc32.Castagnoli)
//
// 	crc := crc32.New(castagnoliTable)
// 	_, err = crc.Write(file.Data)
// 	assert(err)
// 	hCRC32Sum = fmt.Sprintf("%x", crc.Sum32())
//
// 	file.CRC32 = hCRC32Sum
//
// 	return
// }

// GetMD5 calculates the Files md5sum
func (file *File) GetMD5() (hMd5Sum string, err error) {

	hmd5 := md5.New()
	_, err = hmd5.Write(file.Data)
	assert(err)
	hMd5Sum = fmt.Sprintf("%x", hmd5.Sum(nil))

	file.MD5 = hMd5Sum

	return
}

// GetSHA1 calculates the Files sha256sum
func (file *File) GetSHA1() (h1Sum string, err error) {

	h1 := sha1.New()
	_, err = h1.Write(file.Data)
	assert(err)
	h1Sum = fmt.Sprintf("%x", h1.Sum(nil))

	file.SHA1 = h1Sum

	return
}

// GetSHA256 calculates the Files sha256sum
func (file *File) GetSHA256() (h256Sum string, err error) {

	h256 := sha256.New()
	_, err = h256.Write(file.Data)
	assert(err)
	h256Sum = fmt.Sprintf("%x", h256.Sum(nil))

	file.SHA256 = h256Sum

	return
}

// GetSHA512 calculates the Files sha256sum
func (file *File) GetSHA512() (h512Sum string, err error) {

	h512 := sha512.New()
	_, err = h512.Write(file.Data)
	assert(err)
	h512Sum = fmt.Sprintf("%x", h512.Sum(nil))

	file.SHA512 = h512Sum

	return
}

// GetFileMimeType returns the mime-type of a file path
func (file *File) GetFileMimeType() (mimetype string, err error) {

	if err := magicmime.Open(magicmime.MAGIC_MIME_TYPE | magicmime.MAGIC_SYMLINK | magicmime.MAGIC_ERROR); err != nil {
		log.Fatal(err)
	}
	defer magicmime.Close()

	mimetype, err = magicmime.TypeByFile(file.Path)
	if err != nil {
		log.Fatalf("error occured during type lookup: %v", err)
	}

	file.Mime = mimetype
	// log.Printf("mime-type: %v", mimetype)
	return
}

// ToJSON converts File object to []byte JSON
func (file *File) ToJSON() []byte {
	fileJSON, err := json.Marshal(file)
	assert(err)
	return fileJSON
}

// PrintFileDetails prints file details
func (file *File) PrintFileDetails() {
	table := clitable.New([]string{"Field", "Value"})
	table.AddRow(map[string]interface{}{"Field": "Name", "Value": file.Name})
	table.AddRow(map[string]interface{}{"Field": "Path", "Value": file.Path})
	table.AddRow(map[string]interface{}{"Field": "Valid", "Value": file.Valid})
	table.AddRow(map[string]interface{}{"Field": "Size", "Value": file.Size})
	// table.AddRow(map[string]interface{}{"Field": "CRC32", "Value": file.CRC32})
	table.AddRow(map[string]interface{}{"Field": "MD5", "Value": file.MD5})
	table.AddRow(map[string]interface{}{"Field": "SHA1", "Value": file.SHA1})
	table.AddRow(map[string]interface{}{"Field": "SHA256", "Value": file.SHA256})
	table.AddRow(map[string]interface{}{"Field": "SHA512", "Value": file.SHA512})
	// table.AddRow(map[string]interface{}{"Field": "Ssdeep", "Value": file.Ssdeep})
	table.AddRow(map[string]interface{}{"Field": "Mime", "Value": file.Mime})
	table.Markdown = true
	table.Print()
	// fmt.Println("Name: ", file.Name)
	// fmt.Println("Path: ", file.Path)
	// fmt.Println("Valid: ", file.Valid)
	// fmt.Println("Size: ", file.Size)
	// fmt.Println("CRC32: ", file.CRC32)
	// fmt.Println("MD5: ", file.MD5)
	// fmt.Println("SHA1: ", file.SHA1)
	// fmt.Println("SHA256: ", file.SHA256)
	// fmt.Println("SHA512: ", file.SHA512)
	// fmt.Println("Ssdeep: ", file.Ssdeep)
	// fmt.Println("Mime: ", file.Mime)
}

// // GetSsdeep calculates the Files ssdeep
// func (file File) GetSsdeep(data []byte) (hssdeep string, err error) {
//
// 	hssdeep, err = ssdeep.HashFilename(file)
// 	if err != nil {
// 		return "", err
// 	}
//
// 	return
// }
//
// // CompareSsdeep returns the percent that two hashes are similar
// func CompareSsdeep(firstHash, secondHash string) (percent int, err error) {
//
// 	percent, err = ssdeep.Compare(firstHash, secondHash)
// 	if err != nil {
// 		return 0, err
// 	}
//
// 	return
// }

func assert(err error) {
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"io/ioutil"
	"net/http"
	"os"
	"time"

	"github.com/andelf/go-curl"
	"github.com/nu7hatch/gouuid"
)

type Config struct {
	Location      string
	ChecksUrl     string
	MeasurerCount int
}

type Check struct {
	Id  string `json:"id"`
	Url string `json:"url"`
}

type Measurement struct {
	Check             Check   `json:"check"`
	Id                string  `json:"id"`
	Location          string  `json:"location"`
	T                 int     `json:"t"`
	ExitStatus        int     `json:"exit_status"`
	ConnectTime       float64 `json:"connect_time,omitempty"`
	StartTransferTime float64 `json:"starttransfer_time,omitempty"`
	LocalIp           string  `json:"local_ip,omitempty"`
	PrimaryIp         string  `json:"primary_ip,omitempty"`
	TotalTime         float64 `json:"total_time,omitempty"`
	HttpStatus        int     `json:"http_status,omitempty"`
	NameLookupTime    float64 `json:"namelookup_time,omitempty"`
	SizeDownload      float64 `json:"size_download,omitempty"`
}

func (c *Check) Measure(config Config) Measurement {
	var m Measurement

	id, _ := uuid.NewV4()
	m.Id = id.String()
	m.Check = *c
	m.Location = config.Location

	easy := curl.EasyInit()
	defer easy.Cleanup()

	easy.Setopt(curl.OPT_URL, c.Url)

	// dummy func for curl output
	noOut := func(buf []byte, userdata interface{}) bool {
		return true
	}

	easy.Setopt(curl.OPT_WRITEFUNCTION, noOut)
	easy.Setopt(curl.OPT_CONNECTTIMEOUT, 10)
	easy.Setopt(curl.OPT_TIMEOUT, 10)

	now := time.Now()
	m.T = int(now.Unix())

	if err := easy.Perform(); err != nil {
		if e, ok := err.(curl.CurlError); ok {
			m.ExitStatus = (int(e))
			return m
		}
		os.Exit(1)
	}

	m.ExitStatus = 0
	http_status, _ := easy.Getinfo(curl.INFO_RESPONSE_CODE)
	m.HttpStatus = http_status.(int)

	connect_time, _ := easy.Getinfo(curl.INFO_CONNECT_TIME)
	m.ConnectTime = connect_time.(float64)

	namelookup_time, _ := easy.Getinfo(curl.INFO_NAMELOOKUP_TIME)
	m.NameLookupTime = namelookup_time.(float64)

	starttransfer_time, _ := easy.Getinfo(curl.INFO_STARTTRANSFER_TIME)
	m.StartTransferTime = starttransfer_time.(float64)

	total_time, _ := easy.Getinfo(curl.INFO_TOTAL_TIME)
	m.TotalTime = total_time.(float64)

	local_ip, _ := easy.Getinfo(curl.INFO_LOCAL_IP)
	m.LocalIp = local_ip.(string)

	primary_ip, _ := easy.Getinfo(curl.INFO_PRIMARY_IP)
	m.PrimaryIp = primary_ip.(string)

	size_download, _ := easy.Getinfo(curl.INFO_SIZE_DOWNLOAD)
	m.SizeDownload = size_download.(float64)

	return m
}

func measurer(config Config, toMeasurer chan Check, toStreamer chan Measurement) {
	for {
		c := <-toMeasurer
		m := c.Measure(config)

		toStreamer <- m
	}
}

func streamer(config Config, toStreamer chan Measurement) {
	h := func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		enc := json.NewEncoder(w)
		for {
			err := enc.Encode(<-toStreamer)
			if err != nil {
				return
			}

			if f, ok := w.(http.Flusher); ok {
				f.Flush()
			}
		}
	}

	http.HandleFunc("/measurements", h)
	http.ListenAndServe(":5000", nil)
}

func getChecks(config Config) []Check {
	url := config.ChecksUrl

	res, err := http.Get(url)
	if err != nil {
		panic(err)
	}
	defer res.Body.Close()

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		panic(err)
	}

	var checks []Check
	err = json.Unmarshal(body, &checks)
	if err != nil {
		panic(err)
	}

	return checks
}

func scheduler(check Check, toMeasurer chan Check) {
	for {
		toMeasurer <- check
		time.Sleep(1000 * time.Millisecond)
	}
}

func main() {
	config := Config{}
	flag.StringVar(&config.Location, "location", "undefined", "location of this sensor")
	flag.StringVar(&config.ChecksUrl, "checks_url", "https://s3.amazonaws.com/canary-public-data/checks.json", "URL for check data")
	flag.IntVar(&config.MeasurerCount, "measurer_count", 1, "number of measurers to run")
	flag.Parse()

	check_list := getChecks(config)

	toMeasurer := make(chan Check)
	toStreamer := make(chan Measurement)

	// spawn one scheduler per check
	for _, c := range check_list {
		go scheduler(c, toMeasurer)
	}

	// spawn N measurers
	for i := 0; i < config.MeasurerCount; i++ {
		go measurer(config, toMeasurer, toStreamer)
	}

	// stream measurements to clients over HTTP
	go streamer(config, toStreamer)

	select {}
}
<commit_msg>move flag parsing to init()<commit_after>package main

import (
	"encoding/json"
	"flag"
	"io/ioutil"
	"net/http"
	"os"
	"time"

	"github.com/andelf/go-curl"
	"github.com/nu7hatch/gouuid"
)

var config Config

type Config struct {
	Location      string
	ChecksUrl     string
	MeasurerCount int
}

type Check struct {
	Id  string `json:"id"`
	Url string `json:"url"`
}

type Measurement struct {
	Check             Check   `json:"check"`
	Id                string  `json:"id"`
	Location          string  `json:"location"`
	T                 int     `json:"t"`
	ExitStatus        int     `json:"exit_status"`
	ConnectTime       float64 `json:"connect_time,omitempty"`
	StartTransferTime float64 `json:"starttransfer_time,omitempty"`
	LocalIp           string  `json:"local_ip,omitempty"`
	PrimaryIp         string  `json:"primary_ip,omitempty"`
	TotalTime         float64 `json:"total_time,omitempty"`
	HttpStatus        int     `json:"http_status,omitempty"`
	NameLookupTime    float64 `json:"namelookup_time,omitempty"`
	SizeDownload      float64 `json:"size_download,omitempty"`
}

func (c *Check) Measure(config Config) Measurement {
	var m Measurement

	id, _ := uuid.NewV4()
	m.Id = id.String()
	m.Check = *c
	m.Location = config.Location

	easy := curl.EasyInit()
	defer easy.Cleanup()

	easy.Setopt(curl.OPT_URL, c.Url)

	// dummy func for curl output
	noOut := func(buf []byte, userdata interface{}) bool {
		return true
	}

	easy.Setopt(curl.OPT_WRITEFUNCTION, noOut)
	easy.Setopt(curl.OPT_CONNECTTIMEOUT, 10)
	easy.Setopt(curl.OPT_TIMEOUT, 10)

	now := time.Now()
	m.T = int(now.Unix())

	if err := easy.Perform(); err != nil {
		if e, ok := err.(curl.CurlError); ok {
			m.ExitStatus = (int(e))
			return m
		}
		os.Exit(1)
	}

	m.ExitStatus = 0
	http_status, _ := easy.Getinfo(curl.INFO_RESPONSE_CODE)
	m.HttpStatus = http_status.(int)

	connect_time, _ := easy.Getinfo(curl.INFO_CONNECT_TIME)
	m.ConnectTime = connect_time.(float64)

	namelookup_time, _ := easy.Getinfo(curl.INFO_NAMELOOKUP_TIME)
	m.NameLookupTime = namelookup_time.(float64)

	starttransfer_time, _ := easy.Getinfo(curl.INFO_STARTTRANSFER_TIME)
	m.StartTransferTime = starttransfer_time.(float64)

	total_time, _ := easy.Getinfo(curl.INFO_TOTAL_TIME)
	m.TotalTime = total_time.(float64)

	local_ip, _ := easy.Getinfo(curl.INFO_LOCAL_IP)
	m.LocalIp = local_ip.(string)

	primary_ip, _ := easy.Getinfo(curl.INFO_PRIMARY_IP)
	m.PrimaryIp = primary_ip.(string)

	size_download, _ := easy.Getinfo(curl.INFO_SIZE_DOWNLOAD)
	m.SizeDownload = size_download.(float64)

	return m
}

func measurer(config Config, toMeasurer chan Check, toStreamer chan Measurement) {
	for {
		c := <-toMeasurer
		m := c.Measure(config)

		toStreamer <- m
	}
}

func streamer(config Config, toStreamer chan Measurement) {
	h := func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		enc := json.NewEncoder(w)
		for {
			err := enc.Encode(<-toStreamer)
			if err != nil {
				return
			}

			if f, ok := w.(http.Flusher); ok {
				f.Flush()
			}
		}
	}

	http.HandleFunc("/measurements", h)
	http.ListenAndServe(":5000", nil)
}

func getChecks(config Config) []Check {
	url := config.ChecksUrl

	res, err := http.Get(url)
	if err != nil {
		panic(err)
	}
	defer res.Body.Close()

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		panic(err)
	}

	var checks []Check
	err = json.Unmarshal(body, &checks)
	if err != nil {
		panic(err)
	}

	return checks
}

func scheduler(check Check, toMeasurer chan Check) {
	for {
		toMeasurer <- check
		time.Sleep(1000 * time.Millisecond)
	}
}

func init() {
	flag.StringVar(&config.Location, "location", "undefined", "location of this sensor")
	flag.StringVar(&config.ChecksUrl, "checks_url", "https://s3.amazonaws.com/canary-public-data/checks.json", "URL for check data")
	flag.IntVar(&config.MeasurerCount, "measurer_count", 1, "number of measurers to run")
}

func main() {
	flag.Parse()

	check_list := getChecks(config)

	toMeasurer := make(chan Check)
	toStreamer := make(chan Measurement)

	// spawn one scheduler per check
	for _, c := range check_list {
		go scheduler(c, toMeasurer)
	}

	// spawn N measurers
	for i := 0; i < config.MeasurerCount; i++ {
		go measurer(config, toMeasurer, toStreamer)
	}

	// stream measurements to clients over HTTP
	go streamer(config, toStreamer)

	select {}
}
<|endoftext|>


<commit_before>package engine

import (
	"context"
	"fmt"
	"sync"
	"time"

	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/lager/lagerctx"
	"github.com/concourse/concourse/atc"
	"github.com/concourse/concourse/atc/creds"
	"github.com/concourse/concourse/atc/db"
	"github.com/concourse/concourse/atc/exec"
	"github.com/concourse/concourse/atc/metric"
	"github.com/concourse/concourse/tracing"
)

//go:generate counterfeiter . Engine

type Engine interface {
	NewBuild(db.Build) Runnable
	NewCheck(db.Check) Runnable
	ReleaseAll(lager.Logger)
}

//go:generate counterfeiter . Runnable

type Runnable interface {
	Run(logger lager.Logger)
}

//go:generate counterfeiter . StepBuilder

type StepBuilder interface {
	BuildStep(lager.Logger, db.Build) (exec.Step, error)
	CheckStep(lager.Logger, db.Check) (exec.Step, error)

	BuildStepErrored(lager.Logger, db.Build, error)
}

func NewEngine(builder StepBuilder) Engine {
	return &engine{
		builder:       builder,
		release:       make(chan bool),
		trackedStates: new(sync.Map),
		waitGroup:     new(sync.WaitGroup),
	}
}

type engine struct {
	builder       StepBuilder
	release       chan bool
	trackedStates *sync.Map
	waitGroup     *sync.WaitGroup
}

func (engine *engine) ReleaseAll(logger lager.Logger) {
	logger.Info("calling-release-on-builds")

	close(engine.release)

	logger.Info("waiting-on-builds")

	engine.waitGroup.Wait()

	logger.Info("finished-waiting-on-builds")
}

func (engine *engine) NewBuild(build db.Build) Runnable {

	ctx, cancel := context.WithCancel(context.Background())

	return NewBuild(
		ctx,
		cancel,
		build,
		engine.builder,
		engine.release,
		engine.trackedStates,
		engine.waitGroup,
	)
}

func (engine *engine) NewCheck(check db.Check) Runnable {

	ctx, cancel := context.WithCancel(context.Background())

	return NewCheck(
		ctx,
		cancel,
		check,
		engine.builder,
		engine.release,
		engine.trackedStates,
		engine.waitGroup,
	)
}

func NewBuild(
	ctx context.Context,
	cancel func(),
	build db.Build,
	builder StepBuilder,
	release chan bool,
	trackedStates *sync.Map,
	waitGroup *sync.WaitGroup,
) Runnable {
	return &engineBuild{
		ctx:    ctx,
		cancel: cancel,

		build:   build,
		builder: builder,

		release:       release,
		trackedStates: trackedStates,
		waitGroup:     waitGroup,
	}
}

type engineBuild struct {
	ctx    context.Context
	cancel func()

	build   db.Build
	builder StepBuilder

	release       chan bool
	trackedStates *sync.Map
	waitGroup     *sync.WaitGroup

	pipelineCredMgrs []creds.Manager
}

func (b *engineBuild) Run(logger lager.Logger) {
	b.waitGroup.Add(1)
	defer b.waitGroup.Done()

	logger = logger.WithData(lager.Data{
		"build":    b.build.ID(),
		"pipeline": b.build.PipelineName(),
		"job":      b.build.JobName(),
	})

	lock, acquired, err := b.build.AcquireTrackingLock(logger, time.Minute)
	if err != nil {
		logger.Error("failed-to-get-lock", err)
		return
	}

	if !acquired {
		logger.Debug("build-already-tracked")
		return
	}

	defer lock.Release()

	found, err := b.build.Reload()
	if err != nil {
		logger.Error("failed-to-load-build-from-db", err)
		return
	}

	if !found {
		logger.Info("build-not-found")
		return
	}

	if !b.build.IsRunning() {
		logger.Info("build-already-finished")
		return
	}

	notifier, err := b.build.AbortNotifier()
	if err != nil {
		logger.Error("failed-to-listen-for-aborts", err)
		return
	}

	defer notifier.Close()

	ctx, span := tracing.StartSpan(b.ctx, "build", tracing.Attrs{
		"team":     b.build.TeamName(),
		"pipeline": b.build.PipelineName(),
		"job":      b.build.JobName(),
		"build":    b.build.Name(),
	})
	defer span.End()

	step, err := b.builder.BuildStep(logger, b.build)
	if err != nil {
		logger.Error("failed-to-build-step", err)

		// Fails the build if BuildStep returned error. Because some unrecoverable error,
		// like pipeline var_source is wrong, will cause a build to never start
		// to run.
		b.builder.BuildStepErrored(logger, b.build, err)
		b.finish(logger.Session("finish"), err, false)

		return
	}
	b.trackStarted(logger)
	defer b.trackFinished(logger)

	logger.Info("running")

	state := b.runState()
	defer b.clearRunState()

	noleak := make(chan bool)
	defer close(noleak)

	go func() {
		select {
		case <-noleak:
		case <-notifier.Notify():
			logger.Info("aborting")
			b.cancel()
		}
	}()

	done := make(chan error)
	go func() {
		ctx = lagerctx.NewContext(ctx, logger)
		done <- step.Run(ctx, state)
	}()

	select {
	case <-b.release:
		logger.Info("releasing")

	case err = <-done:
		logger.Debug("engine-build-done")
		b.finish(logger.Session("finish"), err, step.Succeeded())
	}
}

func (b *engineBuild) finish(logger lager.Logger, err error, succeeded bool) {
	if err == context.Canceled {
		b.saveStatus(logger, atc.StatusAborted)
		logger.Info("aborted")

	} else if err != nil {
		b.saveStatus(logger, atc.StatusErrored)
		logger.Info("errored", lager.Data{"error": err.Error()})

	} else if succeeded {
		b.saveStatus(logger, atc.StatusSucceeded)
		logger.Info("succeeded")

	} else {
		b.saveStatus(logger, atc.StatusFailed)
		logger.Info("failed")
	}
}

func (b *engineBuild) saveStatus(logger lager.Logger, status atc.BuildStatus) {
	if err := b.build.Finish(db.BuildStatus(status)); err != nil {
		logger.Error("failed-to-finish-build", err)
	}
}

func (b *engineBuild) trackStarted(logger lager.Logger) {
	metric.BuildStarted{
		PipelineName: b.build.PipelineName(),
		JobName:      b.build.JobName(),
		BuildName:    b.build.Name(),
		BuildID:      b.build.ID(),
		TeamName:     b.build.TeamName(),
	}.Emit(logger)
}

func (b *engineBuild) trackFinished(logger lager.Logger) {
	found, err := b.build.Reload()
	if err != nil {
		logger.Error("failed-to-load-build-from-db", err)
		return
	}

	if !found {
		logger.Info("build-removed")
		return
	}

	if !b.build.IsRunning() {
		metric.BuildFinished{
			PipelineName:  b.build.PipelineName(),
			JobName:       b.build.JobName(),
			BuildName:     b.build.Name(),
			BuildID:       b.build.ID(),
			BuildStatus:   b.build.Status(),
			BuildDuration: b.build.EndTime().Sub(b.build.StartTime()),
			TeamName:      b.build.TeamName(),
		}.Emit(logger)
	}
}

func (b *engineBuild) runState() exec.RunState {
	id := fmt.Sprintf("build:%v", b.build.ID())
	existingState, _ := b.trackedStates.LoadOrStore(id, exec.NewRunState())
	return existingState.(exec.RunState)
}

func (b *engineBuild) clearRunState() {
	id := fmt.Sprintf("build:%v", b.build.ID())
	b.trackedStates.Delete(id)
}

func NewCheck(
	ctx context.Context,
	cancel func(),
	check db.Check,
	builder StepBuilder,
	release chan bool,
	trackedStates *sync.Map,
	waitGroup *sync.WaitGroup,
) Runnable {
	return &engineCheck{
		ctx:    ctx,
		cancel: cancel,

		check:   check,
		builder: builder,

		release:       release,
		trackedStates: trackedStates,
		waitGroup:     waitGroup,
	}
}

type engineCheck struct {
	ctx    context.Context
	cancel func()

	check   db.Check
	builder StepBuilder

	release       chan bool
	trackedStates *sync.Map
	waitGroup     *sync.WaitGroup
}

func (c *engineCheck) Run(logger lager.Logger) {
	c.waitGroup.Add(1)
	defer c.waitGroup.Done()

	logger = logger.WithData(lager.Data{
		"check": c.check.ID(),
	})

	lock, acquired, err := c.check.AcquireTrackingLock(logger)
	if err != nil {
		logger.Error("failed-to-get-lock", err)
		return
	}

	if !acquired {
		logger.Debug("check-already-tracked")
		return
	}

	defer lock.Release()

	err = c.check.Start()
	if err != nil {
		logger.Error("failed-to-start-check", err)
		return
	}

	c.trackStarted(logger)
	defer c.trackFinished(logger)

	step, err := c.builder.CheckStep(logger, c.check)
	if err != nil {
		logger.Error("failed-to-create-check-step", err)
		c.check.FinishWithError(fmt.Errorf("create check step: %w", err))
		return
	}

	logger.Info("running")

	state := c.runState()
	defer c.clearRunState()

	done := make(chan error)
	go func() {
		ctx := lagerctx.NewContext(c.ctx, logger)
		done <- step.Run(ctx, state)
	}()

	select {
	case <-c.release:
		logger.Info("releasing")

	case err = <-done:
		if err != nil {
			logger.Info("errored", lager.Data{"error": err.Error()})
			c.check.FinishWithError(fmt.Errorf("run check step: %w", err))
		} else {
			logger.Info("succeeded")
			if err = c.check.Finish(); err != nil {
				logger.Error("failed-to-finish-check", err)
			}
		}
	}
}

func (c *engineCheck) runState() exec.RunState {
	id := fmt.Sprintf("check:%v", c.check.ID())
	existingState, _ := c.trackedStates.LoadOrStore(id, exec.NewRunState())
	return existingState.(exec.RunState)
}

func (c *engineCheck) clearRunState() {
	id := fmt.Sprintf("check:%v", c.check.ID())
	c.trackedStates.Delete(id)
}

func (c *engineCheck) trackStarted(logger lager.Logger) {
	metric.CheckStarted{
		CheckName:             c.check.Plan().Check.Name,
		ResourceConfigScopeID: c.check.ResourceConfigScopeID(),
		CheckStatus:           c.check.Status(),
		CheckPendingDuration:  c.check.StartTime().Sub(c.check.CreateTime()),
	}.Emit(logger)
}

func (c *engineCheck) trackFinished(logger lager.Logger) {
	metric.CheckFinished{
		CheckName:             c.check.Plan().Check.Name,
		ResourceConfigScopeID: c.check.ResourceConfigScopeID(),
		CheckStatus:           c.check.Status(),
		CheckDuration:         c.check.EndTime().Sub(c.check.StartTime()),
	}.Emit(logger)
}
<commit_msg>atc/engine: add build_id to span context<commit_after>package engine

import (
	"context"
	"fmt"
	"strconv"
	"sync"
	"time"

	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/lager/lagerctx"
	"github.com/concourse/concourse/atc"
	"github.com/concourse/concourse/atc/creds"
	"github.com/concourse/concourse/atc/db"
	"github.com/concourse/concourse/atc/exec"
	"github.com/concourse/concourse/atc/metric"
	"github.com/concourse/concourse/tracing"
)

//go:generate counterfeiter . Engine

type Engine interface {
	NewBuild(db.Build) Runnable
	NewCheck(db.Check) Runnable
	ReleaseAll(lager.Logger)
}

//go:generate counterfeiter . Runnable

type Runnable interface {
	Run(logger lager.Logger)
}

//go:generate counterfeiter . StepBuilder

type StepBuilder interface {
	BuildStep(lager.Logger, db.Build) (exec.Step, error)
	CheckStep(lager.Logger, db.Check) (exec.Step, error)

	BuildStepErrored(lager.Logger, db.Build, error)
}

func NewEngine(builder StepBuilder) Engine {
	return &engine{
		builder:       builder,
		release:       make(chan bool),
		trackedStates: new(sync.Map),
		waitGroup:     new(sync.WaitGroup),
	}
}

type engine struct {
	builder       StepBuilder
	release       chan bool
	trackedStates *sync.Map
	waitGroup     *sync.WaitGroup
}

func (engine *engine) ReleaseAll(logger lager.Logger) {
	logger.Info("calling-release-on-builds")

	close(engine.release)

	logger.Info("waiting-on-builds")

	engine.waitGroup.Wait()

	logger.Info("finished-waiting-on-builds")
}

func (engine *engine) NewBuild(build db.Build) Runnable {

	ctx, cancel := context.WithCancel(context.Background())

	return NewBuild(
		ctx,
		cancel,
		build,
		engine.builder,
		engine.release,
		engine.trackedStates,
		engine.waitGroup,
	)
}

func (engine *engine) NewCheck(check db.Check) Runnable {

	ctx, cancel := context.WithCancel(context.Background())

	return NewCheck(
		ctx,
		cancel,
		check,
		engine.builder,
		engine.release,
		engine.trackedStates,
		engine.waitGroup,
	)
}

func NewBuild(
	ctx context.Context,
	cancel func(),
	build db.Build,
	builder StepBuilder,
	release chan bool,
	trackedStates *sync.Map,
	waitGroup *sync.WaitGroup,
) Runnable {
	return &engineBuild{
		ctx:    ctx,
		cancel: cancel,

		build:   build,
		builder: builder,

		release:       release,
		trackedStates: trackedStates,
		waitGroup:     waitGroup,
	}
}

type engineBuild struct {
	ctx    context.Context
	cancel func()

	build   db.Build
	builder StepBuilder

	release       chan bool
	trackedStates *sync.Map
	waitGroup     *sync.WaitGroup

	pipelineCredMgrs []creds.Manager
}

func (b *engineBuild) Run(logger lager.Logger) {
	b.waitGroup.Add(1)
	defer b.waitGroup.Done()

	logger = logger.WithData(lager.Data{
		"build":    b.build.ID(),
		"pipeline": b.build.PipelineName(),
		"job":      b.build.JobName(),
	})

	lock, acquired, err := b.build.AcquireTrackingLock(logger, time.Minute)
	if err != nil {
		logger.Error("failed-to-get-lock", err)
		return
	}

	if !acquired {
		logger.Debug("build-already-tracked")
		return
	}

	defer lock.Release()

	found, err := b.build.Reload()
	if err != nil {
		logger.Error("failed-to-load-build-from-db", err)
		return
	}

	if !found {
		logger.Info("build-not-found")
		return
	}

	if !b.build.IsRunning() {
		logger.Info("build-already-finished")
		return
	}

	notifier, err := b.build.AbortNotifier()
	if err != nil {
		logger.Error("failed-to-listen-for-aborts", err)
		return
	}

	defer notifier.Close()

	ctx, span := tracing.StartSpan(b.ctx, "build", tracing.Attrs{
		"team":     b.build.TeamName(),
		"pipeline": b.build.PipelineName(),
		"job":      b.build.JobName(),
		"build":    b.build.Name(),
		"build_id": strconv.Itoa(b.build.ID()),
	})
	defer span.End()

	step, err := b.builder.BuildStep(logger, b.build)
	if err != nil {
		logger.Error("failed-to-build-step", err)

		// Fails the build if BuildStep returned error. Because some unrecoverable error,
		// like pipeline var_source is wrong, will cause a build to never start
		// to run.
		b.builder.BuildStepErrored(logger, b.build, err)
		b.finish(logger.Session("finish"), err, false)

		return
	}
	b.trackStarted(logger)
	defer b.trackFinished(logger)

	logger.Info("running")

	state := b.runState()
	defer b.clearRunState()

	noleak := make(chan bool)
	defer close(noleak)

	go func() {
		select {
		case <-noleak:
		case <-notifier.Notify():
			logger.Info("aborting")
			b.cancel()
		}
	}()

	done := make(chan error)
	go func() {
		ctx = lagerctx.NewContext(ctx, logger)
		done <- step.Run(ctx, state)
	}()

	select {
	case <-b.release:
		logger.Info("releasing")

	case err = <-done:
		logger.Debug("engine-build-done")
		b.finish(logger.Session("finish"), err, step.Succeeded())
	}
}

func (b *engineBuild) finish(logger lager.Logger, err error, succeeded bool) {
	if err == context.Canceled {
		b.saveStatus(logger, atc.StatusAborted)
		logger.Info("aborted")

	} else if err != nil {
		b.saveStatus(logger, atc.StatusErrored)
		logger.Info("errored", lager.Data{"error": err.Error()})

	} else if succeeded {
		b.saveStatus(logger, atc.StatusSucceeded)
		logger.Info("succeeded")

	} else {
		b.saveStatus(logger, atc.StatusFailed)
		logger.Info("failed")
	}
}

func (b *engineBuild) saveStatus(logger lager.Logger, status atc.BuildStatus) {
	if err := b.build.Finish(db.BuildStatus(status)); err != nil {
		logger.Error("failed-to-finish-build", err)
	}
}

func (b *engineBuild) trackStarted(logger lager.Logger) {
	metric.BuildStarted{
		PipelineName: b.build.PipelineName(),
		JobName:      b.build.JobName(),
		BuildName:    b.build.Name(),
		BuildID:      b.build.ID(),
		TeamName:     b.build.TeamName(),
	}.Emit(logger)
}

func (b *engineBuild) trackFinished(logger lager.Logger) {
	found, err := b.build.Reload()
	if err != nil {
		logger.Error("failed-to-load-build-from-db", err)
		return
	}

	if !found {
		logger.Info("build-removed")
		return
	}

	if !b.build.IsRunning() {
		metric.BuildFinished{
			PipelineName:  b.build.PipelineName(),
			JobName:       b.build.JobName(),
			BuildName:     b.build.Name(),
			BuildID:       b.build.ID(),
			BuildStatus:   b.build.Status(),
			BuildDuration: b.build.EndTime().Sub(b.build.StartTime()),
			TeamName:      b.build.TeamName(),
		}.Emit(logger)
	}
}

func (b *engineBuild) runState() exec.RunState {
	id := fmt.Sprintf("build:%v", b.build.ID())
	existingState, _ := b.trackedStates.LoadOrStore(id, exec.NewRunState())
	return existingState.(exec.RunState)
}

func (b *engineBuild) clearRunState() {
	id := fmt.Sprintf("build:%v", b.build.ID())
	b.trackedStates.Delete(id)
}

func NewCheck(
	ctx context.Context,
	cancel func(),
	check db.Check,
	builder StepBuilder,
	release chan bool,
	trackedStates *sync.Map,
	waitGroup *sync.WaitGroup,
) Runnable {
	return &engineCheck{
		ctx:    ctx,
		cancel: cancel,

		check:   check,
		builder: builder,

		release:       release,
		trackedStates: trackedStates,
		waitGroup:     waitGroup,
	}
}

type engineCheck struct {
	ctx    context.Context
	cancel func()

	check   db.Check
	builder StepBuilder

	release       chan bool
	trackedStates *sync.Map
	waitGroup     *sync.WaitGroup
}

func (c *engineCheck) Run(logger lager.Logger) {
	c.waitGroup.Add(1)
	defer c.waitGroup.Done()

	logger = logger.WithData(lager.Data{
		"check": c.check.ID(),
	})

	lock, acquired, err := c.check.AcquireTrackingLock(logger)
	if err != nil {
		logger.Error("failed-to-get-lock", err)
		return
	}

	if !acquired {
		logger.Debug("check-already-tracked")
		return
	}

	defer lock.Release()

	err = c.check.Start()
	if err != nil {
		logger.Error("failed-to-start-check", err)
		return
	}

	c.trackStarted(logger)
	defer c.trackFinished(logger)

	step, err := c.builder.CheckStep(logger, c.check)
	if err != nil {
		logger.Error("failed-to-create-check-step", err)
		c.check.FinishWithError(fmt.Errorf("create check step: %w", err))
		return
	}

	logger.Info("running")

	state := c.runState()
	defer c.clearRunState()

	done := make(chan error)
	go func() {
		ctx := lagerctx.NewContext(c.ctx, logger)
		done <- step.Run(ctx, state)
	}()

	select {
	case <-c.release:
		logger.Info("releasing")

	case err = <-done:
		if err != nil {
			logger.Info("errored", lager.Data{"error": err.Error()})
			c.check.FinishWithError(fmt.Errorf("run check step: %w", err))
		} else {
			logger.Info("succeeded")
			if err = c.check.Finish(); err != nil {
				logger.Error("failed-to-finish-check", err)
			}
		}
	}
}

func (c *engineCheck) runState() exec.RunState {
	id := fmt.Sprintf("check:%v", c.check.ID())
	existingState, _ := c.trackedStates.LoadOrStore(id, exec.NewRunState())
	return existingState.(exec.RunState)
}

func (c *engineCheck) clearRunState() {
	id := fmt.Sprintf("check:%v", c.check.ID())
	c.trackedStates.Delete(id)
}

func (c *engineCheck) trackStarted(logger lager.Logger) {
	metric.CheckStarted{
		CheckName:             c.check.Plan().Check.Name,
		ResourceConfigScopeID: c.check.ResourceConfigScopeID(),
		CheckStatus:           c.check.Status(),
		CheckPendingDuration:  c.check.StartTime().Sub(c.check.CreateTime()),
	}.Emit(logger)
}

func (c *engineCheck) trackFinished(logger lager.Logger) {
	metric.CheckFinished{
		CheckName:             c.check.Plan().Check.Name,
		ResourceConfigScopeID: c.check.ResourceConfigScopeID(),
		CheckStatus:           c.check.Status(),
		CheckDuration:         c.check.EndTime().Sub(c.check.StartTime()),
	}.Emit(logger)
}
<|endoftext|>


<commit_before>package nakadi

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/pkg/errors"
)

// EventMetadata represents the meta information which comes along with all Nakadi events. For publishing
// purposes only the fields eid and occurred_at must be present.
type EventMetadata struct {
	EID        string      `json:"eid"`
	OccurredAt time.Time   `json:"occurred_at"`
	EventType  string      `json:"event_type,omitempty"`
	Partition  string      `json:"partition,omitempty"`
	ParentEIDs []string    `json:"parent_eids,omitempty"`
	FlowID     string      `json:"flow_id,omitempty"`
	ReceivedAt *time.Time  `json:"received_at,omitempty"`
	SpanCTX    interface{} `json:"span_ctx,omitempty"`
}

// UndefinedEvent can be embedded in structs representing Nakadi events from the event category "undefined".
type UndefinedEvent struct {
	Metadata EventMetadata `json:"metadata"`
}

// BusinessEvent represents a Nakadi events from the category "business".
//
// Deprecated: use a custom struct and embed UndefinedEvent instead.
type BusinessEvent struct {
	Metadata    EventMetadata `json:"metadata"`
	OrderNumber string        `json:"order_number"`
}

// DataChangeEvent is a Nakadi event from the event category "data".
type DataChangeEvent struct {
	Metadata EventMetadata `json:"metadata"`
	Data     interface{}   `json:"data"`
	DataOP   string        `json:"data_op"`
	DataType string        `json:"data_type"`
}

// PublishOptions is a set of optional parameters used to configure the PublishAPI.
type PublishOptions struct {
	// Whether or not publish methods retry when publishing fails. If set to true
	// InitialRetryInterval, MaxRetryInterval, and MaxElapsedTime have no effect
	// (default: false).
	Retry bool
	// The initial (minimal) retry interval used for the exponential backoff algorithm
	// when retry is enables.
	InitialRetryInterval time.Duration
	// MaxRetryInterval the maximum retry interval. Once the exponential backoff reaches
	// this value the retry intervals remain constant.
	MaxRetryInterval time.Duration
	// MaxElapsedTime is the maximum time spent on retries when publishing events. Once
	// this value was reached the exponential backoff is halted and the events will not be
	// published.
	MaxElapsedTime time.Duration
}

func (o *PublishOptions) withDefaults() *PublishOptions {
	var copyOptions PublishOptions
	if o != nil {
		copyOptions = *o
	}
	if copyOptions.InitialRetryInterval == 0 {
		copyOptions.InitialRetryInterval = defaultInitialRetryInterval
	}
	if copyOptions.MaxRetryInterval == 0 {
		copyOptions.MaxRetryInterval = defaultMaxRetryInterval
	}
	if copyOptions.MaxElapsedTime == 0 {
		copyOptions.MaxElapsedTime = defaultMaxElapsedTime
	}
	return &copyOptions
}

// NewPublishAPI creates a new instance of the PublishAPI which can be used to publish
// Nakadi events. As for all sub APIs of the `go-nakadi` package NewPublishAPI receives a
// configured Nakadi client. Furthermore the name of the event type must be provided.
// The last parameter is a struct containing only optional parameters. The options may be
// nil.
func NewPublishAPI(client *Client, eventType string, options *PublishOptions) *PublishAPI {
	options = options.withDefaults()

	return &PublishAPI{
		client:     client,
		publishURL: fmt.Sprintf("%s/event-types/%s/events", client.nakadiURL, eventType),
		backOffConf: backOffConfiguration{
			Retry:                options.Retry,
			InitialRetryInterval: options.InitialRetryInterval,
			MaxRetryInterval:     options.MaxRetryInterval,
			MaxElapsedTime:       options.MaxElapsedTime}}
}

// PublishAPI is a sub API for publishing Nakadi events. All publish methods emit events as a single batch. If
// a publish method returns an error, the caller should check whether the error is a BatchItemsError in order to
// verify which events of a batch have been published.
type PublishAPI struct {
	client      *Client
	publishURL  string
	backOffConf backOffConfiguration
}

// PublishDataChangeEvent emits a batch of data change events. Depending on the options used when creating
// the PublishAPI this method will retry to publish the events if the were not successfully published.
func (p *PublishAPI) PublishDataChangeEvent(events []DataChangeEvent) error {
	return p.Publish(events)
}

// PublishBusinessEvent emits a batch of business events. Depending on the options used when creating
// the PublishAPI this method will retry to publish the events if the were not successfully published.
//
// Deprecated: use Publish with a custom struct with embedded UndefinedEvent instead.
func (p *PublishAPI) PublishBusinessEvent(events []BusinessEvent) error {
	return p.Publish(events)
}

// Publish is used to emit a batch of undefined events. But can also be used to publish data change or
// business events. Depending on the options used when creating the PublishAPI this method will retry
// to publish the events if the were not successfully published.
func (p *PublishAPI) Publish(events interface{}) error {
	const errMsg = "unable to request event types"

	response, err := p.client.httpPOST(p.backOffConf.create(), p.publishURL, events, errMsg)
	if err != nil {
		return err
	}
	defer response.Body.Close()

	if response.StatusCode == http.StatusMultiStatus || response.StatusCode == http.StatusUnprocessableEntity {
		batchItemError := BatchItemsError{}
		err := json.NewDecoder(response.Body).Decode(&batchItemError)
		if err != nil {
			return errors.Wrapf(err, "%s: unable to decode response body", errMsg)
		}
		return batchItemError
	}

	if response.StatusCode != http.StatusOK {
		buffer, err := ioutil.ReadAll(response.Body)
		if err != nil {
			return errors.Wrapf(err, "%s: unable to read response body", errMsg)
		}
		return decodeResponseToError(buffer, "unable to request event types")
	}

	return nil
}

// BatchItemResponse if a batch is only published partially each batch item response contains information
// about whether a singe event was successfully published or not.
type BatchItemResponse struct {
	EID              string `json:"eid"`
	PublishingStatus string `json:"publishing_status"`
	Step             string `json:"step"`
	Detail           string `json:"detail"`
}

// BatchItemsError represents an error which contains information about the publishing status of each single
// event in a batch.
type BatchItemsError []BatchItemResponse

// Error implements the error interface for BatchItemsError.
func (err BatchItemsError) Error() string {
	if err == nil {
		return ""
	}
	return "one or many events may have not been published"
}

// Format implements fmt.Formatter for BatchItemsError
func (err BatchItemsError) Format(s fmt.State, verb rune) {
	if err == nil {
		io.WriteString(s, "nil")
		return
	}
	switch verb {
	case 'v':
		if s.Flag('+') {
			var messages []string
			for k, v := range err {
				messages = append(messages, fmt.Sprintf("[%d]: %+v", k, v))
			}

			builder := bytes.NewBuffer(make([]byte, 0))
			switch len(err) {
			case 0:
				builder.WriteString("an unknown error occurred while publishing event")
			case 1:
				builder.WriteString("an error occurred while publishing event: ")
			default:
				builder.WriteString("errors occurred while publishing events: ")
			}

			builder.WriteString(strings.Join(messages, ", "))
			io.WriteString(s, builder.String())

			return
		}
		fallthrough
	case 's', 'q':
		io.WriteString(s, err.Error())
	}
}
<commit_msg>Update publish.go<commit_after>package nakadi

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/pkg/errors"
)

// EventMetadata represents the meta information which comes along with all Nakadi events. For publishing
// purposes only the fields eid and occurred_at must be present.
type EventMetadata struct {
	EID        string      `json:"eid"`
	OccurredAt time.Time   `json:"occurred_at"`
	EventType  string      `json:"event_type,omitempty"`
	Partition  string      `json:"partition,omitempty"`
	ParentEIDs []string    `json:"parent_eids,omitempty"`
	FlowID     string      `json:"flow_id,omitempty"`
	ReceivedAt *time.Time  `json:"received_at,omitempty"`
	SpanCtx    interface{} `json:"span_ctx,omitempty"`
}

// UndefinedEvent can be embedded in structs representing Nakadi events from the event category "undefined".
type UndefinedEvent struct {
	Metadata EventMetadata `json:"metadata"`
}

// BusinessEvent represents a Nakadi events from the category "business".
//
// Deprecated: use a custom struct and embed UndefinedEvent instead.
type BusinessEvent struct {
	Metadata    EventMetadata `json:"metadata"`
	OrderNumber string        `json:"order_number"`
}

// DataChangeEvent is a Nakadi event from the event category "data".
type DataChangeEvent struct {
	Metadata EventMetadata `json:"metadata"`
	Data     interface{}   `json:"data"`
	DataOP   string        `json:"data_op"`
	DataType string        `json:"data_type"`
}

// PublishOptions is a set of optional parameters used to configure the PublishAPI.
type PublishOptions struct {
	// Whether or not publish methods retry when publishing fails. If set to true
	// InitialRetryInterval, MaxRetryInterval, and MaxElapsedTime have no effect
	// (default: false).
	Retry bool
	// The initial (minimal) retry interval used for the exponential backoff algorithm
	// when retry is enables.
	InitialRetryInterval time.Duration
	// MaxRetryInterval the maximum retry interval. Once the exponential backoff reaches
	// this value the retry intervals remain constant.
	MaxRetryInterval time.Duration
	// MaxElapsedTime is the maximum time spent on retries when publishing events. Once
	// this value was reached the exponential backoff is halted and the events will not be
	// published.
	MaxElapsedTime time.Duration
}

func (o *PublishOptions) withDefaults() *PublishOptions {
	var copyOptions PublishOptions
	if o != nil {
		copyOptions = *o
	}
	if copyOptions.InitialRetryInterval == 0 {
		copyOptions.InitialRetryInterval = defaultInitialRetryInterval
	}
	if copyOptions.MaxRetryInterval == 0 {
		copyOptions.MaxRetryInterval = defaultMaxRetryInterval
	}
	if copyOptions.MaxElapsedTime == 0 {
		copyOptions.MaxElapsedTime = defaultMaxElapsedTime
	}
	return &copyOptions
}

// NewPublishAPI creates a new instance of the PublishAPI which can be used to publish
// Nakadi events. As for all sub APIs of the `go-nakadi` package NewPublishAPI receives a
// configured Nakadi client. Furthermore the name of the event type must be provided.
// The last parameter is a struct containing only optional parameters. The options may be
// nil.
func NewPublishAPI(client *Client, eventType string, options *PublishOptions) *PublishAPI {
	options = options.withDefaults()

	return &PublishAPI{
		client:     client,
		publishURL: fmt.Sprintf("%s/event-types/%s/events", client.nakadiURL, eventType),
		backOffConf: backOffConfiguration{
			Retry:                options.Retry,
			InitialRetryInterval: options.InitialRetryInterval,
			MaxRetryInterval:     options.MaxRetryInterval,
			MaxElapsedTime:       options.MaxElapsedTime}}
}

// PublishAPI is a sub API for publishing Nakadi events. All publish methods emit events as a single batch. If
// a publish method returns an error, the caller should check whether the error is a BatchItemsError in order to
// verify which events of a batch have been published.
type PublishAPI struct {
	client      *Client
	publishURL  string
	backOffConf backOffConfiguration
}

// PublishDataChangeEvent emits a batch of data change events. Depending on the options used when creating
// the PublishAPI this method will retry to publish the events if the were not successfully published.
func (p *PublishAPI) PublishDataChangeEvent(events []DataChangeEvent) error {
	return p.Publish(events)
}

// PublishBusinessEvent emits a batch of business events. Depending on the options used when creating
// the PublishAPI this method will retry to publish the events if the were not successfully published.
//
// Deprecated: use Publish with a custom struct with embedded UndefinedEvent instead.
func (p *PublishAPI) PublishBusinessEvent(events []BusinessEvent) error {
	return p.Publish(events)
}

// Publish is used to emit a batch of undefined events. But can also be used to publish data change or
// business events. Depending on the options used when creating the PublishAPI this method will retry
// to publish the events if the were not successfully published.
func (p *PublishAPI) Publish(events interface{}) error {
	const errMsg = "unable to request event types"

	response, err := p.client.httpPOST(p.backOffConf.create(), p.publishURL, events, errMsg)
	if err != nil {
		return err
	}
	defer response.Body.Close()

	if response.StatusCode == http.StatusMultiStatus || response.StatusCode == http.StatusUnprocessableEntity {
		batchItemError := BatchItemsError{}
		err := json.NewDecoder(response.Body).Decode(&batchItemError)
		if err != nil {
			return errors.Wrapf(err, "%s: unable to decode response body", errMsg)
		}
		return batchItemError
	}

	if response.StatusCode != http.StatusOK {
		buffer, err := ioutil.ReadAll(response.Body)
		if err != nil {
			return errors.Wrapf(err, "%s: unable to read response body", errMsg)
		}
		return decodeResponseToError(buffer, "unable to request event types")
	}

	return nil
}

// BatchItemResponse if a batch is only published partially each batch item response contains information
// about whether a singe event was successfully published or not.
type BatchItemResponse struct {
	EID              string `json:"eid"`
	PublishingStatus string `json:"publishing_status"`
	Step             string `json:"step"`
	Detail           string `json:"detail"`
}

// BatchItemsError represents an error which contains information about the publishing status of each single
// event in a batch.
type BatchItemsError []BatchItemResponse

// Error implements the error interface for BatchItemsError.
func (err BatchItemsError) Error() string {
	if err == nil {
		return ""
	}
	return "one or many events may have not been published"
}

// Format implements fmt.Formatter for BatchItemsError
func (err BatchItemsError) Format(s fmt.State, verb rune) {
	if err == nil {
		io.WriteString(s, "nil")
		return
	}
	switch verb {
	case 'v':
		if s.Flag('+') {
			var messages []string
			for k, v := range err {
				messages = append(messages, fmt.Sprintf("[%d]: %+v", k, v))
			}

			builder := bytes.NewBuffer(make([]byte, 0))
			switch len(err) {
			case 0:
				builder.WriteString("an unknown error occurred while publishing event")
			case 1:
				builder.WriteString("an error occurred while publishing event: ")
			default:
				builder.WriteString("errors occurred while publishing events: ")
			}

			builder.WriteString(strings.Join(messages, ", "))
			io.WriteString(s, builder.String())

			return
		}
		fallthrough
	case 's', 'q':
		io.WriteString(s, err.Error())
	}
}
<|endoftext|>


<commit_before>// List objects

package py

var ListType = NewType("list", "list() -> new empty list\nlist(iterable) -> new list initialized from iterable's items")

type List []Object

// Type of this List object
func (o List) Type() *Type {
	return ListType
}

// Copy a list object
func (l List) Copy() List {
	newL := make(List, len(l))
	for i := range l {
		newL[i] = l[i]
	}
	return newL
}

func (t List) M__len__() Object {
	return Int(len(t))
}

func (t List) M__bool__() Object {
	if len(t) > 0 {
		return True
	}
	return False
}

func (t List) M__iter__() Object {
	return NewIterator(t)
}

func (t List) M__getitem__(key Object) Object {
	i := IndexIntCheck(key, len(t))
	return t[i]
}

func (t List) M__setitem__(key, value Object) Object {
	i := IndexIntCheck(key, len(t))
	t[i] = value
	return None
}

// Check interface is satisfied
var _ I__len__ = List(nil)
var _ I__bool__ = List(nil)
var _ I__iter__ = List(nil)
var _ I__getitem__ = List(nil)
var _ I__setitem__ = List(nil)

// var _ richComparison = List(nil)
<commit_msg>Notes on list implementation<commit_after>// List objects

package py

var ListType = NewType("list", "list() -> new empty list\nlist(iterable) -> new list initialized from iterable's items")

// FIXME lists are mutable so this should probably be struct { Tuple } then can use the sub methods on Tuple
type List []Object

// Type of this List object
func (o List) Type() *Type {
	return ListType
}

// Copy a list object
func (l List) Copy() List {
	newL := make(List, len(l))
	for i := range l {
		newL[i] = l[i]
	}
	return newL
}

func (t List) M__len__() Object {
	return Int(len(t))
}

func (t List) M__bool__() Object {
	if len(t) > 0 {
		return True
	}
	return False
}

func (t List) M__iter__() Object {
	return NewIterator(t)
}

func (t List) M__getitem__(key Object) Object {
	i := IndexIntCheck(key, len(t))
	return t[i]
}

func (t List) M__setitem__(key, value Object) Object {
	i := IndexIntCheck(key, len(t))
	t[i] = value
	return None
}

// Check interface is satisfied
var _ I__len__ = List(nil)
var _ I__bool__ = List(nil)
var _ I__iter__ = List(nil)
var _ I__getitem__ = List(nil)
var _ I__setitem__ = List(nil)

// var _ richComparison = List(nil)
<|endoftext|>


<commit_before>package controller

import (
	"strconv"
	"strings"

	"github.com/tidwall/tile38/controller/server"
	"github.com/tidwall/tile38/geojson"
)

var tmfmt = "2006-01-02T15:04:05.999999999Z07:00"

// FenceMatch executes a fence match returns back json messages for fence detection.
func FenceMatch(hookName string, sw *scanWriter, fence *liveFenceSwitches, details *commandDetailsT) []string {
	jshookName := jsonString(hookName)
	jstime := jsonString(details.timestamp.Format(tmfmt))
	glob := fence.glob
	if details.command == "drop" {
		return []string{`{"cmd":"drop","hook":` + jshookName + `,"time":` + jstime + `}`}
	}
	match := true
	if glob != "" && glob != "*" {
		match, _ = globMatch(glob, details.id)
	}
	if !match {
		return nil
	}

	sw.mu.Lock()
	nofields := sw.nofields
	sw.mu.Unlock()

	if details.obj == nil || (details.command == "fset" && nofields) {
		return nil
	}
	match = false

	var roamkeys, roamids []string
	var roammeters []float64
	detect := "outside"
	if fence != nil {
		if fence.roam.on {
			if details.command == "set" {
				// println("roam", fence.roam.key, fence.roam.id, strconv.FormatFloat(fence.roam.meters, 'f', -1, 64))
				roamkeys, roamids, roammeters = fenceMatchRoam(sw.c, fence, details.key, details.id, details.obj)
			}
			if len(roamids) == 0 || len(roamids) != len(roamkeys) {
				return nil
			}
			match = true
			detect = "roam"
		} else {

			// not using roaming
			match1 := fenceMatchObject(fence, details.oldObj)
			match2 := fenceMatchObject(fence, details.obj)
			if match1 && match2 {
				match = true
				detect = "inside"
			} else if match1 && !match2 {
				match = true
				detect = "exit"
			} else if !match1 && match2 {
				match = true
				detect = "enter"
				if details.command == "fset" {
					detect = "inside"
				}
			} else {
				if details.command != "fset" {
					// Maybe the old object and new object create a line that crosses the fence.
					// Must detect for that possibility.
					if details.oldObj != nil {
						ls := geojson.LineString{
							Coordinates: []geojson.Position{
								details.oldObj.CalculatedPoint(),
								details.obj.CalculatedPoint(),
							},
						}
						temp := false
						if fence.cmd == "within" {
							// because we are testing if the line croses the area we need to use
							// "intersects" instead of "within".
							fence.cmd = "intersects"
							temp = true
						}
						if fenceMatchObject(fence, ls) {
							//match = true
							detect = "cross"
						}
						if temp {
							fence.cmd = "within"
						}
					}
				}
			}
		}
	}
	if details.command == "del" {
		return []string{`{"command":"del","hook":` + jshookName + `,"id":` + jsonString(details.id) + `,"time":` + jstime + `}`}
	}
	if details.fmap == nil {
		return nil
	}
	sw.mu.Lock()
	sw.fmap = details.fmap
	sw.fullFields = true
	sw.msg.OutputType = server.JSON
	sw.writeObject(details.id, details.obj, details.fields, true)
	if sw.wr.Len() == 0 {
		sw.mu.Unlock()
		return nil
	}

	res := sw.wr.String()
	resb := make([]byte, len(res))
	copy(resb, res)
	sw.wr.Reset()
	res = string(resb)
	if sw.output == outputIDs {
		res = `{"id":` + res + `}`
	}
	sw.mu.Unlock()
	if strings.HasPrefix(res, ",") {
		res = res[1:]
	}

	jskey := jsonString(details.key)

	ores := res
	msgs := make([]string, 0, 2)
	if fence.detect == nil || fence.detect[detect] {
		if strings.HasPrefix(ores, "{") {
			res = `{"command":"` + details.command + `","detect":"` + detect + `","hook":` + jshookName + `,"key":` + jskey + `,"time":` + jstime + `,` + ores[1:]
		}
		msgs = append(msgs, res)
	}
	switch detect {
	case "enter":
		if fence.detect == nil || fence.detect["inside"] {
			msgs = append(msgs, `{"command":"`+details.command+`","detect":"inside","hook":`+jshookName+`,"key":`+jskey+`,"time":`+jstime+`,`+ores[1:])
		}
	case "exit", "cross":
		if fence.detect == nil || fence.detect["outside"] {
			msgs = append(msgs, `{"command":"`+details.command+`","detect":"outside","hook":`+jshookName+`,"key":`+jskey+`,"time":`+jstime+`,`+ores[1:])
		}
	case "roam":
		if len(msgs) > 0 {
			var nmsgs []string
			msg := msgs[0][:len(msgs[0])-1]
			for i, id := range roamids {
				nmsgs = append(nmsgs, msg+`,"nearby":{"key":`+jsonString(roamkeys[i])+`,"id":`+jsonString(id)+`,"meters":`+strconv.FormatFloat(roammeters[i], 'f', -1, 64)+`}}`)
			}
			msgs = nmsgs
		}
	}
	return msgs
}

func fenceMatchObject(fence *liveFenceSwitches, obj geojson.Object) bool {
	if obj == nil {
		return false
	}
	if fence.roam.on {
		// we need to check this object against
		return false
	}
	if fence.cmd == "nearby" {
		return obj.Nearby(geojson.Position{X: fence.lon, Y: fence.lat, Z: 0}, fence.meters)
	} else if fence.cmd == "within" {
		if fence.o != nil {
			return obj.Within(fence.o)
		}
		return obj.WithinBBox(geojson.BBox{
			Min: geojson.Position{X: fence.minLon, Y: fence.minLat, Z: 0},
			Max: geojson.Position{X: fence.maxLon, Y: fence.maxLat, Z: 0},
		})
	} else if fence.cmd == "intersects" {
		if fence.o != nil {
			return obj.Intersects(fence.o)
		}
		return obj.IntersectsBBox(geojson.BBox{
			Min: geojson.Position{X: fence.minLon, Y: fence.minLat, Z: 0},
			Max: geojson.Position{X: fence.maxLon, Y: fence.maxLat, Z: 0},
		})
	}
	return false
}

func fenceMatchRoam(c *Controller, fence *liveFenceSwitches, tkey, tid string, obj geojson.Object) (keys, ids []string, meterss []float64) {
	c.mu.RLock()
	defer c.mu.RUnlock()
	col := c.getCol(fence.roam.key)
	if col == nil {
		return
	}
	p := obj.CalculatedPoint()
	col.Nearby(0, 0, p.Y, p.X, fence.roam.meters,
		func(id string, obj geojson.Object, fields []float64) bool {
			var match bool
			if id == tid {
				return true // skip self
			}
			if fence.roam.pattern {
				match, _ = globMatch(fence.roam.id, id)
			} else {
				match = fence.roam.id == id
			}
			if match {
				keys = append(keys, fence.roam.key)
				ids = append(ids, id)
				meterss = append(meterss, obj.CalculatedPoint().DistanceTo(p))
			}
			return true
		},
	)
	return
}
<commit_msg>fixed json syntax error<commit_after>package controller

import (
	"strconv"
	"strings"

	"github.com/tidwall/tile38/controller/server"
	"github.com/tidwall/tile38/geojson"
)

var tmfmt = "2006-01-02T15:04:05.999999999Z07:00"

// FenceMatch executes a fence match returns back json messages for fence detection.
func FenceMatch(hookName string, sw *scanWriter, fence *liveFenceSwitches, details *commandDetailsT) []string {
	jshookName := jsonString(hookName)
	jstime := jsonString(details.timestamp.Format(tmfmt))
	glob := fence.glob
	if details.command == "drop" {
		return []string{`{"cmd":"drop","hook":` + jshookName + `,"time":` + jstime + `}`}
	}
	match := true
	if glob != "" && glob != "*" {
		match, _ = globMatch(glob, details.id)
	}
	if !match {
		return nil
	}

	sw.mu.Lock()
	nofields := sw.nofields
	sw.mu.Unlock()

	if details.obj == nil || (details.command == "fset" && nofields) {
		return nil
	}
	match = false

	var roamkeys, roamids []string
	var roammeters []float64
	detect := "outside"
	if fence != nil {
		if fence.roam.on {
			if details.command == "set" {
				// println("roam", fence.roam.key, fence.roam.id, strconv.FormatFloat(fence.roam.meters, 'f', -1, 64))
				roamkeys, roamids, roammeters = fenceMatchRoam(sw.c, fence, details.key, details.id, details.obj)
			}
			if len(roamids) == 0 || len(roamids) != len(roamkeys) {
				return nil
			}
			match = true
			detect = "roam"
		} else {

			// not using roaming
			match1 := fenceMatchObject(fence, details.oldObj)
			match2 := fenceMatchObject(fence, details.obj)
			if match1 && match2 {
				match = true
				detect = "inside"
			} else if match1 && !match2 {
				match = true
				detect = "exit"
			} else if !match1 && match2 {
				match = true
				detect = "enter"
				if details.command == "fset" {
					detect = "inside"
				}
			} else {
				if details.command != "fset" {
					// Maybe the old object and new object create a line that crosses the fence.
					// Must detect for that possibility.
					if details.oldObj != nil {
						ls := geojson.LineString{
							Coordinates: []geojson.Position{
								details.oldObj.CalculatedPoint(),
								details.obj.CalculatedPoint(),
							},
						}
						temp := false
						if fence.cmd == "within" {
							// because we are testing if the line croses the area we need to use
							// "intersects" instead of "within".
							fence.cmd = "intersects"
							temp = true
						}
						if fenceMatchObject(fence, ls) {
							//match = true
							detect = "cross"
						}
						if temp {
							fence.cmd = "within"
						}
					}
				}
			}
		}
	}
	if details.command == "del" {
		return []string{`{"command":"del","hook":` + jshookName + `,"id":` + jsonString(details.id) + `,"time":` + jstime + `}`}
	}
	if details.fmap == nil {
		return nil
	}
	sw.mu.Lock()
	sw.fmap = details.fmap
	sw.fullFields = true
	sw.msg.OutputType = server.JSON
	sw.writeObject(details.id, details.obj, details.fields, true)
	if sw.wr.Len() == 0 {
		sw.mu.Unlock()
		return nil
	}

	res := sw.wr.String()
	resb := make([]byte, len(res))
	copy(resb, res)
	sw.wr.Reset()
	res = string(resb)
	if strings.HasPrefix(res, ",") {
		res = res[1:]
	}
	if sw.output == outputIDs {
		res = `{"id":` + res + `}`
	}
	sw.mu.Unlock()

	jskey := jsonString(details.key)

	ores := res
	msgs := make([]string, 0, 2)
	if fence.detect == nil || fence.detect[detect] {
		if strings.HasPrefix(ores, "{") {
			res = `{"command":"` + details.command + `","detect":"` + detect + `","hook":` + jshookName + `,"key":` + jskey + `,"time":` + jstime + `,` + ores[1:]
		}
		msgs = append(msgs, res)
	}
	switch detect {
	case "enter":
		if fence.detect == nil || fence.detect["inside"] {
			msgs = append(msgs, `{"command":"`+details.command+`","detect":"inside","hook":`+jshookName+`,"key":`+jskey+`,"time":`+jstime+`,`+ores[1:])
		}
	case "exit", "cross":
		if fence.detect == nil || fence.detect["outside"] {
			msgs = append(msgs, `{"command":"`+details.command+`","detect":"outside","hook":`+jshookName+`,"key":`+jskey+`,"time":`+jstime+`,`+ores[1:])
		}
	case "roam":
		if len(msgs) > 0 {
			var nmsgs []string
			msg := msgs[0][:len(msgs[0])-1]
			for i, id := range roamids {
				nmsgs = append(nmsgs, msg+`,"nearby":{"key":`+jsonString(roamkeys[i])+`,"id":`+jsonString(id)+`,"meters":`+strconv.FormatFloat(roammeters[i], 'f', -1, 64)+`}}`)
			}
			msgs = nmsgs
		}
	}
	return msgs
}

func fenceMatchObject(fence *liveFenceSwitches, obj geojson.Object) bool {
	if obj == nil {
		return false
	}
	if fence.roam.on {
		// we need to check this object against
		return false
	}
	if fence.cmd == "nearby" {
		return obj.Nearby(geojson.Position{X: fence.lon, Y: fence.lat, Z: 0}, fence.meters)
	} else if fence.cmd == "within" {
		if fence.o != nil {
			return obj.Within(fence.o)
		}
		return obj.WithinBBox(geojson.BBox{
			Min: geojson.Position{X: fence.minLon, Y: fence.minLat, Z: 0},
			Max: geojson.Position{X: fence.maxLon, Y: fence.maxLat, Z: 0},
		})
	} else if fence.cmd == "intersects" {
		if fence.o != nil {
			return obj.Intersects(fence.o)
		}
		return obj.IntersectsBBox(geojson.BBox{
			Min: geojson.Position{X: fence.minLon, Y: fence.minLat, Z: 0},
			Max: geojson.Position{X: fence.maxLon, Y: fence.maxLat, Z: 0},
		})
	}
	return false
}

func fenceMatchRoam(c *Controller, fence *liveFenceSwitches, tkey, tid string, obj geojson.Object) (keys, ids []string, meterss []float64) {
	c.mu.RLock()
	defer c.mu.RUnlock()
	col := c.getCol(fence.roam.key)
	if col == nil {
		return
	}
	p := obj.CalculatedPoint()
	col.Nearby(0, 0, p.Y, p.X, fence.roam.meters,
		func(id string, obj geojson.Object, fields []float64) bool {
			var match bool
			if id == tid {
				return true // skip self
			}
			if fence.roam.pattern {
				match, _ = globMatch(fence.roam.id, id)
			} else {
				match = fence.roam.id == id
			}
			if match {
				keys = append(keys, fence.roam.key)
				ids = append(ids, id)
				meterss = append(meterss, obj.CalculatedPoint().DistanceTo(p))
			}
			return true
		},
	)
	return
}
<|endoftext|>


<commit_before>package auth

import (
	"crypto/x509"
	"fmt"
	"reflect"
	"testing"
)

func TestGeneratePrivateKey(t *testing.T) {
	_, err := GeneratePrivateKey(1024)

	if err != nil {
		t.Fatal(err)
	}

	if !testing.Short() {
		GeneratePrivateKey(4096)
	}
}

func TestPrivateKeyToPEM(t *testing.T) {
	key, _ := GeneratePrivateKey(512)
	res := PrivateKeyToPEM(key)

	if res[0] != '-' {
		t.Fatalf("Bad format\n%s", res)
	}

	if IsPEMEncrypted(res) {
		t.Fatal("Result is encrypted")
	}
}

func TestPrivateKeyToEncryptedPEM(t *testing.T) {
	key, _ := GeneratePrivateKey(512)
	res, err := PrivateKeyToEncryptedPEM(key, "password")

	if res[0] != '-' {
		t.Fatalf("Bad format\n%s", res)
	}

	if !IsPEMEncrypted(res) || err != nil {
		t.Fatal("Result is not encrypted: ", err)
	}
}

func TestPEMToPrivateKey(t *testing.T) {
	key, _ := GeneratePrivateKey(512)
	key2, err := PEMToPrivateKey(PrivateKeyToPEM(key))
	if !reflect.DeepEqual(key, key2) || err != nil {
		t.Fatal(err)
	}
}

func TestEncryptedPEMToPrivateKey(t *testing.T) {
	key, _ := GeneratePrivateKey(512)
	res, _ := PrivateKeyToEncryptedPEM(key, "password")

	goodKey, err := EncryptedPEMToPrivateKey(res, "password")

	if !reflect.DeepEqual(key, goodKey) || err != nil {
		t.Fatal(err)
	}

	badKey, err := EncryptedPEMToPrivateKey(res, "badpass")

	if badKey != nil || err != x509.IncorrectPasswordError {
		t.Fatal(err)
	}
}

func ExampleEncryptedPEMToPrivateKey() {

	// Generate a new private key for example
	key, err := GeneratePrivateKey(512)

	if err != nil {
		panic(err)
	}

	// Get the encrypted PEM data

	p, err := PrivateKeyToEncryptedPEM(key, "myPassword")

	if err != nil {
		panic(err)
	}

	// Reverse the process

	newKey, err := EncryptedPEMToPrivateKey(p, "badPassword")

	if err == x509.IncorrectPasswordError {
		fmt.Println("Bad password")
	}

	newKey, err = EncryptedPEMToPrivateKey(p, "myPassword")

	if newKey != nil && err == nil {
		fmt.Println("OK")
	}

	// Output:
	// Bad password
	// OK
}
<commit_msg>[auth] Lint test<commit_after>package auth

import (
	"crypto/x509"
	"fmt"
	"reflect"
	"testing"
)

func TestGeneratePrivateKey(t *testing.T) {
	_, err := GeneratePrivateKey(1024)

	if err != nil {
		t.Fatal(err)
	}

	if !testing.Short() {
		_, err = GeneratePrivateKey(4096)
		if err != nil {
			t.Fatal(err)
		}
	}
}

func TestPrivateKeyToPEM(t *testing.T) {
	key, _ := GeneratePrivateKey(512)
	res := PrivateKeyToPEM(key)

	if res[0] != '-' {
		t.Fatalf("Bad format\n%s", res)
	}

	if IsPEMEncrypted(res) {
		t.Fatal("Result is encrypted")
	}
}

func TestPrivateKeyToEncryptedPEM(t *testing.T) {
	key, _ := GeneratePrivateKey(512)
	res, err := PrivateKeyToEncryptedPEM(key, "password")

	if res[0] != '-' {
		t.Fatalf("Bad format\n%s", res)
	}

	if !IsPEMEncrypted(res) || err != nil {
		t.Fatal("Result is not encrypted: ", err)
	}
}

func TestPEMToPrivateKey(t *testing.T) {
	key, _ := GeneratePrivateKey(512)
	key2, err := PEMToPrivateKey(PrivateKeyToPEM(key))
	if !reflect.DeepEqual(key, key2) || err != nil {
		t.Fatal(err)
	}
}

func TestEncryptedPEMToPrivateKey(t *testing.T) {
	key, _ := GeneratePrivateKey(512)
	res, _ := PrivateKeyToEncryptedPEM(key, "password")

	goodKey, err := EncryptedPEMToPrivateKey(res, "password")

	if !reflect.DeepEqual(key, goodKey) || err != nil {
		t.Fatal(err)
	}

	badKey, err := EncryptedPEMToPrivateKey(res, "badpass")

	if badKey != nil || err != x509.IncorrectPasswordError {
		t.Fatal(err)
	}
}

func ExampleEncryptedPEMToPrivateKey() {

	// Generate a new private key for example
	key, err := GeneratePrivateKey(512)

	if err != nil {
		panic(err)
	}

	// Get the encrypted PEM data

	p, err := PrivateKeyToEncryptedPEM(key, "myPassword")

	if err != nil {
		panic(err)
	}

	// Reverse the process

	newKey, err := EncryptedPEMToPrivateKey(p, "badPassword")

	if err == x509.IncorrectPasswordError {
		fmt.Println("Bad password")
	}

	newKey, err = EncryptedPEMToPrivateKey(p, "myPassword")

	if newKey != nil && err == nil {
		fmt.Println("OK")
	}

	// Output:
	// Bad password
	// OK
}
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package os

import "time"

// FindProcess looks for a running process by its pid.
// The Process it returns can be used to obtain information
// about the underlying operating system process.
func FindProcess(pid int) (p *Process, err error) {
	return findProcess(pid)
}

// StartProcess starts a new process with the program, arguments and attributes
// specified by name, argv and attr.
//
// StartProcess is a low-level interface. The os/exec package provides
// higher-level interfaces.
//
// If there is an error, it will be of type *PathError.
func StartProcess(name string, argv []string, attr *ProcAttr) (*Process, error) {
	return startProcess(name, argv, attr)
}

// Release releases any resources associated with the Process p,
// rendering it unusable in the future.
// Release only needs to be called if Wait is not.
func (p *Process) Release() error {
	return p.release()
}

// Kill causes the Process to exit immediately.
func (p *Process) Kill() error {
	return p.kill()
}

// Wait waits for the Process to exit, and then returns a
// ProcessState describing its status and an error, if any.
// Wait releases any resources associated with the Process.
func (p *Process) Wait() (*ProcessState, error) {
	return p.wait()
}

// Signal sends a signal to the Process.
func (p *Process) Signal(sig Signal) error {
	return p.signal(sig)
}

// UserTime returns the user CPU time of the exited process and its children.
func (p *ProcessState) UserTime() time.Duration {
	return p.userTime()
}

// SystemTime returns the system CPU time of the exited process and its children.
func (p *ProcessState) SystemTime() time.Duration {
	return p.systemTime()
}

// Exited returns whether the program has exited.
func (p *ProcessState) Exited() bool {
	return p.exited()
}

// Success reports whether the program exited successfully,
// such as with exit status 0 on Unix.
func (p *ProcessState) Success() bool {
	return p.success()
}

// Sys returns system-dependent exit information about
// the process.  Convert it to the appropriate underlying
// type, such as syscall.WaitStatus on Unix, to access its contents.
func (p *ProcessState) Sys() interface{} {
	return p.sys()
}

// SysUsage returns system-dependent resource usage information about
// the exited process.  Convert it to the appropriate underlying
// type, such as *syscall.Rusage on Unix, to access its contents.
func (p *ProcessState) SysUsage() interface{} {
	return p.sysUsage()
}

// Hostname returns the host name reported by the kernel.
func Hostname() (name string, err error) {
	return hostname()
}

// Readdir reads the contents of the directory associated with file and
// returns a slice of up to n FileInfo values, as would be returned
// by Lstat, in directory order. Subsequent calls on the same file will yield
// further FileInfos.
//
// If n > 0, Readdir returns at most n FileInfo structures. In this case, if
// Readdir returns an empty slice, it will return a non-nil error
// explaining why. At the end of a directory, the error is io.EOF.
//
// If n <= 0, Readdir returns all the FileInfo from the directory in
// a single slice. In this case, if Readdir succeeds (reads all
// the way to the end of the directory), it returns the slice and a
// nil error. If it encounters an error before the end of the
// directory, Readdir returns the FileInfo read until that point
// and a non-nil error.
func (f *File) Readdir(n int) (fi []FileInfo, err error) {
	return f.readdir(n)
}

// Readdirnames reads and returns a slice of names from the directory f.
//
// If n > 0, Readdirnames returns at most n names. In this case, if
// Readdirnames returns an empty slice, it will return a non-nil error
// explaining why. At the end of a directory, the error is io.EOF.
//
// If n <= 0, Readdirnames returns all the names from the directory in
// a single slice. In this case, if Readdirnames succeeds (reads all
// the way to the end of the directory), it returns the slice and a
// nil error. If it encounters an error before the end of the
// directory, Readdirnames returns the names read until that point and
// a non-nil error.
func (f *File) Readdirnames(n int) (names []string, err error) {
	return f.readdirnames(n)
}
<commit_msg>os: point users of ProcessState.SysUsage to getrusage(2)<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package os

import "time"

// FindProcess looks for a running process by its pid.
// The Process it returns can be used to obtain information
// about the underlying operating system process.
func FindProcess(pid int) (p *Process, err error) {
	return findProcess(pid)
}

// StartProcess starts a new process with the program, arguments and attributes
// specified by name, argv and attr.
//
// StartProcess is a low-level interface. The os/exec package provides
// higher-level interfaces.
//
// If there is an error, it will be of type *PathError.
func StartProcess(name string, argv []string, attr *ProcAttr) (*Process, error) {
	return startProcess(name, argv, attr)
}

// Release releases any resources associated with the Process p,
// rendering it unusable in the future.
// Release only needs to be called if Wait is not.
func (p *Process) Release() error {
	return p.release()
}

// Kill causes the Process to exit immediately.
func (p *Process) Kill() error {
	return p.kill()
}

// Wait waits for the Process to exit, and then returns a
// ProcessState describing its status and an error, if any.
// Wait releases any resources associated with the Process.
func (p *Process) Wait() (*ProcessState, error) {
	return p.wait()
}

// Signal sends a signal to the Process.
func (p *Process) Signal(sig Signal) error {
	return p.signal(sig)
}

// UserTime returns the user CPU time of the exited process and its children.
func (p *ProcessState) UserTime() time.Duration {
	return p.userTime()
}

// SystemTime returns the system CPU time of the exited process and its children.
func (p *ProcessState) SystemTime() time.Duration {
	return p.systemTime()
}

// Exited returns whether the program has exited.
func (p *ProcessState) Exited() bool {
	return p.exited()
}

// Success reports whether the program exited successfully,
// such as with exit status 0 on Unix.
func (p *ProcessState) Success() bool {
	return p.success()
}

// Sys returns system-dependent exit information about
// the process.  Convert it to the appropriate underlying
// type, such as syscall.WaitStatus on Unix, to access its contents.
func (p *ProcessState) Sys() interface{} {
	return p.sys()
}

// SysUsage returns system-dependent resource usage information about
// the exited process.  Convert it to the appropriate underlying
// type, such as *syscall.Rusage on Unix, to access its contents.
// (On Unix, *syscall.Rusage matches struct rusage as defined in the
// getrusage(2) manual page.)
func (p *ProcessState) SysUsage() interface{} {
	return p.sysUsage()
}

// Hostname returns the host name reported by the kernel.
func Hostname() (name string, err error) {
	return hostname()
}

// Readdir reads the contents of the directory associated with file and
// returns a slice of up to n FileInfo values, as would be returned
// by Lstat, in directory order. Subsequent calls on the same file will yield
// further FileInfos.
//
// If n > 0, Readdir returns at most n FileInfo structures. In this case, if
// Readdir returns an empty slice, it will return a non-nil error
// explaining why. At the end of a directory, the error is io.EOF.
//
// If n <= 0, Readdir returns all the FileInfo from the directory in
// a single slice. In this case, if Readdir succeeds (reads all
// the way to the end of the directory), it returns the slice and a
// nil error. If it encounters an error before the end of the
// directory, Readdir returns the FileInfo read until that point
// and a non-nil error.
func (f *File) Readdir(n int) (fi []FileInfo, err error) {
	return f.readdir(n)
}

// Readdirnames reads and returns a slice of names from the directory f.
//
// If n > 0, Readdirnames returns at most n names. In this case, if
// Readdirnames returns an empty slice, it will return a non-nil error
// explaining why. At the end of a directory, the error is io.EOF.
//
// If n <= 0, Readdirnames returns all the names from the directory in
// a single slice. In this case, if Readdirnames succeeds (reads all
// the way to the end of the directory), it returns the slice and a
// nil error. If it encounters an error before the end of the
// directory, Readdirnames returns the names read until that point and
// a non-nil error.
func (f *File) Readdirnames(n int) (names []string, err error) {
	return f.readdirnames(n)
}
<|endoftext|>


<commit_before>package quintus

import (
	"bytes"
	"encoding/gob"
	"io/ioutil"
	"log"
)

type Quintus struct {
	Name  string
	Date  string
	Times int
}

func checkerror(err error, comment string) {
	if err != nil {
		log.Fatalln("[ERROR]", comment, err)
	}
}

func Write(somequintus []Quintus) {
	buffer := new(bytes.Buffer)
	encoder := gob.NewEncoder(buffer)

	encoder.Encode(somequintus)

	err := ioutil.WriteFile("quintus_gob", buffer.Bytes(), 0600)

	checkerror(err, "WRITE")
}

func Read() []Quintus {
	rawfile, err := ioutil.ReadFile("quintus_gob")

	checkerror(err, "READ")

	buffer := bytes.NewBuffer(rawfile)
	decoder := gob.NewDecoder(buffer)

	somequintus := make([]Quintus, 5)

	err = decoder.Decode(&somequintus)

	checkerror(err, "DECODE")

	return somequintus
}
<commit_msg>edge case<commit_after>package quintus

import (
	"bytes"
	"encoding/gob"
	"io/ioutil"
	"log"
  "os"
)

type Quintus struct {
	Name  string
	Date  string
	Times int
}

func checkerror(err error, comment string) {
	if err != nil {
		log.Fatalln("[ERROR]", comment, err)
	}
}

func Write(somequintus []Quintus) {
	buffer := new(bytes.Buffer)
	encoder := gob.NewEncoder(buffer)

	encoder.Encode(somequintus)

	err := ioutil.WriteFile("quintus_gob", buffer.Bytes(), 0600)

	checkerror(err, "WRITE")
}

func checkfile() {
  if _, err := os.Stat("quintus_gob"); os.IsNotExist(err) {
    Write(make([]Quintus, 0))
  }
}

func Read() []Quintus {
	rawfile, err := ioutil.ReadFile("quintus_gob")

	checkerror(err, "READ")

	buffer := bytes.NewBuffer(rawfile)
	decoder := gob.NewDecoder(buffer)

	somequintus := make([]Quintus, 5)

	err = decoder.Decode(&somequintus)

	checkerror(err, "DECODE")

	return somequintus
}
<|endoftext|>