<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"reflect"
	"strings"

	"github.com/spf13/cobra"

	"github.com/NebulousLabs/Sia/api"
	"github.com/NebulousLabs/Sia/build"
)

// flags
var (
	addr              string // override default API address
	initPassword      bool   // supply a custom password when creating a wallet
	hostVerbose       bool   // display additional host info
	renterShowHistory bool   // Show download history in addition to download queue.
	renterListVerbose bool   // Show additional info about uploaded files.
)

// exit codes
// inspired by sysexits.h
const (
	exitCodeGeneral = 1  // Not in sysexits.h, but is standard practice.
	exitCodeUsage   = 64 // EX_USAGE in sysexits.h
)

// apiGet wraps a GET request with a status code check, such that if the GET does
// not return 200, the error will be read and returned. The response body is
// not closed.
func apiGet(call string) (*http.Response, error) {
	if host, port, _ := net.SplitHostPort(addr); host == "" {
		addr = net.JoinHostPort("localhost", port)
	}
	resp, err := api.HttpGET("http://" + addr + call)
	if err != nil {
		return nil, errors.New("no response from daemon")
	}
	// check error code
	if resp.StatusCode == http.StatusNotFound {
		resp.Body.Close()
		err = errors.New("API call not recognized: " + call)
	} else if resp.StatusCode != http.StatusOK {
		errResp, _ := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		err = errors.New(strings.TrimSpace(string(errResp)))
	}
	return resp, err
}

// getAPI makes a GET API call and decodes the response.
func getAPI(call string, obj interface{}) error {
	resp, err := apiGet(call)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	err = json.NewDecoder(resp.Body).Decode(obj)
	if err != nil {
		return err
	}
	return nil
}

// get makes an API call and discards the response.
func get(call string) error {
	resp, err := apiGet(call)
	if err != nil {
		return err
	}
	resp.Body.Close()
	return nil
}

// apiPost wraps a POST request with a status code check, such that if the POST
// does not return 200, the error will be read and returned. The response body
// is not closed.
func apiPost(call, vals string) (*http.Response, error) {
	if host, port, _ := net.SplitHostPort(addr); host == "" {
		addr = net.JoinHostPort("localhost", port)
	}

	resp, err := api.HttpPOST("http://"+addr+call, vals)
	if err != nil {
		return nil, errors.New("no response from daemon")
	}
	// check error code
	if resp.StatusCode == http.StatusNotFound {
		resp.Body.Close()
		err = errors.New("API call not recognized: " + call)
	} else if resp.StatusCode != http.StatusOK {
		errResp, _ := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		err = errors.New(strings.TrimSpace(string(errResp)))
	}
	return resp, err
}

// postResp makes a POST API call and decodes the response.
func postResp(call, vals string, obj interface{}) error {
	resp, err := apiPost(call, vals)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	err = json.NewDecoder(resp.Body).Decode(obj)
	if err != nil {
		return err
	}
	return nil
}

func post(call, vals string) error {
	resp, err := apiPost(call, vals)
	if err != nil {
		return err
	}
	resp.Body.Close()
	return nil
}

// wrap wraps a generic command with a check that the command has been
// passed the correct number of arguments. The command must take only strings
// as arguments.
func wrap(fn interface{}) func(*cobra.Command, []string) {
	fnVal, fnType := reflect.ValueOf(fn), reflect.TypeOf(fn)
	if fnType.Kind() != reflect.Func {
		panic("wrapped function has wrong type signature")
	}
	for i := 0; i < fnType.NumIn(); i++ {
		if fnType.In(i).Kind() != reflect.String {
			panic("wrapped function has wrong type signature")
		}
	}

	return func(cmd *cobra.Command, args []string) {
		if len(args) != fnType.NumIn() {
			cmd.Usage()
			os.Exit(exitCodeUsage)
		}
		argVals := make([]reflect.Value, fnType.NumIn())
		for i := range args {
			argVals[i] = reflect.ValueOf(args[i])
		}
		fnVal.Call(argVals)
	}
}

// die prints its arguments to stderr, then exits the program with the default
// error code.
func die(args ...interface{}) {
	fmt.Fprintln(os.Stderr, args...)
	os.Exit(exitCodeGeneral)
}

// yesNo returns "Yes" if b is true, and "No" if b is false.
func yesNo(b bool) string {
	if b {
		return "Yes"
	}
	return "No"
}

func version() {
	println("Sia Client v" + build.Version)
}

func main() {
	root := &cobra.Command{
		Use:   os.Args[0],
		Short: "Sia Client v" + build.Version,
		Long:  "Sia Client v" + build.Version,
		Run:   wrap(consensuscmd),
	}

	// create command tree
	root.AddCommand(&cobra.Command{
		Use:   "version",
		Short: "Print version information",
		Long:  "Print version information.",
		Run:   wrap(version),
	})

	root.AddCommand(stopCmd)

	root.AddCommand(hostCmd)
	hostCmd.AddCommand(hostConfigCmd, hostAnnounceCmd, hostFolderCmd, hostSectorCmd)
	hostFolderCmd.AddCommand(hostFolderAddCmd, hostFolderRemoveCmd, hostFolderResizeCmd)
	hostSectorCmd.AddCommand(hostSectorDeleteCmd)
	hostCmd.Flags().BoolVarP(&hostVerbose, "verbose", "v", false, "Display detailed host info")

	root.AddCommand(hostdbCmd)

	root.AddCommand(minerCmd)
	minerCmd.AddCommand(minerStartCmd, minerStopCmd)

	root.AddCommand(walletCmd)
	walletCmd.AddCommand(walletAddressCmd, walletAddressesCmd, walletInitCmd,
		walletLoadCmd, walletLockCmd, walletSeedsCmd, walletSendCmd,
		walletBalanceCmd, walletTransactionsCmd, walletUnlockCmd)
	walletInitCmd.Flags().BoolVarP(&initPassword, "password", "p", false, "Prompt for a custom password")
	walletLoadCmd.AddCommand(walletLoad033xCmd, walletLoadSeedCmd, walletLoadSiagCmd)
	walletSendCmd.AddCommand(walletSendSiacoinsCmd, walletSendSiafundsCmd)

	root.AddCommand(renterCmd)
	renterCmd.AddCommand(renterFilesDeleteCmd, renterFilesDownloadCmd,
		renterDownloadsCmd, renterAllowanceCmd, renterSetAllowanceCmd,
		renterFilesListCmd, renterFilesLoadCmd, renterFilesLoadASCIICmd,
		renterFilesRenameCmd, renterFilesShareCmd, renterFilesShareASCIICmd,
		renterFilesUploadCmd, renterUploadsCmd)
	renterDownloadsCmd.Flags().BoolVarP(&renterShowHistory, "history", "H", false, "Show download history in addition to the download queue")
	renterFilesListCmd.Flags().BoolVarP(&renterListVerbose, "verbose", "v", false, "Show additional file info such as redundancy")

	root.AddCommand(gatewayCmd)
	gatewayCmd.AddCommand(gatewayAddCmd, gatewayRemoveCmd, gatewayAddressCmd, gatewayListCmd)

	root.AddCommand(consensusCmd)

	// parse flags
	root.PersistentFlags().StringVarP(&addr, "addr", "a", "localhost:9980", "which host/port to communicate with (i.e. the host/port siad is listening on)")

	// run
	if err := root.Execute(); err != nil {
		// Since no commands return errors (all commands set Command.Run instead of
		// Command.RunE), Command.Execute() should only return an error on an
		// invalid command or flag. Therefore Command.Usage() was called (assuming
		// Command.SilenceUsage is false) and we should exit with exitCodeUsage.
		os.Exit(exitCodeUsage)
	}
}
<commit_msg>Add a `-v, --verbose` flag to `siac renter` (#1146)<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"reflect"
	"strings"

	"github.com/spf13/cobra"

	"github.com/NebulousLabs/Sia/api"
	"github.com/NebulousLabs/Sia/build"
)

// flags
var (
	addr              string // override default API address
	initPassword      bool   // supply a custom password when creating a wallet
	hostVerbose       bool   // display additional host info
	renterShowHistory bool   // Show download history in addition to download queue.
	renterListVerbose bool   // Show additional info about uploaded files.
)

// exit codes
// inspired by sysexits.h
const (
	exitCodeGeneral = 1  // Not in sysexits.h, but is standard practice.
	exitCodeUsage   = 64 // EX_USAGE in sysexits.h
)

// apiGet wraps a GET request with a status code check, such that if the GET does
// not return 200, the error will be read and returned. The response body is
// not closed.
func apiGet(call string) (*http.Response, error) {
	if host, port, _ := net.SplitHostPort(addr); host == "" {
		addr = net.JoinHostPort("localhost", port)
	}
	resp, err := api.HttpGET("http://" + addr + call)
	if err != nil {
		return nil, errors.New("no response from daemon")
	}
	// check error code
	if resp.StatusCode == http.StatusNotFound {
		resp.Body.Close()
		err = errors.New("API call not recognized: " + call)
	} else if resp.StatusCode != http.StatusOK {
		errResp, _ := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		err = errors.New(strings.TrimSpace(string(errResp)))
	}
	return resp, err
}

// getAPI makes a GET API call and decodes the response.
func getAPI(call string, obj interface{}) error {
	resp, err := apiGet(call)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	err = json.NewDecoder(resp.Body).Decode(obj)
	if err != nil {
		return err
	}
	return nil
}

// get makes an API call and discards the response.
func get(call string) error {
	resp, err := apiGet(call)
	if err != nil {
		return err
	}
	resp.Body.Close()
	return nil
}

// apiPost wraps a POST request with a status code check, such that if the POST
// does not return 200, the error will be read and returned. The response body
// is not closed.
func apiPost(call, vals string) (*http.Response, error) {
	if host, port, _ := net.SplitHostPort(addr); host == "" {
		addr = net.JoinHostPort("localhost", port)
	}

	resp, err := api.HttpPOST("http://"+addr+call, vals)
	if err != nil {
		return nil, errors.New("no response from daemon")
	}
	// check error code
	if resp.StatusCode == http.StatusNotFound {
		resp.Body.Close()
		err = errors.New("API call not recognized: " + call)
	} else if resp.StatusCode != http.StatusOK {
		errResp, _ := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		err = errors.New(strings.TrimSpace(string(errResp)))
	}
	return resp, err
}

// postResp makes a POST API call and decodes the response.
func postResp(call, vals string, obj interface{}) error {
	resp, err := apiPost(call, vals)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	err = json.NewDecoder(resp.Body).Decode(obj)
	if err != nil {
		return err
	}
	return nil
}

func post(call, vals string) error {
	resp, err := apiPost(call, vals)
	if err != nil {
		return err
	}
	resp.Body.Close()
	return nil
}

// wrap wraps a generic command with a check that the command has been
// passed the correct number of arguments. The command must take only strings
// as arguments.
func wrap(fn interface{}) func(*cobra.Command, []string) {
	fnVal, fnType := reflect.ValueOf(fn), reflect.TypeOf(fn)
	if fnType.Kind() != reflect.Func {
		panic("wrapped function has wrong type signature")
	}
	for i := 0; i < fnType.NumIn(); i++ {
		if fnType.In(i).Kind() != reflect.String {
			panic("wrapped function has wrong type signature")
		}
	}

	return func(cmd *cobra.Command, args []string) {
		if len(args) != fnType.NumIn() {
			cmd.Usage()
			os.Exit(exitCodeUsage)
		}
		argVals := make([]reflect.Value, fnType.NumIn())
		for i := range args {
			argVals[i] = reflect.ValueOf(args[i])
		}
		fnVal.Call(argVals)
	}
}

// die prints its arguments to stderr, then exits the program with the default
// error code.
func die(args ...interface{}) {
	fmt.Fprintln(os.Stderr, args...)
	os.Exit(exitCodeGeneral)
}

// yesNo returns "Yes" if b is true, and "No" if b is false.
func yesNo(b bool) string {
	if b {
		return "Yes"
	}
	return "No"
}

func version() {
	println("Sia Client v" + build.Version)
}

func main() {
	root := &cobra.Command{
		Use:   os.Args[0],
		Short: "Sia Client v" + build.Version,
		Long:  "Sia Client v" + build.Version,
		Run:   wrap(consensuscmd),
	}

	// create command tree
	root.AddCommand(&cobra.Command{
		Use:   "version",
		Short: "Print version information",
		Long:  "Print version information.",
		Run:   wrap(version),
	})

	root.AddCommand(stopCmd)

	root.AddCommand(hostCmd)
	hostCmd.AddCommand(hostConfigCmd, hostAnnounceCmd, hostFolderCmd, hostSectorCmd)
	hostFolderCmd.AddCommand(hostFolderAddCmd, hostFolderRemoveCmd, hostFolderResizeCmd)
	hostSectorCmd.AddCommand(hostSectorDeleteCmd)
	hostCmd.Flags().BoolVarP(&hostVerbose, "verbose", "v", false, "Display detailed host info")

	root.AddCommand(hostdbCmd)

	root.AddCommand(minerCmd)
	minerCmd.AddCommand(minerStartCmd, minerStopCmd)

	root.AddCommand(walletCmd)
	walletCmd.AddCommand(walletAddressCmd, walletAddressesCmd, walletInitCmd,
		walletLoadCmd, walletLockCmd, walletSeedsCmd, walletSendCmd,
		walletBalanceCmd, walletTransactionsCmd, walletUnlockCmd)
	walletInitCmd.Flags().BoolVarP(&initPassword, "password", "p", false, "Prompt for a custom password")
	walletLoadCmd.AddCommand(walletLoad033xCmd, walletLoadSeedCmd, walletLoadSiagCmd)
	walletSendCmd.AddCommand(walletSendSiacoinsCmd, walletSendSiafundsCmd)

	root.AddCommand(renterCmd)
	renterCmd.AddCommand(renterFilesDeleteCmd, renterFilesDownloadCmd,
		renterDownloadsCmd, renterAllowanceCmd, renterSetAllowanceCmd,
		renterFilesListCmd, renterFilesLoadCmd, renterFilesLoadASCIICmd,
		renterFilesRenameCmd, renterFilesShareCmd, renterFilesShareASCIICmd,
		renterFilesUploadCmd, renterUploadsCmd)
	renterCmd.Flags().BoolVarP(&renterListVerbose, "verbose", "v", false, "Show additional file info such as redundancy")
	renterDownloadsCmd.Flags().BoolVarP(&renterShowHistory, "history", "H", false, "Show download history in addition to the download queue")
	renterFilesListCmd.Flags().BoolVarP(&renterListVerbose, "verbose", "v", false, "Show additional file info such as redundancy")

	root.AddCommand(gatewayCmd)
	gatewayCmd.AddCommand(gatewayAddCmd, gatewayRemoveCmd, gatewayAddressCmd, gatewayListCmd)

	root.AddCommand(consensusCmd)

	// parse flags
	root.PersistentFlags().StringVarP(&addr, "addr", "a", "localhost:9980", "which host/port to communicate with (i.e. the host/port siad is listening on)")

	// run
	if err := root.Execute(); err != nil {
		// Since no commands return errors (all commands set Command.Run instead of
		// Command.RunE), Command.Execute() should only return an error on an
		// invalid command or flag. Therefore Command.Usage() was called (assuming
		// Command.SilenceUsage is false) and we should exit with exitCodeUsage.
		os.Exit(exitCodeUsage)
	}
}
<|endoftext|>


<commit_before>/* A hash table of uint64 key-value pairs. */
package file

import (
	"encoding/binary"
	"errors"
	"fmt"
	"log"
	"math"
	"sync"
)

const (
	HASH_TABLE_GROWTH      = uint64(134217728) // Grows every 128MB
	ENTRY_VALID            = byte(1)
	ENTRY_INVALID          = byte(0)
	ENTRY_SIZE             = uint64(1 + 10 + 10) // byte(validity), uint64(hash key), uint64(value)
	BUCKET_HEADER_SIZE     = uint64(10)          // uint64(next bucket)
	HASH_TABLE_REGION_SIZE = 1024 * 4            // 4KB per locking region, roughly the size of a single bucket
)

type HashTable struct {
	File                            *File
	BucketSize, HashBits, PerBucket uint64
	NumBuckets, InitialBuckets      uint64
	tableGrowMutex                  sync.Mutex
	regionRWMutex                   []*sync.RWMutex
}

// Open a hash table file.
func OpenHash(name string, hashBits, perBucket uint64) (ht *HashTable, err error) {
	if hashBits < 2 || perBucket < 2 {
		return nil, errors.New(fmt.Sprintf("ERROR: Hash table is too small (%d hash bits, %d per bucket)", hashBits, perBucket))
	}
	file, err := Open(name, HASH_TABLE_GROWTH)
	if err != nil {
		return
	}
	rwMutexes := make([]*sync.RWMutex, file.Size/HASH_TABLE_REGION_SIZE+1)
	for i := range rwMutexes {
		rwMutexes[i] = new(sync.RWMutex)
	}
	ht = &HashTable{File: file, HashBits: hashBits, PerBucket: perBucket,
		tableGrowMutex: sync.Mutex{},
		regionRWMutex:  rwMutexes}
	ht.BucketSize = BUCKET_HEADER_SIZE + ENTRY_SIZE*perBucket
	// find out how many buckets are there in table
	// first, assume that entire file is full of buckets
	ht.File.UsedSize = ht.File.Size
	ht.NumBuckets = ht.File.Size / ht.BucketSize
	// next, find the longest bucket train
	ht.InitialBuckets = uint64(math.Pow(2, float64(hashBits)))
	longestBucketChain := ht.InitialBuckets
	for i := uint64(0); i < ht.InitialBuckets; i++ {
		lastBucket := ht.lastBucket(i)
		if lastBucket+1 > longestBucketChain && lastBucket+1 <= ht.NumBuckets {
			longestBucketChain = lastBucket + 1
		}
	}
	// last, calculate used size according to number of buckets
	ht.NumBuckets = longestBucketChain
	ht.File.UsedSize = ht.NumBuckets * ht.BucketSize
	log.Printf("%s has %d initial buckets, %d buckets, and %d bytes out of %d bytes in-use", name, ht.InitialBuckets, ht.NumBuckets, ht.File.UsedSize, ht.File.Size)
	return ht, nil
}

// Return the number of next chained bucket, 0 if there is not any.
func (ht *HashTable) nextBucket(bucket uint64) uint64 {
	if bucket >= ht.NumBuckets {
		return 0
	}
	bucketAddr := bucket * ht.BucketSize
	if next, _ := binary.Uvarint(ht.File.Buf[bucketAddr : bucketAddr+BUCKET_HEADER_SIZE]); next == 0 {
		return 0
	} else if next <= bucket {
		log.Printf("ERROR: Bucket loop in hash table %s at bucket no.%d, address %d", ht.File.Name, bucket, bucketAddr)
		return 0
	} else if next >= ht.NumBuckets || next < ht.InitialBuckets {
		log.Printf("ERROR: Bad bucket refernece (%d is out of range %d - %d) in %s", next, ht.InitialBuckets, ht.NumBuckets, ht.File.Name)
		return 0
	} else {
		return next
	}
}

// Return the last bucket number in chain.
func (ht *HashTable) lastBucket(bucket uint64) uint64 {
	curr := bucket
	for {
		next := ht.nextBucket(curr)
		if next == 0 {
			return curr
		}
		curr = next
	}
}

// Grow a new bucket on the chain of buckets.
func (ht *HashTable) grow(bucket uint64) {
	// lock both bucket creation and the bucket affected
	ht.tableGrowMutex.Lock()
	// when file is full, we have to lock down everything before growing the file
	if !ht.File.CheckSize(ht.BucketSize) {
		originalMutexes := ht.regionRWMutex
		for _, region := range originalMutexes {
			region.Lock()
		}
		ht.File.CheckSizeAndEnsure(ht.BucketSize)
		// make more mutexes
		moreMutexes := make([]*sync.RWMutex, HASH_TABLE_GROWTH/HASH_TABLE_REGION_SIZE+1)
		for i := range moreMutexes {
			moreMutexes[i] = new(sync.RWMutex)
		}
		// merge mutexes together
		ht.regionRWMutex = append(ht.regionRWMutex, moreMutexes...)
		for _, region := range originalMutexes {
			region.Unlock()
		}
	}
	lastBucketAddr := ht.lastBucket(bucket) * ht.BucketSize
	binary.PutUvarint(ht.File.Buf[lastBucketAddr:lastBucketAddr+10], ht.NumBuckets)
	ht.File.UsedSize += ht.BucketSize
	ht.NumBuckets += 1
	ht.tableGrowMutex.Unlock()
}

// Return a hash key to be used by hash table by masking non-key bits.
func (ht *HashTable) hashKey(key uint64) uint64 {
	return key & ((1 << ht.HashBits) - 1)
}

// Put a new key-value pair.
func (ht *HashTable) Put(key, val uint64) {
	var bucket, entry uint64 = ht.hashKey(key), 0
	region := bucket / HASH_TABLE_REGION_SIZE
	mutex := ht.regionRWMutex[region]
	mutex.Lock()
	for {
		entryAddr := bucket*ht.BucketSize + BUCKET_HEADER_SIZE + entry*ENTRY_SIZE
		if ht.File.Buf[entryAddr] != ENTRY_VALID {
			ht.File.Buf[entryAddr] = ENTRY_VALID
			binary.PutUvarint(ht.File.Buf[entryAddr+1:entryAddr+11], key)
			binary.PutUvarint(ht.File.Buf[entryAddr+11:entryAddr+21], val)
			mutex.Unlock()
			return
		}
		if entry++; entry == ht.PerBucket {
			mutex.Unlock()
			entry = 0
			if bucket = ht.nextBucket(bucket); bucket == 0 {
				ht.grow(ht.hashKey(key))
				ht.Put(key, val)
				return
			}
			region = bucket / HASH_TABLE_REGION_SIZE
			mutex = ht.regionRWMutex[region]
			mutex.Lock()
		}
	}
}

// Get key-value pairs.
func (ht *HashTable) Get(key, limit uint64, filter func(uint64, uint64) bool) (keys, vals []uint64) {
	var count, entry, bucket uint64 = 0, 0, ht.hashKey(key)
	if limit == 0 {
		keys = make([]uint64, 0, 10)
		vals = make([]uint64, 0, 10)
	} else {
		keys = make([]uint64, 0, limit)
		vals = make([]uint64, 0, limit)
	}
	region := bucket / HASH_TABLE_REGION_SIZE
	mutex := ht.regionRWMutex[region]
	mutex.RLock()
	for {
		entryAddr := bucket*ht.BucketSize + BUCKET_HEADER_SIZE + entry*ENTRY_SIZE
		entryKey, _ := binary.Uvarint(ht.File.Buf[entryAddr+1 : entryAddr+11])
		entryVal, _ := binary.Uvarint(ht.File.Buf[entryAddr+11 : entryAddr+21])
		if ht.File.Buf[entryAddr] == ENTRY_VALID {
			if entryKey == key && filter(entryKey, entryVal) {
				keys = append(keys, entryKey)
				vals = append(vals, entryVal)
				if count++; count == limit {
					mutex.RUnlock()
					return
				}
			}
		} else if entryKey == 0 && entryVal == 0 {
			mutex.RUnlock()
			return
		}
		if entry++; entry == ht.PerBucket {
			mutex.RUnlock()
			entry = 0
			if bucket = ht.nextBucket(bucket); bucket == 0 {
				return
			}
			region = bucket / HASH_TABLE_REGION_SIZE
			mutex = ht.regionRWMutex[region]
			mutex.RLock()
		}
	}
}

// Remove specific key-value pair.
func (ht *HashTable) Remove(key, val uint64) {
	var entry, bucket uint64 = 0, ht.hashKey(key)
	region := bucket / HASH_TABLE_REGION_SIZE
	mutex := ht.regionRWMutex[region]
	mutex.Lock()
	for {
		entryAddr := bucket*ht.BucketSize + BUCKET_HEADER_SIZE + entry*ENTRY_SIZE
		entryKey, _ := binary.Uvarint(ht.File.Buf[entryAddr+1 : entryAddr+11])
		entryVal, _ := binary.Uvarint(ht.File.Buf[entryAddr+11 : entryAddr+21])
		if ht.File.Buf[entryAddr] == ENTRY_VALID {
			if entryKey == key && entryVal == val {
				ht.File.Buf[entryAddr] = ENTRY_INVALID
				mutex.Unlock()
				return
			}
		} else if entryKey == 0 && entryVal == 0 {
			mutex.Unlock()
			return
		}
		if entry++; entry == ht.PerBucket {
			mutex.Unlock()
			entry = 0
			if bucket = ht.nextBucket(bucket); bucket == 0 {
				return
			}
			region = bucket / HASH_TABLE_REGION_SIZE
			mutex = ht.regionRWMutex[region]
			mutex.Lock()
		}
	}
}

// Return all entries in the hash table.
func (ht *HashTable) GetAll(limit uint64) (keys, vals []uint64) {
	keys = make([]uint64, 0, 100)
	vals = make([]uint64, 0, 100)
	counter := uint64(0)
	for head := uint64(0); head < uint64(math.Pow(2, float64(ht.HashBits))); head++ {
		var entry, bucket uint64 = 0, head
		region := bucket / HASH_TABLE_REGION_SIZE
		mutex := ht.regionRWMutex[region]
		mutex.RLock()
		for {
			entryAddr := bucket*ht.BucketSize + BUCKET_HEADER_SIZE + entry*ENTRY_SIZE
			entryKey, _ := binary.Uvarint(ht.File.Buf[entryAddr+1 : entryAddr+11])
			entryVal, _ := binary.Uvarint(ht.File.Buf[entryAddr+11 : entryAddr+21])
			if ht.File.Buf[entryAddr] == ENTRY_VALID {
				counter++
				keys = append(keys, entryKey)
				vals = append(vals, entryVal)
				if counter == limit {
					mutex.RUnlock()
					return
				}
			} else if entryKey == 0 && entryVal == 0 {
				mutex.RUnlock()
				break
			}
			if entry++; entry == ht.PerBucket {
				mutex.RUnlock()
				entry = 0
				if bucket = ht.nextBucket(bucket); bucket == 0 {
					return
				}
				region = bucket / HASH_TABLE_REGION_SIZE
				mutex = ht.regionRWMutex[region]
				mutex.RLock()
			}
		}
	}
	return
}
<commit_msg>almost forgot to grow the hash table file when file does not have enough room<commit_after>/* A hash table of uint64 key-value pairs. */
package file

import (
	"encoding/binary"
	"errors"
	"fmt"
	"log"
	"math"
	"sync"
)

const (
	HASH_TABLE_GROWTH      = uint64(134217728) // Grows every 128MB
	ENTRY_VALID            = byte(1)
	ENTRY_INVALID          = byte(0)
	ENTRY_SIZE             = uint64(1 + 10 + 10) // byte(validity), uint64(hash key), uint64(value)
	BUCKET_HEADER_SIZE     = uint64(10)          // uint64(next bucket)
	HASH_TABLE_REGION_SIZE = 1024 * 4            // 4KB per locking region, roughly the size of a single bucket
)

type HashTable struct {
	File                            *File
	BucketSize, HashBits, PerBucket uint64
	NumBuckets, InitialBuckets      uint64
	tableGrowMutex                  sync.Mutex
	regionRWMutex                   []*sync.RWMutex
}

// Open a hash table file.
func OpenHash(name string, hashBits, perBucket uint64) (ht *HashTable, err error) {
	if hashBits < 2 || perBucket < 2 {
		return nil, errors.New(fmt.Sprintf("ERROR: Hash table is too small (%d hash bits, %d per bucket)", hashBits, perBucket))
	}
	file, err := Open(name, HASH_TABLE_GROWTH)
	if err != nil {
		return
	}
	rwMutexes := make([]*sync.RWMutex, file.Size/HASH_TABLE_REGION_SIZE+1)
	for i := range rwMutexes {
		rwMutexes[i] = new(sync.RWMutex)
	}
	ht = &HashTable{File: file, HashBits: hashBits, PerBucket: perBucket,
		tableGrowMutex: sync.Mutex{},
		regionRWMutex:  rwMutexes}
	ht.BucketSize = BUCKET_HEADER_SIZE + ENTRY_SIZE*perBucket
	// find out how many buckets are there in table
	// first, assume that entire file is full of buckets
	ht.File.UsedSize = ht.File.Size
	ht.NumBuckets = ht.File.Size / ht.BucketSize
	// next, find the longest bucket train
	ht.InitialBuckets = uint64(math.Pow(2, float64(hashBits)))
	longestBucketChain := ht.InitialBuckets
	for i := uint64(0); i < ht.InitialBuckets; i++ {
		lastBucket := ht.lastBucket(i)
		if lastBucket+1 > longestBucketChain && lastBucket+1 <= ht.NumBuckets {
			longestBucketChain = lastBucket + 1
		}
	}
	// calculate used size according to number of buckets
	ht.NumBuckets = longestBucketChain
	usedSize := ht.NumBuckets * ht.BucketSize
	// grow the file if the size is not enough
	if usedSize > ht.File.Size {
		ht.File.UsedSize = ht.File.Size
		ht.File.CheckSizeAndEnsure(((usedSize-ht.File.Size)/ht.BucketSize + 1) * ht.BucketSize)
	}
	ht.File.UsedSize = usedSize
	log.Printf("%s has %d initial buckets, %d buckets, and %d bytes out of %d bytes in-use", name, ht.InitialBuckets, ht.NumBuckets, ht.File.UsedSize, ht.File.Size)
	return ht, nil
}

// Return the number of next chained bucket, 0 if there is not any.
func (ht *HashTable) nextBucket(bucket uint64) uint64 {
	if bucket >= ht.NumBuckets {
		return 0
	}
	bucketAddr := bucket * ht.BucketSize
	if next, _ := binary.Uvarint(ht.File.Buf[bucketAddr : bucketAddr+BUCKET_HEADER_SIZE]); next == 0 {
		return 0
	} else if next <= bucket {
		log.Printf("ERROR: Bucket loop in hash table %s at bucket no.%d, address %d", ht.File.Name, bucket, bucketAddr)
		return 0
	} else if next >= ht.NumBuckets || next < ht.InitialBuckets {
		log.Printf("ERROR: Bad bucket refernece (%d is out of range %d - %d) in %s", next, ht.InitialBuckets, ht.NumBuckets, ht.File.Name)
		return 0
	} else {
		return next
	}
}

// Return the last bucket number in chain.
func (ht *HashTable) lastBucket(bucket uint64) uint64 {
	curr := bucket
	for {
		next := ht.nextBucket(curr)
		if next == 0 {
			return curr
		}
		curr = next
	}
}

// Grow a new bucket on the chain of buckets.
func (ht *HashTable) grow(bucket uint64) {
	// lock both bucket creation and the bucket affected
	ht.tableGrowMutex.Lock()
	// when file is full, we have to lock down everything before growing the file
	if !ht.File.CheckSize(ht.BucketSize) {
		originalMutexes := ht.regionRWMutex
		for _, region := range originalMutexes {
			region.Lock()
		}
		ht.File.CheckSizeAndEnsure(ht.BucketSize)
		// make more mutexes
		moreMutexes := make([]*sync.RWMutex, HASH_TABLE_GROWTH/HASH_TABLE_REGION_SIZE+1)
		for i := range moreMutexes {
			moreMutexes[i] = new(sync.RWMutex)
		}
		// merge mutexes together
		ht.regionRWMutex = append(ht.regionRWMutex, moreMutexes...)
		for _, region := range originalMutexes {
			region.Unlock()
		}
	}
	lastBucketAddr := ht.lastBucket(bucket) * ht.BucketSize
	binary.PutUvarint(ht.File.Buf[lastBucketAddr:lastBucketAddr+10], ht.NumBuckets)
	ht.File.UsedSize += ht.BucketSize
	ht.NumBuckets += 1
	ht.tableGrowMutex.Unlock()
}

// Return a hash key to be used by hash table by masking non-key bits.
func (ht *HashTable) hashKey(key uint64) uint64 {
	return key & ((1 << ht.HashBits) - 1)
}

// Put a new key-value pair.
func (ht *HashTable) Put(key, val uint64) {
	var bucket, entry uint64 = ht.hashKey(key), 0
	region := bucket / HASH_TABLE_REGION_SIZE
	mutex := ht.regionRWMutex[region]
	mutex.Lock()
	for {
		entryAddr := bucket*ht.BucketSize + BUCKET_HEADER_SIZE + entry*ENTRY_SIZE
		if ht.File.Buf[entryAddr] != ENTRY_VALID {
			ht.File.Buf[entryAddr] = ENTRY_VALID
			binary.PutUvarint(ht.File.Buf[entryAddr+1:entryAddr+11], key)
			binary.PutUvarint(ht.File.Buf[entryAddr+11:entryAddr+21], val)
			mutex.Unlock()
			return
		}
		if entry++; entry == ht.PerBucket {
			mutex.Unlock()
			entry = 0
			if bucket = ht.nextBucket(bucket); bucket == 0 {
				ht.grow(ht.hashKey(key))
				ht.Put(key, val)
				return
			}
			region = bucket / HASH_TABLE_REGION_SIZE
			mutex = ht.regionRWMutex[region]
			mutex.Lock()
		}
	}
}

// Get key-value pairs.
func (ht *HashTable) Get(key, limit uint64, filter func(uint64, uint64) bool) (keys, vals []uint64) {
	var count, entry, bucket uint64 = 0, 0, ht.hashKey(key)
	if limit == 0 {
		keys = make([]uint64, 0, 10)
		vals = make([]uint64, 0, 10)
	} else {
		keys = make([]uint64, 0, limit)
		vals = make([]uint64, 0, limit)
	}
	region := bucket / HASH_TABLE_REGION_SIZE
	mutex := ht.regionRWMutex[region]
	mutex.RLock()
	for {
		entryAddr := bucket*ht.BucketSize + BUCKET_HEADER_SIZE + entry*ENTRY_SIZE
		entryKey, _ := binary.Uvarint(ht.File.Buf[entryAddr+1 : entryAddr+11])
		entryVal, _ := binary.Uvarint(ht.File.Buf[entryAddr+11 : entryAddr+21])
		if ht.File.Buf[entryAddr] == ENTRY_VALID {
			if entryKey == key && filter(entryKey, entryVal) {
				keys = append(keys, entryKey)
				vals = append(vals, entryVal)
				if count++; count == limit {
					mutex.RUnlock()
					return
				}
			}
		} else if entryKey == 0 && entryVal == 0 {
			mutex.RUnlock()
			return
		}
		if entry++; entry == ht.PerBucket {
			mutex.RUnlock()
			entry = 0
			if bucket = ht.nextBucket(bucket); bucket == 0 {
				return
			}
			region = bucket / HASH_TABLE_REGION_SIZE
			mutex = ht.regionRWMutex[region]
			mutex.RLock()
		}
	}
}

// Remove specific key-value pair.
func (ht *HashTable) Remove(key, val uint64) {
	var entry, bucket uint64 = 0, ht.hashKey(key)
	region := bucket / HASH_TABLE_REGION_SIZE
	mutex := ht.regionRWMutex[region]
	mutex.Lock()
	for {
		entryAddr := bucket*ht.BucketSize + BUCKET_HEADER_SIZE + entry*ENTRY_SIZE
		entryKey, _ := binary.Uvarint(ht.File.Buf[entryAddr+1 : entryAddr+11])
		entryVal, _ := binary.Uvarint(ht.File.Buf[entryAddr+11 : entryAddr+21])
		if ht.File.Buf[entryAddr] == ENTRY_VALID {
			if entryKey == key && entryVal == val {
				ht.File.Buf[entryAddr] = ENTRY_INVALID
				mutex.Unlock()
				return
			}
		} else if entryKey == 0 && entryVal == 0 {
			mutex.Unlock()
			return
		}
		if entry++; entry == ht.PerBucket {
			mutex.Unlock()
			entry = 0
			if bucket = ht.nextBucket(bucket); bucket == 0 {
				return
			}
			region = bucket / HASH_TABLE_REGION_SIZE
			mutex = ht.regionRWMutex[region]
			mutex.Lock()
		}
	}
}

// Return all entries in the hash table.
func (ht *HashTable) GetAll(limit uint64) (keys, vals []uint64) {
	keys = make([]uint64, 0, 100)
	vals = make([]uint64, 0, 100)
	counter := uint64(0)
	for head := uint64(0); head < uint64(math.Pow(2, float64(ht.HashBits))); head++ {
		var entry, bucket uint64 = 0, head
		region := bucket / HASH_TABLE_REGION_SIZE
		mutex := ht.regionRWMutex[region]
		mutex.RLock()
		for {
			entryAddr := bucket*ht.BucketSize + BUCKET_HEADER_SIZE + entry*ENTRY_SIZE
			entryKey, _ := binary.Uvarint(ht.File.Buf[entryAddr+1 : entryAddr+11])
			entryVal, _ := binary.Uvarint(ht.File.Buf[entryAddr+11 : entryAddr+21])
			if ht.File.Buf[entryAddr] == ENTRY_VALID {
				counter++
				keys = append(keys, entryKey)
				vals = append(vals, entryVal)
				if counter == limit {
					mutex.RUnlock()
					return
				}
			} else if entryKey == 0 && entryVal == 0 {
				mutex.RUnlock()
				break
			}
			if entry++; entry == ht.PerBucket {
				mutex.RUnlock()
				entry = 0
				if bucket = ht.nextBucket(bucket); bucket == 0 {
					return
				}
				region = bucket / HASH_TABLE_REGION_SIZE
				mutex = ht.regionRWMutex[region]
				mutex.RLock()
			}
		}
	}
	return
}
<|endoftext|>


<commit_before>// Package simplelog provides a dead simple logging system.
package simplelog

import (
	"errors"
	"log"
	"os"
)

// Constants for the different log levels supported by the library
const (
	LevelDebug   int = iota // Threshold for Debug log level
	LevelInfo               // Threshold for Info log level
	LevelWarning            // Threshold for Warning log level
	LevelError              // Threshold for Error log level
)

// LogLevel represents a logger object for a given log level.
type LogLevel struct {
	logger *log.Logger
	prefix string
	level  int
}

// Logger objects that will be used to perform the actual logging.
// Each of them represents a different logging level and can be pointed to a different backend (file, stdout, etc...)
var (
	Error   = &LogLevel{prefix: "ERROR: ", level: LevelError}
	Warning = &LogLevel{prefix: "WARNING: ", level: LevelWarning}
	Info    = &LogLevel{prefix: "INFO: ", level: LevelInfo}
	Debug   = &LogLevel{prefix: "DEBUG: ", level: LevelDebug}
)
var logThreshold = LevelError

// Common errors that can be returned
var (
	ErrInvalidThreshold = errors.New("Invalid Threshold. Need one between LOG_DEBUG and LOG_ERROR") //
)

func init() {
	var levels = []*LogLevel{Error, Warning, Info, Debug}
	for _, level := range levels {
		// TODO: Allow for different handles than stdout
		level.logger = log.New(os.Stdout, level.prefix, log.LstdFlags)
	}
}

// SetThreshold sets the logging threshold level.
// Will return InvalidThreshold if new threshold isn't in the accepted range
func SetThreshold(t int) error {
	if t < LevelDebug || t > LevelError {
		return ErrInvalidThreshold
	}
	logThreshold = t
	return nil
}

// IsDebug will return true if logging threshold is currently set at Debug level
func IsDebug() bool {
	return logThreshold == LevelDebug
}

// LogThreshold will return the current log level
func LogThreshold() int {
	return logThreshold
}

// Printf will use the logger attached to this LogLevel to write a log message.
// Message will only get written if current log level allows it (it won't write INFO messages if we're at ERROR)
func (l *LogLevel) Printf(format string, v ...interface{}) {
	if l.level >= logThreshold {
		l.logger.Printf(format, v...)
	}
}
<commit_msg>Output to different places in different log levels<commit_after>// Package simplelog provides a dead simple logging system.
package simplelog

import (
	"errors"
	"io"
	"log"
	"os"
)

// Constants for the different log levels supported by the library
const (
	LevelDebug   int = iota // Threshold for Debug log level
	LevelInfo               // Threshold for Info log level
	LevelWarning            // Threshold for Warning log level
	LevelError              // Threshold for Error log level
)

// LogLevel represents a logger object for a given log level.
type LogLevel struct {
	logger      *log.Logger
	prefix      string
	level       int
	destination io.Writer
}

// Logger objects that will be used to perform the actual logging.
// Each of them represents a different logging level and can be pointed to a different backend (file, stdout, etc...)
var (
	Error   = &LogLevel{prefix: "ERROR: ", level: LevelError, destination: os.Stderr}
	Warning = &LogLevel{prefix: "WARNING: ", level: LevelWarning, destination: os.Stderr}
	Info    = &LogLevel{prefix: "INFO: ", level: LevelInfo, destination: os.Stdout}
	Debug   = &LogLevel{prefix: "DEBUG: ", level: LevelDebug, destination: os.Stdout}
)
var logThreshold = LevelError

// Common errors that can be returned
var (
	ErrInvalidThreshold = errors.New("Invalid Threshold. Need one between LOG_DEBUG and LOG_ERROR") //
)

func init() {
	var levels = []*LogLevel{Error, Warning, Info, Debug}
	for _, level := range levels {
		level.logger = log.New(level.destination, level.prefix, log.LstdFlags)
	}
}

// SetThreshold sets the logging threshold level.
// Will return InvalidThreshold if new threshold isn't in the accepted range
func SetThreshold(t int) error {
	if t < LevelDebug || t > LevelError {
		return ErrInvalidThreshold
	}
	logThreshold = t
	return nil
}

// IsDebug will return true if logging threshold is currently set at Debug level
func IsDebug() bool {
	return logThreshold == LevelDebug
}

// LogThreshold will return the current log level
func LogThreshold() int {
	return logThreshold
}

// Printf will use the logger attached to this LogLevel to write a log message.
// Message will only get written if current log level allows it (it won't write INFO messages if we're at ERROR)
func (l *LogLevel) Printf(format string, v ...interface{}) {
	if l.level >= logThreshold {
		l.logger.Printf(format, v...)
	}
}
<|endoftext|>


<commit_before>package api

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/go-gorp/gorp"
	"github.com/gorilla/mux"

	"github.com/ovh/cds/engine/api/application"
	"github.com/ovh/cds/engine/api/cache"
	"github.com/ovh/cds/engine/api/hook"
	"github.com/ovh/cds/engine/api/pipeline"
	"github.com/ovh/cds/engine/api/project"
	"github.com/ovh/cds/engine/api/workflowv0"
	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/log"
)

func processStashHook(w http.ResponseWriter, r *http.Request, data []byte) hook.ReceivedHook {
	rh := hook.ReceivedHook{
		URL:        *r.URL,
		Data:       data,
		ProjectKey: r.FormValue("project"),
		Repository: r.FormValue("name"),
		Branch:     r.FormValue("branch"),
		Hash:       r.FormValue("hash"),
		Author:     r.FormValue("author"),
		Message:    r.FormValue("message"),
		UID:        r.FormValue("uid"),
	}

	return rh
}

func processGitlabHook(w http.ResponseWriter, r *http.Request, data []byte) (hook.ReceivedHook, error) {

	type gitlabEvent struct {
		ObjectKind  string `json:"object_kind"`
		Ref         string `json:"ref"`
		UserName    string `json:"user_name"`
		CheckoutSha string `json:"checkout_sha"`
	}

	var ge gitlabEvent
	if err := json.Unmarshal(data, &ge); err != nil {
		return hook.ReceivedHook{}, err
	}

	ge.Ref = strings.TrimPrefix(ge.Ref, "refs/heads/")

	rh := hook.ReceivedHook{
		URL:        *r.URL,
		Data:       data,
		ProjectKey: r.FormValue("project"),
		Repository: r.FormValue("name"),
		Branch:     ge.Ref,
		Hash:       ge.CheckoutSha,
		Author:     ge.UserName,
		Message:    ge.ObjectKind,
		UID:        r.FormValue("uid"),
	}

	return rh, nil
}

func (api *API) receiveHookHandler() Handler {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		// Get body
		data, err := ioutil.ReadAll(r.Body)
		if err != nil {
			return sdk.ErrWrongRequest

		}

		if err = r.ParseForm(); err != nil {
			return sdk.WrapError(err, "receiveHook> cannot parse query params")
		}

		var rh hook.ReceivedHook
		if r.Header.Get("X-Gitlab-Event") != "" {
			rh, err = processGitlabHook(w, r, data)
			if err != nil {
				return err
			}
		} else {
			rh = processStashHook(w, r, data)
		}

		db := api.DBConnectionFactory.GetDBMap()
		if db == nil {
			hook.Recovery(api.Cache, rh, fmt.Errorf("database not available"))
			return err
		}

		if err := processHook(api.DBConnectionFactory.GetDBMap, api.Cache, rh); err != nil {
			hook.Recovery(api.Cache, rh, err)
			return err

		}

		return nil
	}
}

func (api *API) addHookHandler() Handler {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		vars := mux.Vars(r)
		projectKey := vars["key"]
		appName := vars["permApplicationName"]

		var h sdk.Hook
		if err := UnmarshalBody(r, &h); err != nil {
			return err
		}
		h.Enabled = true

		// Insert hook in database
		if err := hook.InsertHook(api.mustDB(), &h); err != nil {
			return sdk.WrapError(err, "addHook: cannot insert hook in db")

		}

		app, errA := application.LoadByName(api.mustDB(), api.Cache, projectKey, appName, getUser(ctx), application.LoadOptions.WithHooks)
		if errA != nil {
			return sdk.WrapError(errA, "addHook: Cannot load application")
		}
		var errW error
		app.Workflows, errW = workflowv0.LoadCDTree(api.mustDB(), api.Cache, projectKey, appName, getUser(ctx), "", "", 0)
		if errW != nil {
			return sdk.WrapError(errA, "addHook: Cannot load workflow")
		}

		return WriteJSON(w, r, app, http.StatusOK)
	}
}

func (api *API) updateHookHandler() Handler {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		vars := mux.Vars(r)
		projectKey := vars["key"]
		appName := vars["permApplicationName"]

		var h sdk.Hook
		if err := UnmarshalBody(r, &h); err != nil {
			return sdk.WrapError(err, "updateHookHandler")
		}

		app, errA := application.LoadByName(api.mustDB(), api.Cache, projectKey, appName, getUser(ctx), application.LoadOptions.WithHooks)
		if errA != nil {
			return sdk.WrapError(errA, "updateHookHandler> Cannot load application")
		}

		found := false
		for _, hookInApp := range app.Hooks {
			if hookInApp.ID == h.ID {
				found = true
				break
			}
		}

		if !found {
			return sdk.WrapError(sdk.ErrNoHook, "updateHookHandler")
		}

		// Update hook in database
		if err := hook.UpdateHook(api.mustDB(), h); err != nil {
			return sdk.WrapError(err, "updateHookHandler: cannot update hook")
		}

		var errW error
		app.Workflows, errW = workflowv0.LoadCDTree(api.mustDB(), api.Cache, projectKey, app.Name, getUser(ctx), "", "", 0)
		if errW != nil {
			return sdk.WrapError(errW, "updateHookHandler: Cannot load workflow")
		}

		return WriteJSON(w, r, app, http.StatusOK)
	}
}

func (api *API) getApplicationHooksHandler() Handler {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		vars := mux.Vars(r)
		projectName := vars["key"]
		appName := vars["permApplicationName"]

		a, err := application.LoadByName(api.mustDB(), api.Cache, projectName, appName, getUser(ctx), application.LoadOptions.WithHooks)
		if err != nil {
			return sdk.WrapError(err, "getApplicationHooksHandler> cannot load application %s/%s", projectName, appName)
		}

		return WriteJSON(w, r, a.Hooks, http.StatusOK)
	}
}

func (api *API) getHooksHandler() Handler {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		vars := mux.Vars(r)
		projectName := vars["key"]
		appName := vars["permApplicationName"]
		pipelineName := vars["permPipelineKey"]

		p, err := pipeline.LoadPipeline(api.mustDB(), projectName, pipelineName, false)
		if err != nil {
			if err != sdk.ErrPipelineNotFound {
				log.Warning("getHooks> cannot load pipeline %s/%s: %s\n", projectName, pipelineName, err)
			}
			return err
		}

		a, err := application.LoadByName(api.mustDB(), api.Cache, projectName, appName, getUser(ctx))
		if err != nil {
			return sdk.WrapError(err, "getHooks> cannot load application %s/%s", projectName, appName)
		}

		hooks, err := hook.LoadPipelineHooks(api.mustDB(), p.ID, a.ID)
		if err != nil {
			return sdk.WrapError(err, "getHooks> cannot load hooks")
		}

		return WriteJSON(w, r, hooks, http.StatusOK)
	}
}

func (api *API) deleteHookHandler() Handler {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		vars := mux.Vars(r)
		idS := vars["id"]

		id, err := strconv.ParseInt(idS, 10, 64)
		if err != nil {
			return sdk.ErrWrongRequest

		}

		_, err = hook.LoadHook(api.mustDB(), id)
		if err != nil {
			return sdk.WrapError(err, "deleteHook> cannot load hook")

		}

		err = hook.DeleteHook(api.mustDB(), id)
		if err != nil {
			return sdk.WrapError(err, "deleteHook> cannot delete hook")

		}
		return nil
	}
}

//hookRecoverer is the go-routine which catches on-error hook
func hookRecoverer(c context.Context, DBFunc func() *gorp.DbMap, store cache.Store) {
	tick := time.NewTicker(10 * time.Second).C
	for {
		select {
		case <-c.Done():
			if c.Err() != nil {
				log.Error("Exiting hookRecoverer: %v", c.Err())
				return
			}
		case <-tick:
			h := hook.ReceivedHook{}
			store.DequeueWithContext(c, "hook:recovery", &h)
			if err := c.Err(); err != nil {
				log.Error("Exiting hookRecoverer: %v", err)
				return
			}
			if h.Repository != "" {
				if err := processHook(DBFunc, store, h); err != nil {
					hook.Recovery(store, h, err)
				}
			}
		}
	}
}

//processHook is the core function for hook processing
func processHook(DBFunc func() *gorp.DbMap, store cache.Store, h hook.ReceivedHook) error {
	db := DBFunc()
	if db == nil {
		return fmt.Errorf("database not available")
	}

	// Logging stuff
	if err := hook.InsertReceivedHook(db, h.URL.String(), string(h.Data)); err != nil {
		return sdk.WrapError(err, "processHook> cannot insert received hook in db")
	}

	// Actual search of hook binding
	hooks, err := hook.LoadHooks(db, h.ProjectKey, h.Repository)
	if err != nil {
		return sdk.WrapError(err, "processHook> cannot load hook for %s/%s", h.ProjectKey, h.Repository)
	}

	// If branch is DELETE'd, remove all builds related to this branch
	if h.Message == "DELETE" {
		log.Warning("processHook> Removing builds in %s/%s on branch %s\n", h.ProjectKey, h.Repository, h.Branch)
		if err := hook.DeleteBranchBuilds(db, hooks, h.Branch); err != nil {
			return err
		}
		return nil
	}

	log.Debug("Executing %d hooks for %s/%s on branch %s\n", len(hooks), h.ProjectKey, h.Repository, h.Branch)
	found := false

	for i := range hooks {
		//begin a tx
		tx, err := db.Begin()
		if err != nil {
			return err
		}
		defer tx.Rollback()

		if !hooks[i].Enabled {
			continue
		}

		if hooks[i].UID != h.UID {
			continue
		}

		found = true

		// create pipeline object
		p, err := pipeline.LoadPipelineByID(tx, hooks[i].Pipeline.ID, true)
		if err != nil {
			return sdk.WrapError(err, "processHook> Cannot load pipeline")
		}

		// get Project
		// Load project
		projectData, err := project.LoadByPipelineID(tx, store, nil, p.ID)
		if err != nil {
			return sdk.WrapError(err, "processHook> Cannot load project for pipeline %s", p.Name)
		}

		projectsVar, err := project.GetAllVariableInProject(tx, projectData.ID)
		if err != nil {
			return sdk.WrapError(err, "processHook> Cannot load project variable")
		}
		projectData.Variable = projectsVar

		pb, err := application.TriggerPipeline(tx, store, hooks[i], h.Branch, h.Hash, h.Author, p, projectData)
		if err != nil {
			return sdk.WrapError(err, "processHook> cannot trigger pipeline %d", hooks[i].Pipeline.ID)
		}
		if pb != nil {
			log.Debug("processHook> Triggered %s/%s/%s", h.ProjectKey, h.Repository, h.Branch)
		} else {
			log.Info("processHook> Did not trigger %s/%s/%s", h.ProjectKey, h.Repository, h.Branch)
		}

		if err := tx.Commit(); err != nil {
			log.Error("processHook> Cannot commit tx; %s", err)
			return err
		}

		go func(h *sdk.Hook) {
			app, errapp := application.LoadByID(DBFunc(), store, h.ApplicationID, nil, application.LoadOptions.WithRepositoryManager)
			if errapp != nil {
				log.Warning("processHook> Unable to load application %s", errapp)
			}

			if _, err := pipeline.UpdatePipelineBuildCommits(DBFunc(), projectData, p, app, &sdk.DefaultEnv, pb); err != nil {
				log.Warning("processHook> Unable to update pipeline build commits: %s", err)
			}
		}(&hooks[i])
	}

	if !found {
		return sdk.WrapError(sdk.ErrUnauthorized, "processHook> Bad uid for hook [%s/%s], got uid='%s'", h.ProjectKey, h.Repository, h.UID)
	}

	return nil
}
<commit_msg>fix (api): process  hooks in goroutines (#1285)<commit_after>package api

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/go-gorp/gorp"
	"github.com/gorilla/mux"

	"github.com/ovh/cds/engine/api/application"
	"github.com/ovh/cds/engine/api/cache"
	"github.com/ovh/cds/engine/api/hook"
	"github.com/ovh/cds/engine/api/pipeline"
	"github.com/ovh/cds/engine/api/project"
	"github.com/ovh/cds/engine/api/workflowv0"
	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/log"
)

func processStashHook(w http.ResponseWriter, r *http.Request, data []byte) hook.ReceivedHook {
	rh := hook.ReceivedHook{
		URL:        *r.URL,
		Data:       data,
		ProjectKey: r.FormValue("project"),
		Repository: r.FormValue("name"),
		Branch:     r.FormValue("branch"),
		Hash:       r.FormValue("hash"),
		Author:     r.FormValue("author"),
		Message:    r.FormValue("message"),
		UID:        r.FormValue("uid"),
	}

	return rh
}

func processGitlabHook(w http.ResponseWriter, r *http.Request, data []byte) (hook.ReceivedHook, error) {

	type gitlabEvent struct {
		ObjectKind  string `json:"object_kind"`
		Ref         string `json:"ref"`
		UserName    string `json:"user_name"`
		CheckoutSha string `json:"checkout_sha"`
	}

	var ge gitlabEvent
	if err := json.Unmarshal(data, &ge); err != nil {
		return hook.ReceivedHook{}, err
	}

	ge.Ref = strings.TrimPrefix(ge.Ref, "refs/heads/")

	rh := hook.ReceivedHook{
		URL:        *r.URL,
		Data:       data,
		ProjectKey: r.FormValue("project"),
		Repository: r.FormValue("name"),
		Branch:     ge.Ref,
		Hash:       ge.CheckoutSha,
		Author:     ge.UserName,
		Message:    ge.ObjectKind,
		UID:        r.FormValue("uid"),
	}

	return rh, nil
}

func (api *API) receiveHookHandler() Handler {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		// Get body
		data, err := ioutil.ReadAll(r.Body)
		if err != nil {
			return sdk.ErrWrongRequest

		}

		if err = r.ParseForm(); err != nil {
			return sdk.WrapError(err, "receiveHook> cannot parse query params")
		}

		var rh hook.ReceivedHook
		if r.Header.Get("X-Gitlab-Event") != "" {
			rh, err = processGitlabHook(w, r, data)
			if err != nil {
				return err
			}
		} else {
			rh = processStashHook(w, r, data)
		}

		go func() {
			db := api.DBConnectionFactory.GetDBMap()
			if db == nil {
				err := fmt.Errorf("database not available")
				hook.Recovery(api.Cache, rh, err)
				log.Error("receiveHookHandler> Error, try to recover...: %v", err)
				return
			}

			if err := processHook(api.DBConnectionFactory.GetDBMap, api.Cache, rh); err != nil {
				log.Error("receiveHookHandler> Error, try to recover...: %v", err)
				hook.Recovery(api.Cache, rh, err)
			}
		}()

		return nil
	}
}

func (api *API) addHookHandler() Handler {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		vars := mux.Vars(r)
		projectKey := vars["key"]
		appName := vars["permApplicationName"]

		var h sdk.Hook
		if err := UnmarshalBody(r, &h); err != nil {
			return err
		}
		h.Enabled = true

		// Insert hook in database
		if err := hook.InsertHook(api.mustDB(), &h); err != nil {
			return sdk.WrapError(err, "addHook: cannot insert hook in db")

		}

		app, errA := application.LoadByName(api.mustDB(), api.Cache, projectKey, appName, getUser(ctx), application.LoadOptions.WithHooks)
		if errA != nil {
			return sdk.WrapError(errA, "addHook: Cannot load application")
		}
		var errW error
		app.Workflows, errW = workflowv0.LoadCDTree(api.mustDB(), api.Cache, projectKey, appName, getUser(ctx), "", "", 0)
		if errW != nil {
			return sdk.WrapError(errA, "addHook: Cannot load workflow")
		}

		return WriteJSON(w, r, app, http.StatusOK)
	}
}

func (api *API) updateHookHandler() Handler {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		vars := mux.Vars(r)
		projectKey := vars["key"]
		appName := vars["permApplicationName"]

		var h sdk.Hook
		if err := UnmarshalBody(r, &h); err != nil {
			return sdk.WrapError(err, "updateHookHandler")
		}

		app, errA := application.LoadByName(api.mustDB(), api.Cache, projectKey, appName, getUser(ctx), application.LoadOptions.WithHooks)
		if errA != nil {
			return sdk.WrapError(errA, "updateHookHandler> Cannot load application")
		}

		found := false
		for _, hookInApp := range app.Hooks {
			if hookInApp.ID == h.ID {
				found = true
				break
			}
		}

		if !found {
			return sdk.WrapError(sdk.ErrNoHook, "updateHookHandler")
		}

		// Update hook in database
		if err := hook.UpdateHook(api.mustDB(), h); err != nil {
			return sdk.WrapError(err, "updateHookHandler: cannot update hook")
		}

		var errW error
		app.Workflows, errW = workflowv0.LoadCDTree(api.mustDB(), api.Cache, projectKey, app.Name, getUser(ctx), "", "", 0)
		if errW != nil {
			return sdk.WrapError(errW, "updateHookHandler: Cannot load workflow")
		}

		return WriteJSON(w, r, app, http.StatusOK)
	}
}

func (api *API) getApplicationHooksHandler() Handler {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		vars := mux.Vars(r)
		projectName := vars["key"]
		appName := vars["permApplicationName"]

		a, err := application.LoadByName(api.mustDB(), api.Cache, projectName, appName, getUser(ctx), application.LoadOptions.WithHooks)
		if err != nil {
			return sdk.WrapError(err, "getApplicationHooksHandler> cannot load application %s/%s", projectName, appName)
		}

		return WriteJSON(w, r, a.Hooks, http.StatusOK)
	}
}

func (api *API) getHooksHandler() Handler {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		vars := mux.Vars(r)
		projectName := vars["key"]
		appName := vars["permApplicationName"]
		pipelineName := vars["permPipelineKey"]

		p, err := pipeline.LoadPipeline(api.mustDB(), projectName, pipelineName, false)
		if err != nil {
			if err != sdk.ErrPipelineNotFound {
				log.Warning("getHooks> cannot load pipeline %s/%s: %s\n", projectName, pipelineName, err)
			}
			return err
		}

		a, err := application.LoadByName(api.mustDB(), api.Cache, projectName, appName, getUser(ctx))
		if err != nil {
			return sdk.WrapError(err, "getHooks> cannot load application %s/%s", projectName, appName)
		}

		hooks, err := hook.LoadPipelineHooks(api.mustDB(), p.ID, a.ID)
		if err != nil {
			return sdk.WrapError(err, "getHooks> cannot load hooks")
		}

		return WriteJSON(w, r, hooks, http.StatusOK)
	}
}

func (api *API) deleteHookHandler() Handler {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		vars := mux.Vars(r)
		idS := vars["id"]

		id, err := strconv.ParseInt(idS, 10, 64)
		if err != nil {
			return sdk.ErrWrongRequest

		}

		_, err = hook.LoadHook(api.mustDB(), id)
		if err != nil {
			return sdk.WrapError(err, "deleteHook> cannot load hook")

		}

		err = hook.DeleteHook(api.mustDB(), id)
		if err != nil {
			return sdk.WrapError(err, "deleteHook> cannot delete hook")

		}
		return nil
	}
}

//hookRecoverer is the go-routine which catches on-error hook
func hookRecoverer(c context.Context, DBFunc func() *gorp.DbMap, store cache.Store) {
	tick := time.NewTicker(10 * time.Second).C
	for {
		select {
		case <-c.Done():
			if c.Err() != nil {
				log.Error("Exiting hookRecoverer: %v", c.Err())
				return
			}
		case <-tick:
			h := hook.ReceivedHook{}
			store.DequeueWithContext(c, "hook:recovery", &h)
			if err := c.Err(); err != nil {
				log.Error("Exiting hookRecoverer: %v", err)
				return
			}
			if h.Repository != "" {
				if err := processHook(DBFunc, store, h); err != nil {
					hook.Recovery(store, h, err)
				}
			}
		}
	}
}

//processHook is the core function for hook processing
func processHook(DBFunc func() *gorp.DbMap, store cache.Store, h hook.ReceivedHook) error {
	db := DBFunc()
	if db == nil {
		return fmt.Errorf("database not available")
	}

	// Logging stuff
	if err := hook.InsertReceivedHook(db, h.URL.String(), string(h.Data)); err != nil {
		return sdk.WrapError(err, "processHook> cannot insert received hook in db")
	}

	// Actual search of hook binding
	hooks, err := hook.LoadHooks(db, h.ProjectKey, h.Repository)
	if err != nil {
		return sdk.WrapError(err, "processHook> cannot load hook for %s/%s", h.ProjectKey, h.Repository)
	}

	// If branch is DELETE'd, remove all builds related to this branch
	if h.Message == "DELETE" {
		log.Warning("processHook> Removing builds in %s/%s on branch %s\n", h.ProjectKey, h.Repository, h.Branch)
		if err := hook.DeleteBranchBuilds(db, hooks, h.Branch); err != nil {
			return err
		}
		return nil
	}

	log.Debug("Executing %d hooks for %s/%s on branch %s\n", len(hooks), h.ProjectKey, h.Repository, h.Branch)
	found := false

	for i := range hooks {
		//begin a tx
		tx, err := db.Begin()
		if err != nil {
			return err
		}
		defer tx.Rollback()

		if !hooks[i].Enabled {
			continue
		}

		if hooks[i].UID != h.UID {
			continue
		}

		found = true

		// create pipeline object
		p, err := pipeline.LoadPipelineByID(tx, hooks[i].Pipeline.ID, true)
		if err != nil {
			return sdk.WrapError(err, "processHook> Cannot load pipeline")
		}

		// get Project
		// Load project
		projectData, err := project.LoadByPipelineID(tx, store, nil, p.ID)
		if err != nil {
			return sdk.WrapError(err, "processHook> Cannot load project for pipeline %s", p.Name)
		}

		projectsVar, err := project.GetAllVariableInProject(tx, projectData.ID)
		if err != nil {
			return sdk.WrapError(err, "processHook> Cannot load project variable")
		}
		projectData.Variable = projectsVar

		pb, err := application.TriggerPipeline(tx, store, hooks[i], h.Branch, h.Hash, h.Author, p, projectData)
		if err != nil {
			return sdk.WrapError(err, "processHook> cannot trigger pipeline %d", hooks[i].Pipeline.ID)
		}
		if pb != nil {
			log.Debug("processHook> Triggered %s/%s/%s", h.ProjectKey, h.Repository, h.Branch)
		} else {
			log.Info("processHook> Did not trigger %s/%s/%s", h.ProjectKey, h.Repository, h.Branch)
		}

		if err := tx.Commit(); err != nil {
			log.Error("processHook> Cannot commit tx; %s", err)
			return err
		}

		go func(h *sdk.Hook) {
			app, errapp := application.LoadByID(DBFunc(), store, h.ApplicationID, nil, application.LoadOptions.WithRepositoryManager)
			if errapp != nil {
				log.Warning("processHook> Unable to load application %s", errapp)
			}

			if _, err := pipeline.UpdatePipelineBuildCommits(DBFunc(), projectData, p, app, &sdk.DefaultEnv, pb); err != nil {
				log.Warning("processHook> Unable to update pipeline build commits: %s", err)
			}
		}(&hooks[i])
	}

	if !found {
		return sdk.WrapError(sdk.ErrUnauthorized, "processHook> Bad uid for hook [%s/%s], got uid='%s'", h.ProjectKey, h.Repository, h.UID)
	}

	return nil
}
<|endoftext|>


<commit_before>package models

import (
	"time"
)

const (
	STATUS_TESTING   = "testing"
	STATUS_REJECTED  = "rejected"
	STATUS_PUBLISHED = "published"
)

type BuildList struct {
	// represent unique id
	Id uint64 `xml:"id,attr" orm:"auto;pk"`
	// represent platform
	Platform string `xml:"info>platform"`
	// represent the repo it's being saved to
	Repo string `xml:"info>platform>repo"`
	// represent the arch
	Architecture string `xml:"info>platform>arch"`
	// represent its name
	Name string `xml:"info>name"`
	// represents the user
	Submitter *User `xml:"info>submitter" orm:"rel(fk)"`
	// represents the update type (bugfix, new package, security, etc)
	Type string `xml:"type"`
	// represents the current status of this update
	Status string `xml:"status"`
	// lists packages
	Packages []*BuildListPkg `xml:"packages" orm:"reverse(many)"`
	// lists the changelog url
	Changelog string `xml:"changelog" orm:"type(text)"` // url
	// lists the builddate
	BuildDate time.Time `xml:"time"`
	// lists the updatetime
	Updated time.Time `xml:"updated" orm:"auto_now"`
	// lists the karma
	Karma []*Karma `xml:"karma" orm:"reverse(many)"`
	// shows the diff
	Diff string `xml:"diff" orm:"type(text)"`
	// shows the link to an advisory, if any
	Advisory *Advisory `xml:"advisory" orm:"rel(fk)"`

	// abf specifics (abf is represented as the handler)
	HandleId      string `xml:"handle>id,attr"` // for the handler to identify the package in the buildsystem
	HandleProject string `xml:"handle>project" orm:"type(text)"`
	// end handler specifics
}

type BuildListPkg struct {
	Id      uint64     `orm:"auto;pk"`
	List    *BuildList `orm:"rel(fk)"`
	Name    string
	Type    string
	Epoch   int64
	Version string
	Release string
	Url     string `orm:"type(text)"`
}
<commit_msg>modification to advisory flags<commit_after>package models

import (
	"time"
)

const (
	STATUS_TESTING   = "testing"
	STATUS_REJECTED  = "rejected"
	STATUS_PUBLISHED = "published"
)

type BuildList struct {
	// represent unique id
	Id uint64 `xml:"id,attr" orm:"auto;pk"`
	// represent platform
	Platform string `xml:"info>platform"`
	// represent the repo it's being saved to
	Repo string `xml:"info>platform>repo"`
	// represent the arch
	Architecture string `xml:"info>platform>arch"`
	// represent its name
	Name string `xml:"info>name"`
	// represents the user
	Submitter *User `xml:"info>submitter" orm:"rel(fk)"`
	// represents the update type (bugfix, new package, security, etc)
	Type string `xml:"type"`
	// represents the current status of this update
	Status string `xml:"status"`
	// lists packages
	Packages []*BuildListPkg `xml:"packages" orm:"reverse(many)"`
	// lists the changelog url
	Changelog string `xml:"changelog" orm:"type(text)"` // url
	// lists the builddate
	BuildDate time.Time `xml:"time"`
	// lists the updatetime
	Updated time.Time `xml:"updated" orm:"auto_now"`
	// lists the karma
	Karma []*Karma `xml:"karma" orm:"reverse(many)"`
	// shows the diff
	Diff string `xml:"diff" orm:"type(text)"`
	// shows the link to an advisory, if any
	Advisory *Advisory `xml:"advisory" orm:"null;rel(fk);on_delete(set_null)"`

	// abf specifics (abf is represented as the handler)
	HandleId      string `xml:"handle>id,attr"` // for the handler to identify the package in the buildsystem
	HandleProject string `xml:"handle>project" orm:"type(text)"`
	// end handler specifics
}

type BuildListPkg struct {
	Id      uint64     `orm:"auto;pk"`
	List    *BuildList `orm:"rel(fk)"`
	Name    string
	Type    string
	Epoch   int64
	Version string
	Release string
	Url     string `orm:"type(text)"`
}
<|endoftext|>


<commit_before>// Copyright 2017 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package models

import (
	"fmt"
	"strings"

	"github.com/go-xorm/builder"
	"github.com/go-xorm/xorm"
)

// RepositoryList contains a list of repositories
type RepositoryList []*Repository

func (repos RepositoryList) loadAttributes(e Engine) error {
	if len(repos) == 0 {
		return nil
	}

	// Load owners.
	set := make(map[int64]*User)
	for i := range repos {
		set[repos[i].OwnerID] = nil
	}
	userIDs := make([]int64, 0, len(set))
	for userID := range set {
		userIDs = append(userIDs, userID)
	}
	users := make([]*User, 0, len(userIDs))
	if err := e.
		Where("id > 0").
		In("id", userIDs).
		Find(&users); err != nil {
		return fmt.Errorf("find users: %v", err)
	}
	for i := range users {
		set[users[i].ID] = users[i]
	}
	for i := range repos {
		repos[i].Owner = set[repos[i].OwnerID]
	}
	return nil
}

// LoadAttributes loads the attributes for the given RepositoryList
func (repos RepositoryList) LoadAttributes() error {
	return repos.loadAttributes(x)
}

// MirrorRepositoryList contains the mirror repositories
type MirrorRepositoryList []*Repository

func (repos MirrorRepositoryList) loadAttributes(e Engine) error {
	if len(repos) == 0 {
		return nil
	}

	// Load mirrors.
	repoIDs := make([]int64, 0, len(repos))
	for i := range repos {
		if !repos[i].IsMirror {
			continue
		}

		repoIDs = append(repoIDs, repos[i].ID)
	}
	mirrors := make([]*Mirror, 0, len(repoIDs))
	if err := e.
		Where("id > 0").
		In("repo_id", repoIDs).
		Find(&mirrors); err != nil {
		return fmt.Errorf("find mirrors: %v", err)
	}

	set := make(map[int64]*Mirror)
	for i := range mirrors {
		set[mirrors[i].RepoID] = mirrors[i]
	}
	for i := range repos {
		repos[i].Mirror = set[repos[i].ID]
	}
	return nil
}

// LoadAttributes loads the attributes for the given MirrorRepositoryList
func (repos MirrorRepositoryList) LoadAttributes() error {
	return repos.loadAttributes(x)
}

// SearchRepoOptions holds the search options
type SearchRepoOptions struct {
	Keyword   string
	OwnerID   int64
	Searcher  *User //ID of the person who's seeking
	OrderBy   string
	Private   bool // Include private repositories in results
	Starred   bool
	Page      int
	IsProfile bool
	PageSize  int // Can be smaller than or equal to setting.ExplorePagingNum
}

// SearchRepositoryByName takes keyword and part of repository name to search,
// it returns results in given range and number of total results.
func SearchRepositoryByName(opts *SearchRepoOptions) (repos RepositoryList, count int64, err error) {
	var (
		sess *xorm.Session
		cond = builder.NewCond()
	)

	if len(opts.Keyword) == 0 {
		return repos, 0, nil
	}
	opts.Keyword = strings.ToLower(opts.Keyword)

	if opts.Page <= 0 {
		opts.Page = 1
	}

	repos = make([]*Repository, 0, opts.PageSize)

	if opts.Starred && opts.OwnerID > 0 {
		cond = builder.Eq{
			"star.uid": opts.OwnerID,
		}
	}
	cond = cond.And(builder.Like{"lower_name", opts.Keyword})

	// Append conditions
	if !opts.Starred && opts.OwnerID > 0 {
		cond = cond.And(builder.Eq{"owner_id": opts.OwnerID})
	}
	if !opts.Private {
		cond = cond.And(builder.Eq{"is_private": false})
	}

	if opts.Searcher != nil {
		var ownerIds []int64

		ownerIds = append(ownerIds, opts.Searcher.ID)
		err = opts.Searcher.GetOrganizations(true)

		if err != nil {
			return nil, 0, fmt.Errorf("Organization: %v", err)
		}

		for _, org := range opts.Searcher.Orgs {
			ownerIds = append(ownerIds, org.ID)
		}

		cond = cond.Or(builder.And(builder.Like{"lower_name", opts.Keyword}, builder.In("owner_id", ownerIds)))
	}

	if len(opts.OrderBy) == 0 {
		opts.OrderBy = "name ASC"
	}

	if opts.Starred && opts.OwnerID > 0 {
		sess = x.
			Join("INNER", "star", "star.repo_id = repository.id").
			Where(cond)
		count, err = x.
			Join("INNER", "star", "star.repo_id = repository.id").
			Where(cond).
			Count(new(Repository))
		if err != nil {
			return nil, 0, fmt.Errorf("Count: %v", err)
		}
	} else {
		sess = x.Where(cond)
		count, err = x.
			Where(cond).
			Count(new(Repository))
		if err != nil {
			return nil, 0, fmt.Errorf("Count: %v", err)
		}
	}

	if err = sess.
		Limit(opts.PageSize, (opts.Page-1)*opts.PageSize).
		OrderBy(opts.OrderBy).
		Find(&repos); err != nil {
		return nil, 0, fmt.Errorf("Repo: %v", err)
	}

	if !opts.IsProfile {
		if err = repos.loadAttributes(x); err != nil {
			return nil, 0, fmt.Errorf("LoadAttributes: %v", err)
		}
	}

	return
}

// Repositories returns all repositories
func Repositories(opts *SearchRepoOptions) (_ RepositoryList, count int64, err error) {
	if len(opts.OrderBy) == 0 {
		opts.OrderBy = "id ASC"
	}

	repos := make(RepositoryList, 0, opts.PageSize)

	if err = x.
		Limit(opts.PageSize, (opts.Page-1)*opts.PageSize).
		OrderBy(opts.OrderBy).
		Find(&repos); err != nil {
		return nil, 0, fmt.Errorf("Repo: %v", err)
	}

	if err = repos.loadAttributes(x); err != nil {
		return nil, 0, fmt.Errorf("LoadAttributes: %v", err)
	}

	count = countRepositories(-1, opts.Private)

	return repos, count, nil
}

// GetRecentUpdatedRepositories returns the list of repositories that are recently updated.
func GetRecentUpdatedRepositories(opts *SearchRepoOptions) (repos RepositoryList, _ int64, _ error) {
	var cond = builder.NewCond()

	if len(opts.OrderBy) == 0 {
		opts.OrderBy = "updated_unix DESC"
	}

	if !opts.Private {
		cond = builder.Eq{
			"is_private": false,
		}
	}

	if opts.Searcher != nil && !opts.Searcher.IsAdmin {
		var ownerIds []int64

		ownerIds = append(ownerIds, opts.Searcher.ID)
		err := opts.Searcher.GetOrganizations(true)

		if err != nil {
			return nil, 0, fmt.Errorf("Organization: %v", err)
		}

		for _, org := range opts.Searcher.Orgs {
			ownerIds = append(ownerIds, org.ID)
		}

		cond = cond.Or(builder.In("owner_id", ownerIds))
	}

	count, err := x.Where(cond).Count(new(Repository))
	if err != nil {
		return nil, 0, fmt.Errorf("Count: %v", err)
	}

	if err = x.Where(cond).
		Limit(opts.PageSize, (opts.Page-1)*opts.PageSize).
		Limit(opts.PageSize).
		OrderBy(opts.OrderBy).
		Find(&repos); err != nil {
		return nil, 0, fmt.Errorf("Repo: %v", err)
	}

	if err = repos.loadAttributes(x); err != nil {
		return nil, 0, fmt.Errorf("LoadAttributes: %v", err)
	}

	return repos, count, nil
}
<commit_msg>Simplify RepositoryList.loadAttributes() (#1211)<commit_after>// Copyright 2017 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package models

import (
	"fmt"
	"strings"

	"github.com/go-xorm/builder"
	"github.com/go-xorm/xorm"
)

// RepositoryList contains a list of repositories
type RepositoryList []*Repository

func (repos RepositoryList) loadAttributes(e Engine) error {
	if len(repos) == 0 {
		return nil
	}

	// Load owners.
	set := make(map[int64]struct{})
	for i := range repos {
		set[repos[i].OwnerID] = struct{}{}
	}
	users := make(map[int64]*User, len(set))
	if err := e.
		Where("id > 0").
		In("id", keysInt64(set)).
		Find(&users); err != nil {
		return fmt.Errorf("find users: %v", err)
	}
	for i := range repos {
		repos[i].Owner = users[repos[i].OwnerID]
	}
	return nil
}

// LoadAttributes loads the attributes for the given RepositoryList
func (repos RepositoryList) LoadAttributes() error {
	return repos.loadAttributes(x)
}

// MirrorRepositoryList contains the mirror repositories
type MirrorRepositoryList []*Repository

func (repos MirrorRepositoryList) loadAttributes(e Engine) error {
	if len(repos) == 0 {
		return nil
	}

	// Load mirrors.
	repoIDs := make([]int64, 0, len(repos))
	for i := range repos {
		if !repos[i].IsMirror {
			continue
		}

		repoIDs = append(repoIDs, repos[i].ID)
	}
	mirrors := make([]*Mirror, 0, len(repoIDs))
	if err := e.
		Where("id > 0").
		In("repo_id", repoIDs).
		Find(&mirrors); err != nil {
		return fmt.Errorf("find mirrors: %v", err)
	}

	set := make(map[int64]*Mirror)
	for i := range mirrors {
		set[mirrors[i].RepoID] = mirrors[i]
	}
	for i := range repos {
		repos[i].Mirror = set[repos[i].ID]
	}
	return nil
}

// LoadAttributes loads the attributes for the given MirrorRepositoryList
func (repos MirrorRepositoryList) LoadAttributes() error {
	return repos.loadAttributes(x)
}

// SearchRepoOptions holds the search options
type SearchRepoOptions struct {
	Keyword   string
	OwnerID   int64
	Searcher  *User //ID of the person who's seeking
	OrderBy   string
	Private   bool // Include private repositories in results
	Starred   bool
	Page      int
	IsProfile bool
	PageSize  int // Can be smaller than or equal to setting.ExplorePagingNum
}

// SearchRepositoryByName takes keyword and part of repository name to search,
// it returns results in given range and number of total results.
func SearchRepositoryByName(opts *SearchRepoOptions) (repos RepositoryList, count int64, err error) {
	var (
		sess *xorm.Session
		cond = builder.NewCond()
	)

	if len(opts.Keyword) == 0 {
		return repos, 0, nil
	}
	opts.Keyword = strings.ToLower(opts.Keyword)

	if opts.Page <= 0 {
		opts.Page = 1
	}

	repos = make([]*Repository, 0, opts.PageSize)

	if opts.Starred && opts.OwnerID > 0 {
		cond = builder.Eq{
			"star.uid": opts.OwnerID,
		}
	}
	cond = cond.And(builder.Like{"lower_name", opts.Keyword})

	// Append conditions
	if !opts.Starred && opts.OwnerID > 0 {
		cond = cond.And(builder.Eq{"owner_id": opts.OwnerID})
	}
	if !opts.Private {
		cond = cond.And(builder.Eq{"is_private": false})
	}

	if opts.Searcher != nil {
		var ownerIds []int64

		ownerIds = append(ownerIds, opts.Searcher.ID)
		err = opts.Searcher.GetOrganizations(true)

		if err != nil {
			return nil, 0, fmt.Errorf("Organization: %v", err)
		}

		for _, org := range opts.Searcher.Orgs {
			ownerIds = append(ownerIds, org.ID)
		}

		cond = cond.Or(builder.And(builder.Like{"lower_name", opts.Keyword}, builder.In("owner_id", ownerIds)))
	}

	if len(opts.OrderBy) == 0 {
		opts.OrderBy = "name ASC"
	}

	if opts.Starred && opts.OwnerID > 0 {
		sess = x.
			Join("INNER", "star", "star.repo_id = repository.id").
			Where(cond)
		count, err = x.
			Join("INNER", "star", "star.repo_id = repository.id").
			Where(cond).
			Count(new(Repository))
		if err != nil {
			return nil, 0, fmt.Errorf("Count: %v", err)
		}
	} else {
		sess = x.Where(cond)
		count, err = x.
			Where(cond).
			Count(new(Repository))
		if err != nil {
			return nil, 0, fmt.Errorf("Count: %v", err)
		}
	}

	if err = sess.
		Limit(opts.PageSize, (opts.Page-1)*opts.PageSize).
		OrderBy(opts.OrderBy).
		Find(&repos); err != nil {
		return nil, 0, fmt.Errorf("Repo: %v", err)
	}

	if !opts.IsProfile {
		if err = repos.loadAttributes(x); err != nil {
			return nil, 0, fmt.Errorf("LoadAttributes: %v", err)
		}
	}

	return
}

// Repositories returns all repositories
func Repositories(opts *SearchRepoOptions) (_ RepositoryList, count int64, err error) {
	if len(opts.OrderBy) == 0 {
		opts.OrderBy = "id ASC"
	}

	repos := make(RepositoryList, 0, opts.PageSize)

	if err = x.
		Limit(opts.PageSize, (opts.Page-1)*opts.PageSize).
		OrderBy(opts.OrderBy).
		Find(&repos); err != nil {
		return nil, 0, fmt.Errorf("Repo: %v", err)
	}

	if err = repos.loadAttributes(x); err != nil {
		return nil, 0, fmt.Errorf("LoadAttributes: %v", err)
	}

	count = countRepositories(-1, opts.Private)

	return repos, count, nil
}

// GetRecentUpdatedRepositories returns the list of repositories that are recently updated.
func GetRecentUpdatedRepositories(opts *SearchRepoOptions) (repos RepositoryList, _ int64, _ error) {
	var cond = builder.NewCond()

	if len(opts.OrderBy) == 0 {
		opts.OrderBy = "updated_unix DESC"
	}

	if !opts.Private {
		cond = builder.Eq{
			"is_private": false,
		}
	}

	if opts.Searcher != nil && !opts.Searcher.IsAdmin {
		var ownerIds []int64

		ownerIds = append(ownerIds, opts.Searcher.ID)
		err := opts.Searcher.GetOrganizations(true)

		if err != nil {
			return nil, 0, fmt.Errorf("Organization: %v", err)
		}

		for _, org := range opts.Searcher.Orgs {
			ownerIds = append(ownerIds, org.ID)
		}

		cond = cond.Or(builder.In("owner_id", ownerIds))
	}

	count, err := x.Where(cond).Count(new(Repository))
	if err != nil {
		return nil, 0, fmt.Errorf("Count: %v", err)
	}

	if err = x.Where(cond).
		Limit(opts.PageSize, (opts.Page-1)*opts.PageSize).
		Limit(opts.PageSize).
		OrderBy(opts.OrderBy).
		Find(&repos); err != nil {
		return nil, 0, fmt.Errorf("Repo: %v", err)
	}

	if err = repos.loadAttributes(x); err != nil {
		return nil, 0, fmt.Errorf("LoadAttributes: %v", err)
	}

	return repos, count, nil
}
<|endoftext|>


<commit_before>package dns

import "github.com/chzyer/adrs/utils"

// The second segment of query header section
type DNSHeaderOption struct {
	QR     H_QR
	OpCode H_OPCODE
	AA     H_AA
	TC     H_TC
	RD     H_RD
	RA     H_RA
	Z      H_Z
	Rcode  H_RCODE
}

func NewDNSHeaderOption(option uint64) *DNSHeaderOption {
	return &DNSHeaderOption{
		QR:     H_QR(utils.Read8Bit(option, 15, 1)),
		OpCode: H_OPCODE(utils.Read8Bit(option, 11, 4)),
		AA:     H_AA(utils.ReadBitBool(option, 10)),
		TC:     H_TC(utils.ReadBitBool(option, 9)),
		RD:     H_RD(utils.ReadBitBool(option, 8)),
		RA:     H_RA(utils.ReadBitBool(option, 7)),
		Z:      H_Z(utils.Read8Bit(option, 4, 3)),
		Rcode:  H_RCODE(utils.Read8Bit(option, 0, 4)),
	}
}

// A one bit field that specifies whether this message is a
// query (0), or a response (1).
type H_QR int

const (
	QR_QUERY H_QR = 0
	QR_RESP       = 1
)

// A four bit field that specifies kind of query in this
// message.  This value is set by the originator of a query
// and copied into the response.  The values are:
//     0               a standard query (QUERY)
//     1               an inverse query (IQUERY)
//     2               a server status request (STATUS)
//     3-15            reserved for future use
type H_OPCODE int

const (
	OPCODE_QUERY  H_OPCODE = 0
	OPCODE_IQUERY          = 1
	OPCODE_STATUS          = 2
)

// Authoritative Answer - this bit is valid in responses,
// and specifies that the responding name server is an
// authority for the domain name in question section.
//
// Note that the contents of the answer section may have
// multiple owner names because of aliases.  The AA bit
// corresponds to the name which matches the query name, or
// the first owner name in the answer section.
type H_AA bool

// TrunCation - specifies that this message was truncated
// due to length greater than that permitted on the
// transmission channel.
type H_TC bool

// Recursion Desired - this bit may be set in a query and
// is copied into the response.  If RD is set, it directs
// the name server to pursue the query recursively.
// Recursive query support is optional.
type H_RD bool

// Recursion Available - this be is set or cleared in a
// response, and denotes whether recursive query support is
// available in the name server.
type H_RA bool

// Reserved for future use.  Must be zero in all queries
// and responses.
type H_Z int

// Response code - this 4 bit field is set as part of
// responses.  The values have the following
// interpretation:
//  0               No error condition
//  1               Format error - The name server was
//                  unable to interpret the query.
//  2               Server failure - The name server was
//                  unable to process this query due to a
//                  problem with the name server.
//  3               Name Error - Meaningful only for
//                  responses from an authoritative name
//                  server, this code signifies that the
//                  domain name referenced in the query does
//                  not exist.
//  4               Not Implemented - The name server does
//                  not support the requested kind of query.
//  5               Refused - The name server refuses to
//                  perform the specified operation for
//                  policy reasons.  For example, a name
//                  server may not wish to provide the
//                  information to the particular requester,
//                  or a name server may not wish to perform
//                  a particular operation (e.g., zone
//                  transfer) for particular data.
//  6-15            Reserved for future use.
type H_RCODE int

const (
	RCODE_NO_ERR   H_RCODE = 0
	RCODE_FMT_ERR          = 1
	RCODE_SVR_FAIL         = 2
	RCODE_NAME_ERR         = 3
	RCODE_NOT_IMP          = 4
	RCODE_REFUSED          = 5
	RCODE_RESERVED         = 6
)
<commit_msg>rename header option type name<commit_after>package dns

import "github.com/chzyer/adrs/utils"

// The second segment of query header section
type DNSHeaderOption struct {
	QR     QR
	OpCode OPCODE
	AA     AA
	TC     TC
	RD     RD
	RA     RA
	Z      Z
	Rcode  RCODE
}

func NewDNSHeaderOption(option uint64) *DNSHeaderOption {
	return &DNSHeaderOption{
		QR:     QR(utils.Read8Bit(option, 15, 1)),
		OpCode: OPCODE(utils.Read8Bit(option, 11, 4)),
		AA:     AA(utils.ReadBitBool(option, 10)),
		TC:     TC(utils.ReadBitBool(option, 9)),
		RD:     RD(utils.ReadBitBool(option, 8)),
		RA:     RA(utils.ReadBitBool(option, 7)),
		Z:      Z(utils.Read8Bit(option, 4, 3)),
		Rcode:  RCODE(utils.Read8Bit(option, 0, 4)),
	}
}

// A one bit field that specifies whether this message is a
// query (0), or a response (1).
type QR int

const (
	QR_QUERY QR = 0
	QR_RESP     = 1
)

// A four bit field that specifies kind of query in this
// message.  This value is set by the originator of a query
// and copied into the response.  The values are:
//     0               a standard query (QUERY)
//     1               an inverse query (IQUERY)
//     2               a server status request (STATUS)
//     3-15            reserved for future use
type OPCODE int

const (
	OPCODE_QUERY  OPCODE = 0
	OPCODE_IQUERY        = 1
	OPCODE_STATUS        = 2
)

// Authoritative Answer - this bit is valid in responses,
// and specifies that the responding name server is an
// authority for the domain name in question section.
//
// Note that the contents of the answer section may have
// multiple owner names because of aliases.  The AA bit
// corresponds to the name which matches the query name, or
// the first owner name in the answer section.
type AA bool

// TrunCation - specifies that this message was truncated
// due to length greater than that permitted on the
// transmission channel.
type TC bool

// Recursion Desired - this bit may be set in a query and
// is copied into the response.  If RD is set, it directs
// the name server to pursue the query recursively.
// Recursive query support is optional.
type RD bool

// Recursion Available - this be is set or cleared in a
// response, and denotes whether recursive query support is
// available in the name server.
type RA bool

// Reserved for future use.  Must be zero in all queries
// and responses.
type Z int

// Response code - this 4 bit field is set as part of
// responses.  The values have the following
// interpretation:
//  0               No error condition
//  1               Format error - The name server was
//                  unable to interpret the query.
//  2               Server failure - The name server was
//                  unable to process this query due to a
//                  problem with the name server.
//  3               Name Error - Meaningful only for
//                  responses from an authoritative name
//                  server, this code signifies that the
//                  domain name referenced in the query does
//                  not exist.
//  4               Not Implemented - The name server does
//                  not support the requested kind of query.
//  5               Refused - The name server refuses to
//                  perform the specified operation for
//                  policy reasons.  For example, a name
//                  server may not wish to provide the
//                  information to the particular requester,
//                  or a name server may not wish to perform
//                  a particular operation (e.g., zone
//                  transfer) for particular data.
//  6-15            Reserved for future use.
type RCODE int

const (
	RCODE_NO_ERR   RCODE = 0
	RCODE_FMT_ERR        = 1
	RCODE_SVR_FAIL       = 2
	RCODE_NAME_ERR       = 3
	RCODE_NOT_IMP        = 4
	RCODE_REFUSED        = 5
	RCODE_RESERVED       = 6
)
<|endoftext|>


<commit_before>package entities

import "github.com/powerman/rpc-codec/jsonrpc2"

//Global error codes
var (
	ErrTimeout      = jsonrpc2.NewError(0, "TIMEOUT")
	ErrForbidden    = jsonrpc2.NewError(3, "FORBIDDEN")
	ErrMaintenance  = jsonrpc2.NewError(4, "MAINTENANCE")
	ErrRemoteServer = jsonrpc2.NewError(5, "REMOTE_SERVER_UNAVAILABLE")
)

//Tracker services error codes
var (
	ErrCredentials     = jsonrpc2.NewError(102, "INVALID_CREDENTIALS")
	ErrTrackerType     = jsonrpc2.NewError(103, "INVALID_TRACKER_TYPE")
	ErrTrackerURL      = jsonrpc2.NewError(104, "INVALID_TRACKER_URL")
	ErrIssueURL        = jsonrpc2.NewError(105, "INVALID_ISSUE_URL")
	ErrProjectNotFound = jsonrpc2.NewError(106, "PROJECT_NOT_FOUND")
	ErrIssueNotFound   = jsonrpc2.NewError(107, "ISSUE_NOT_FOUND")
)

const (
	trackerValidationErrCode = 101
	trackerValidationErrMsg  = "TRACKER_VALIDATION_ERROR"
)

//NewTrackerValidationErr return new tracker validation error with message
func NewTrackerValidationErr(msg string) error {
	return &jsonrpc2.Error{
		Code:    trackerValidationErrCode,
		Message: trackerValidationErrMsg,
		Data:    msg,
	}
}
<commit_msg>added method for creation internal errors from status codes<commit_after>package entities

import (
	"github.com/powerman/rpc-codec/jsonrpc2"
	"net/http"
)

//Global error codes
var (
	ErrTimeout      = jsonrpc2.NewError(0, "TIMEOUT")
	ErrForbidden    = jsonrpc2.NewError(3, "FORBIDDEN")
	ErrMaintenance  = jsonrpc2.NewError(4, "MAINTENANCE")
	ErrRemoteServer = jsonrpc2.NewError(5, "REMOTE_SERVER_UNAVAILABLE")
)

//Tracker services error codes
var (
	ErrCredentials     = jsonrpc2.NewError(102, "INVALID_CREDENTIALS")
	ErrTrackerType     = jsonrpc2.NewError(103, "INVALID_TRACKER_TYPE")
	ErrTrackerURL      = jsonrpc2.NewError(104, "INVALID_TRACKER_URL")
	ErrIssueURL        = jsonrpc2.NewError(105, "INVALID_ISSUE_URL")
	ErrProjectNotFound = jsonrpc2.NewError(106, "PROJECT_NOT_FOUND")
	ErrIssueNotFound   = jsonrpc2.NewError(107, "ISSUE_NOT_FOUND")
)

const (
	trackerValidationErrCode = 101
	trackerValidationErrMsg  = "TRACKER_VALIDATION_ERROR"
)

//NewTrackerValidationErr return new tracker validation error with message
func NewTrackerValidationErr(msg string) error {
	return &jsonrpc2.Error{
		Code:    trackerValidationErrCode,
		Message: trackerValidationErrMsg,
		Data:    msg,
	}
}

func ErrorForStatusCode(code int) error {
	switch code {
	case http.StatusRequestTimeout:
		return ErrTimeout
	case http.StatusForbidden:
		return ErrForbidden
	case http.StatusUnauthorized:
		return ErrCredentials
	case http.StatusServiceUnavailable:
		return ErrRemoteServer
	default:
		return nil
	}
}
<|endoftext|>


<commit_before>package fix

import (
	"bytes"
	"github.com/ghthor/journal/git"
	"io"
	"io/ioutil"
	"os"
)

type entry interface {
	// Needs Fixed
	needsFixed() bool

	// Return an Entry that has been fixed
	FixedEntry() (entry, git.Commitable, error)

	// Returns a byte slice of the entry w/o fixes
	Bytes() []byte

	// Returns an io.Reader for the entry w/o fixes
	NewReader() io.Reader
}

type entryCaseNeedsFixed struct {
	bytes []byte
	fixes []entryFix
}

func (e entryCaseNeedsFixed) needsFixed() bool { return len(e.fixes) > 0 }
func (e entryCaseNeedsFixed) FixedEntry() (entry, git.Commitable, error) {
	var (
		data []byte = e.bytes
		err  error
	)

	for _, fix := range e.fixes {
		data, err = fix.Execute(bytes.NewReader(data))
		if err != nil {
			return nil, nil, err
		}
	}

	return entryCaseCurrent{data}, git.Changes{
		Msg: "entry - format updated",
	}, nil
}

func (e entryCaseNeedsFixed) Bytes() []byte { return e.bytes }
func (e entryCaseNeedsFixed) NewReader() io.Reader {
	return bytes.NewReader(e.bytes)
}

type entryCaseCurrent struct {
	bytes []byte
}

func (e entryCaseCurrent) needsFixed() bool { return false }
func (e entryCaseCurrent) FixedEntry() (entry, git.Commitable, error) {
	return e, nil, nil
}
func (e entryCaseCurrent) Bytes() []byte        { return e.bytes }
func (e entryCaseCurrent) NewReader() io.Reader { return bytes.NewReader(e.bytes) }

func findErrorsInEntry(r io.Reader) (fixes []entryFix, err error) {
	data, err := ioutil.ReadAll(r)
	if err != nil {
		return nil, err
	}

	for _, fix := range entryFixes {
		if needsFixed, err := fix.CanFix(bytes.NewReader(data)); err != nil {
			return nil, err
		} else if needsFixed {
			fixes = append(fixes, fix)
		}
	}
	return
}

func newEntry(r io.Reader) (entry, error) {
	data, err := ioutil.ReadAll(r)
	if err != nil {
		return nil, err
	}

	if fixes, err := findErrorsInEntry(bytes.NewReader(data)); err != nil {
		return nil, err
	} else if fixes != nil {
		return entryCaseNeedsFixed{data, fixes}, nil
	}

	return entryCaseCurrent{data}, nil
}

func newEntryFromFile(filepath string) (entry, error) {
	f, err := os.OpenFile(filepath, os.O_RDONLY, 0600)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	return newEntry(f)
}
<commit_msg>Removed unnecessary and clarified doc comments<commit_after>package fix

import (
	"bytes"
	"github.com/ghthor/journal/git"
	"io"
	"io/ioutil"
	"os"
)

type entry interface {
	needsFixed() bool

	// Return an Entry that has been fixed
	FixedEntry() (entry, git.Commitable, error)

	// Returns a byte slice of the entry w/o fixes applied
	Bytes() []byte

	// Returns an io.Reader for the entry w/o fixes applied
	NewReader() io.Reader
}

type entryCaseNeedsFixed struct {
	bytes []byte
	fixes []entryFix
}

func (e entryCaseNeedsFixed) needsFixed() bool { return len(e.fixes) > 0 }
func (e entryCaseNeedsFixed) FixedEntry() (entry, git.Commitable, error) {
	var (
		data []byte = e.bytes
		err  error
	)

	for _, fix := range e.fixes {
		data, err = fix.Execute(bytes.NewReader(data))
		if err != nil {
			return nil, nil, err
		}
	}

	return entryCaseCurrent{data}, git.Changes{
		Msg: "entry - format updated",
	}, nil
}

func (e entryCaseNeedsFixed) Bytes() []byte { return e.bytes }
func (e entryCaseNeedsFixed) NewReader() io.Reader {
	return bytes.NewReader(e.bytes)
}

type entryCaseCurrent struct {
	bytes []byte
}

func (e entryCaseCurrent) needsFixed() bool { return false }
func (e entryCaseCurrent) FixedEntry() (entry, git.Commitable, error) {
	return e, nil, nil
}
func (e entryCaseCurrent) Bytes() []byte        { return e.bytes }
func (e entryCaseCurrent) NewReader() io.Reader { return bytes.NewReader(e.bytes) }

func findErrorsInEntry(r io.Reader) (fixes []entryFix, err error) {
	data, err := ioutil.ReadAll(r)
	if err != nil {
		return nil, err
	}

	for _, fix := range entryFixes {
		if needsFixed, err := fix.CanFix(bytes.NewReader(data)); err != nil {
			return nil, err
		} else if needsFixed {
			fixes = append(fixes, fix)
		}
	}
	return
}

func newEntry(r io.Reader) (entry, error) {
	data, err := ioutil.ReadAll(r)
	if err != nil {
		return nil, err
	}

	if fixes, err := findErrorsInEntry(bytes.NewReader(data)); err != nil {
		return nil, err
	} else if fixes != nil {
		return entryCaseNeedsFixed{data, fixes}, nil
	}

	return entryCaseCurrent{data}, nil
}

func newEntryFromFile(filepath string) (entry, error) {
	f, err := os.OpenFile(filepath, os.O_RDONLY, 0600)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	return newEntry(f)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"sync"
	"time"

	"github.com/BurntSushi/toml"
)

type status struct {
	Name   string `toml:"name"`
	Type   string `toml:"type"`
	Status string
	URL    string `toml:"url"`
}

type sites struct {
	Service []status
}

type Status interface {
	Check(url string) (bool, error)
}

var check map[string]Status

type server struct {
	configfile  string
	lastconfig  time.Time
	htmlfile    string
	templ       *template.Template
	site_status []status
	next_status time.Time
	last_status time.Time
	html        []byte
	sync.Mutex
}

func (s *server) initialize() error {
	var err error

	s.templ, err = template.ParseFiles(s.htmlfile)
	if err != nil {
		return err
	}

	return nil
}

func (s *server) readConfig() error {
	var config sites

	fi, err := os.Stat(s.configfile)
	if err != nil {
		return err
	}

	if s.lastconfig.After(fi.ModTime()) {
		return nil
	}

	_, err = toml.DecodeFile(s.configfile, &config)
	if err != nil {
		return err
	}

	s.lastconfig = time.Now()
	s.site_status = config.Service

	for i, _ := range s.site_status {
		s.site_status[i].Status = "unknown"
	}

	return nil
}

func (s *server) checkStatus() {
	var wg sync.WaitGroup

	for i, stat := range s.site_status {
		ck, ok := check[stat.Type]
		if ok == false {
			log.Println(stat.Type, stat.URL, "unknown type")
			continue
		}

		wg.Add(1)
		go func(idx int) {
			defer wg.Done()
			healthy, err := ck.Check(s.site_status[idx].URL)
			if err == nil && healthy {
				s.site_status[idx].Status = "online"
				return
			}

			s.site_status[idx].Status = "offline"
			log.Println(s.site_status[idx].Type, s.site_status[idx].URL, err)
		}(i)
	}

	wg.Wait()
}

func (s *server) updateStatus() error {
	s.Lock()
	defer s.Unlock()

	if s.next_status.Before(time.Now()) {
		err := s.readConfig()
		if err != nil {
			return err
		}

		s.checkStatus()

		s.last_status = time.Now()
		s.next_status = s.last_status.Add(time.Second * 5)
	}

	x := &struct {
		Status   []status
		DateTime string
	}{
		Status:   s.site_status,
		DateTime: s.last_status.String(),
	}

	b := &bytes.Buffer{}

	err := s.templ.Execute(b, x)
	if err != nil {
		return err
	}

	s.html, err = ioutil.ReadAll(b)
	if err != nil {
		return err
	}

	return nil
}

func (s *server) statusHandler(w http.ResponseWriter, r *http.Request) {
	host, _, _ := net.SplitHostPort(r.RemoteAddr)
	log.Println("request from", host)

	s.updateStatus()

	b := bytes.NewBuffer(s.html)

	io.Copy(w, b)
}

func init() {
	check = map[string]Status{
		"website":  new(Website),
		"etcd":     new(Etcd),
		"docker":   new(Docker),
		"registry": new(Registry),
	}
}

func main() {
	var (
		port     = flag.String("port", "", "HTTP service address (.e.g. 8080)")
		conffile = flag.String("conf", "sitecheck.conf", "Configuration file")
	)

	flag.Parse()

	if *port == "" {
		flag.Usage()
		return
	}

	s := &server{
		configfile: *conffile,
		htmlfile:   "status.html",
	}
	err := s.initialize()
	if err != nil {
		log.Fatal(err)
	}

	http.HandleFunc("/", s.statusHandler)
	log.Fatal(http.ListenAndServe(":"+*port, nil))
}
<commit_msg>split refresh from update<commit_after>package main

import (
	"bytes"
	"flag"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"sync"
	"time"

	"github.com/BurntSushi/toml"
)

type status struct {
	Name   string `toml:"name"`
	Type   string `toml:"type"`
	Status string
	URL    string `toml:"url"`
}

type sites struct {
	Service []status
}

type Status interface {
	Check(url string) (bool, error)
}

var check map[string]Status

type server struct {
	configfile  string
	lastconfig  time.Time
	htmlfile    string
	templ       *template.Template
	site_status []status
	next_status time.Time
	last_status time.Time
	html        []byte
	sync.Mutex
}

func (s *server) initialize() error {
	var err error

	s.templ, err = template.ParseFiles(s.htmlfile)
	if err != nil {
		return err
	}

	return nil
}

func (s *server) readConfig() error {
	var config sites

	fi, err := os.Stat(s.configfile)
	if err != nil {
		return err
	}

	if s.lastconfig.After(fi.ModTime()) {
		return nil
	}

	_, err = toml.DecodeFile(s.configfile, &config)
	if err != nil {
		return err
	}

	s.lastconfig = time.Now()
	s.site_status = config.Service

	for i, _ := range s.site_status {
		s.site_status[i].Status = "unknown"
	}

	return nil
}

func (s *server) checkStatus() {
	var wg sync.WaitGroup

	for i, stat := range s.site_status {
		ck, ok := check[stat.Type]
		if ok == false {
			log.Println(stat.Type, stat.URL, "unknown type")
			continue
		}

		wg.Add(1)
		go func(idx int) {
			defer wg.Done()
			healthy, err := ck.Check(s.site_status[idx].URL)
			if err == nil && healthy {
				s.site_status[idx].Status = "online"
				return
			}

			s.site_status[idx].Status = "offline"
			log.Println(s.site_status[idx].Type, s.site_status[idx].URL, err)
		}(i)
	}

	wg.Wait()
}

func (s *server) refresh() error {
	s.Lock()
	defer s.Unlock()

	if s.next_status.Before(time.Now()) {
		err := s.readConfig()
		if err != nil {
			return err
		}

		s.checkStatus()

		s.last_status = time.Now()
		s.next_status = s.last_status.Add(time.Second * 5)
	}

	return nil
}

func (s *server) updateStatus() error {
	err := s.refresh()
	if err != nil {
		return err
	}

	x := &struct {
		Status   []status
		DateTime string
	}{
		Status:   s.site_status,
		DateTime: s.last_status.String(),
	}

	b := &bytes.Buffer{}

	err = s.templ.Execute(b, x)
	if err != nil {
		return err
	}

	s.html, err = ioutil.ReadAll(b)
	if err != nil {
		return err
	}

	return nil
}

func (s *server) statusHandler(w http.ResponseWriter, r *http.Request) {
	host, _, _ := net.SplitHostPort(r.RemoteAddr)
	log.Println("request from", host)

	s.updateStatus()

	b := bytes.NewBuffer(s.html)

	io.Copy(w, b)
}

func init() {
	check = map[string]Status{
		"website":  new(Website),
		"etcd":     new(Etcd),
		"docker":   new(Docker),
		"registry": new(Registry),
	}
}

func main() {
	var (
		port     = flag.String("port", "", "HTTP service address (.e.g. 8080)")
		conffile = flag.String("conf", "sitecheck.conf", "Configuration file")
	)

	flag.Parse()

	if *port == "" {
		flag.Usage()
		return
	}

	s := &server{
		configfile: *conffile,
		htmlfile:   "status.html",
	}
	err := s.initialize()
	if err != nil {
		log.Fatal(err)
	}

	http.HandleFunc("/", s.statusHandler)
	log.Fatal(http.ListenAndServe(":"+*port, nil))
}
<|endoftext|>