<commit_before>// Copyright 2016 Factom Foundation
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package factom

import (
	"encoding/json"
	"fmt"
)

// BackupWallet returns a formatted string with the wallet seed and the secret
// keys for all of the wallet addresses.
func BackupWallet() (string, error) {
	type walletBackupResponse struct {
		Seed      string             `json:"wallet-seed"`
		Addresses []*addressResponse `json:"addresses"`
	}

	req := NewJSON2Request("wallet-backup", APICounter(), nil)
	resp, err := walletRequest(req)
	if err != nil {
		return "", err
	}
	if resp.Error != nil {
		return "", resp.Error
	}

	w := new(walletBackupResponse)
	if err := json.Unmarshal(resp.JSONResult(), w); err != nil {
		return "", err
	}

	s := fmt.Sprintln(w.Seed)
	s += fmt.Sprintln()
	for _, adr := range w.Addresses {
		s += fmt.Sprintln(adr.Public)
		s += fmt.Sprintln(adr.Secret)
		s += fmt.Sprintln()
	}
	return s, nil
}

func GenerateFactoidAddress() (*FactoidAddress, error) {
	req := NewJSON2Request("generate-factoid-address", APICounter(), nil)
	resp, err := walletRequest(req)
	if err != nil {
		return nil, err
	}
	if resp.Error != nil {
		return nil, resp.Error
	}

	a := new(addressResponse)
	if err := json.Unmarshal(resp.JSONResult(), a); err != nil {
		return nil, err
	}
	f, err := GetFactoidAddress(a.Secret)
	if err != nil {
		return nil, err
	}

	return f, nil
}

func GenerateECAddress() (*ECAddress, error) {
	req := NewJSON2Request("generate-ec-address", APICounter(), nil)
	resp, err := walletRequest(req)
	if err != nil {
		return nil, err
	}
	if resp.Error != nil {
		return nil, resp.Error
	}

	a := new(addressResponse)
	if err := json.Unmarshal(resp.JSONResult(), a); err != nil {
		return nil, err
	}
	e, err := GetECAddress(a.Secret)
	if err != nil {
		return nil, err
	}

	return e, nil
}

func ImportAddresses(addrs ...string) (
	[]*FactoidAddress,
	[]*ECAddress,
	error) {

	params := new(importRequest)
	for _, addr := range addrs {
		s := secretRequest{Secret: addr}
		params.Addresses = append(params.Addresses, s)
	}
	req := NewJSON2Request("import-addresses", APICounter(), params)
	resp, err := walletRequest(req)
	if err != nil {
		return nil, nil, err
	}
	if resp.Error != nil {
		return nil, nil, resp.Error
	}

	r := new(multiAddressResponse)
	if err := json.Unmarshal(resp.JSONResult(), r); err != nil {
		return nil, nil, err
	}
	fs := make([]*FactoidAddress, 0)
	es := make([]*ECAddress, 0)
	for _, address := range r.Addresses {
		switch AddressStringType(address.Secret) {
		case FactoidSec:
			f, err := GetFactoidAddress(address.Secret)
			if err != nil {
				return nil, nil, err
			}
			fs = append(fs, f)
		case ECSec:
			e, err := GetECAddress(address.Secret)
			if err != nil {
				return nil, nil, err
			}
			es = append(es, e)
		default:
			return nil, nil, fmt.Errorf("Unrecognized address type")
		}
	}

	return fs, es, nil
}

func ImportKoinify(mnemonic string) (*FactoidAddress, error) {
	params := new(importKoinifyRequest)
	params.Words = mnemonic

	req := NewJSON2Request("import-koinify", APICounter(), params)
	resp, err := walletRequest(req)
	if err != nil {
		return nil, err
	}
	if resp.Error != nil {
		return nil, resp.Error
	}

	r := new(addressResponse)
	if err := json.Unmarshal(resp.JSONResult(), r); err != nil {
		return nil, err
	}
	f, err := GetFactoidAddress(r.Secret)
	if err != nil {
		return nil, err
	}

	return f, nil
}

func RemoveAddress(address string) error {
	params := new(addressRequest)
	params.Address = address

	req := NewJSON2Request("remove-address", APICounter(), params)
	resp, err := walletRequest(req)
	if err != nil {
		return err
	}
	if resp.Error != nil {
		return resp.Error
	}

	return nil
}

func FetchAddresses() ([]*FactoidAddress, []*ECAddress, error) {
	req := NewJSON2Request("all-addresses", APICounter(), nil)
	resp, err := walletRequest(req)
	if err != nil {
		return nil, nil, err
	}
	if resp.Error != nil {
		return nil, nil, resp.Error
	}

	fs := make([]*FactoidAddress, 0)
	es := make([]*ECAddress, 0)

	as := new(multiAddressResponse)
	if err := json.Unmarshal(resp.JSONResult(), as); err != nil {
		return nil, nil, err
	}

	for _, adr := range as.Addresses {
		switch AddressStringType(adr.Public) {
		case FactoidPub:
			f, err := GetFactoidAddress(adr.Secret)
			if err != nil {
				return nil, nil, err
			}
			fs = append(fs, f)
		case ECPub:
			e, err := GetECAddress(adr.Secret)
			if err != nil {
				return nil, nil, err
			}
			es = append(es, e)
		default:
			return nil, nil, fmt.Errorf("%s is not a valid address", adr.Public)
		}
	}

	return fs, es, nil
}

func FetchECAddress(ecpub string) (*ECAddress, error) {
	if AddressStringType(ecpub) != ECPub {
		return nil, fmt.Errorf(
			"%s is not an Entry Credit Public Address", ecpub)
	}
	params := new(addressRequest)
	params.Address = ecpub

	req := NewJSON2Request("address", APICounter(), params)
	resp, err := walletRequest(req)
	if err != nil {
		return nil, err
	}
	if resp.Error != nil {
		return nil, resp.Error
	}

	r := new(addressResponse)
	if err := json.Unmarshal(resp.JSONResult(), r); err != nil {
		return nil, err
	}

	return GetECAddress(r.Secret)
}

func FetchFactoidAddress(fctpub string) (*FactoidAddress, error) {
	if AddressStringType(fctpub) != FactoidPub {
		return nil, fmt.Errorf("%s is not a Factoid Address", fctpub)
	}
	params := new(addressRequest)
	params.Address = fctpub

	req := NewJSON2Request("address", APICounter(), params)
	resp, err := walletRequest(req)
	if err != nil {
		return nil, err
	}
	if resp.Error != nil {
		return nil, resp.Error
	}

	r := new(addressResponse)
	if err := json.Unmarshal(resp.JSONResult(), r); err != nil {
		return nil, err
	}

	return GetFactoidAddress(r.Secret)
}

func FetchIdentityKey(pub string) (*IdentityKey, error) {
	params := new(addressRequest)
	params.Address = pub

	req := NewJSON2Request("identity-key", APICounter(), params)
	resp, err := walletRequest(req)
	if err != nil {
		return nil, err
	}
	if resp.Error != nil {
		return nil, resp.Error
	}

	r := new(addressResponse)
	if err := json.Unmarshal(resp.JSONResult(), r); err != nil {
		return nil, err
	}

	return GetIdentityKey(r.Secret)
}

func GetWalletHeight() (uint32, error) {
	req := NewJSON2Request("get-height", APICounter(), nil)
	resp, err := walletRequest(req)
	if err != nil {
		return 0, err
	}
	if resp.Error != nil {
		return 0, resp.Error
	}

	r := new(heightResponse)
	if err := json.Unmarshal(resp.JSONResult(), r); err != nil {
		return 0, err
	}

	return uint32(r.Height), nil
}

type addressResponse struct {
	Public string `json:"public"`
	Secret string `json:"secret"`
}

type multiAddressResponse struct {
	Addresses []*addressResponse `json:"addresses"`
}

type composeEntryRequest struct {
	Entry Entry  `json:"entry"`
	ECPub string `json:"ecpub"`
	Force bool   `json:"force"`
}

type composeChainRequest struct {
	Chain Chain  `json:"chain"`
	ECPub string `json:"ecpub"`
	Force bool   `json:"force"`
}

type composeEntryResponse struct {
	Commit *JSON2Request `json:"commit"`
	Reveal *JSON2Request `json:"reveal"`
}

func WalletComposeChainCommitReveal(chain *Chain, ecPub string, force bool) (*JSON2Request, *JSON2Request, error) {
	params := new(composeChainRequest)
	params.Chain = *chain
	params.ECPub = ecPub
	params.Force = force

	req := NewJSON2Request("compose-chain", APICounter(), params)
	resp, err := walletRequest(req)
	if err != nil {
		return nil, nil, err
	}
	if resp.Error != nil {
		return nil, nil, resp.Error
	}

	r := new(composeEntryResponse)
	if err := json.Unmarshal(resp.JSONResult(), r); err != nil {
		return nil, nil, err
	}

	return r.Commit, r.Reveal, nil
}

func WalletComposeEntryCommitReveal(entry *Entry, ecPub string, force bool) (*JSON2Request, *JSON2Request, error) {
	params := new(composeEntryRequest)
	params.Entry = *entry
	params.ECPub = ecPub
	params.Force = force

	req := NewJSON2Request("compose-entry", APICounter(), params)
	resp, err := walletRequest(req)
	if err != nil {
		return nil, nil, err
	}
	if resp.Error != nil {
		return nil, nil, resp.Error
	}

	r := new(composeEntryResponse)
	if err := json.Unmarshal(resp.JSONResult(), r); err != nil {
		return nil, nil, err
	}

	return r.Commit, r.Reveal, nil
}

type heightResponse struct {
	Height int64 `json:"height"`
}
<commit_msg>Added GenerateIdentityKey() function to access from the go library<commit_after>// Copyright 2016 Factom Foundation
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package factom

import (
	"encoding/json"
	"fmt"
)

// BackupWallet returns a formatted string with the wallet seed and the secret
// keys for all of the wallet addresses.
func BackupWallet() (string, error) {
	type walletBackupResponse struct {
		Seed      string             `json:"wallet-seed"`
		Addresses []*addressResponse `json:"addresses"`
	}

	req := NewJSON2Request("wallet-backup", APICounter(), nil)
	resp, err := walletRequest(req)
	if err != nil {
		return "", err
	}
	if resp.Error != nil {
		return "", resp.Error
	}

	w := new(walletBackupResponse)
	if err := json.Unmarshal(resp.JSONResult(), w); err != nil {
		return "", err
	}

	s := fmt.Sprintln(w.Seed)
	s += fmt.Sprintln()
	for _, adr := range w.Addresses {
		s += fmt.Sprintln(adr.Public)
		s += fmt.Sprintln(adr.Secret)
		s += fmt.Sprintln()
	}
	return s, nil
}

func GenerateFactoidAddress() (*FactoidAddress, error) {
	req := NewJSON2Request("generate-factoid-address", APICounter(), nil)
	resp, err := walletRequest(req)
	if err != nil {
		return nil, err
	}
	if resp.Error != nil {
		return nil, resp.Error
	}

	a := new(addressResponse)
	if err := json.Unmarshal(resp.JSONResult(), a); err != nil {
		return nil, err
	}
	f, err := GetFactoidAddress(a.Secret)
	if err != nil {
		return nil, err
	}

	return f, nil
}

func GenerateECAddress() (*ECAddress, error) {
	req := NewJSON2Request("generate-ec-address", APICounter(), nil)
	resp, err := walletRequest(req)
	if err != nil {
		return nil, err
	}
	if resp.Error != nil {
		return nil, resp.Error
	}

	a := new(addressResponse)
	if err := json.Unmarshal(resp.JSONResult(), a); err != nil {
		return nil, err
	}
	e, err := GetECAddress(a.Secret)
	if err != nil {
		return nil, err
	}

	return e, nil
}

func GenerateIdentityKey() (*IdentityKey, error) {
	req := NewJSON2Request("generate-identity-key", APICounter(), nil)
	resp, err := walletRequest(req)
	if err != nil {
		return nil, err
	}
	if resp.Error != nil {
		return nil, resp.Error
	}

	k := new(addressResponse)
	if err := json.Unmarshal(resp.JSONResult(), k); err != nil {
		return nil, err
	}
	e, err := GetIdentityKey(k.Secret)
	if err != nil {
		return nil, err
	}

	return e, nil
}

func ImportAddresses(addrs ...string) (
	[]*FactoidAddress,
	[]*ECAddress,
	error) {

	params := new(importRequest)
	for _, addr := range addrs {
		s := secretRequest{Secret: addr}
		params.Addresses = append(params.Addresses, s)
	}
	req := NewJSON2Request("import-addresses", APICounter(), params)
	resp, err := walletRequest(req)
	if err != nil {
		return nil, nil, err
	}
	if resp.Error != nil {
		return nil, nil, resp.Error
	}

	r := new(multiAddressResponse)
	if err := json.Unmarshal(resp.JSONResult(), r); err != nil {
		return nil, nil, err
	}
	fs := make([]*FactoidAddress, 0)
	es := make([]*ECAddress, 0)
	for _, address := range r.Addresses {
		switch AddressStringType(address.Secret) {
		case FactoidSec:
			f, err := GetFactoidAddress(address.Secret)
			if err != nil {
				return nil, nil, err
			}
			fs = append(fs, f)
		case ECSec:
			e, err := GetECAddress(address.Secret)
			if err != nil {
				return nil, nil, err
			}
			es = append(es, e)
		default:
			return nil, nil, fmt.Errorf("Unrecognized address type")
		}
	}

	return fs, es, nil
}

func ImportKoinify(mnemonic string) (*FactoidAddress, error) {
	params := new(importKoinifyRequest)
	params.Words = mnemonic

	req := NewJSON2Request("import-koinify", APICounter(), params)
	resp, err := walletRequest(req)
	if err != nil {
		return nil, err
	}
	if resp.Error != nil {
		return nil, resp.Error
	}

	r := new(addressResponse)
	if err := json.Unmarshal(resp.JSONResult(), r); err != nil {
		return nil, err
	}
	f, err := GetFactoidAddress(r.Secret)
	if err != nil {
		return nil, err
	}

	return f, nil
}

func RemoveAddress(address string) error {
	params := new(addressRequest)
	params.Address = address

	req := NewJSON2Request("remove-address", APICounter(), params)
	resp, err := walletRequest(req)
	if err != nil {
		return err
	}
	if resp.Error != nil {
		return resp.Error
	}

	return nil
}

func FetchAddresses() ([]*FactoidAddress, []*ECAddress, error) {
	req := NewJSON2Request("all-addresses", APICounter(), nil)
	resp, err := walletRequest(req)
	if err != nil {
		return nil, nil, err
	}
	if resp.Error != nil {
		return nil, nil, resp.Error
	}

	fs := make([]*FactoidAddress, 0)
	es := make([]*ECAddress, 0)

	as := new(multiAddressResponse)
	if err := json.Unmarshal(resp.JSONResult(), as); err != nil {
		return nil, nil, err
	}

	for _, adr := range as.Addresses {
		switch AddressStringType(adr.Public) {
		case FactoidPub:
			f, err := GetFactoidAddress(adr.Secret)
			if err != nil {
				return nil, nil, err
			}
			fs = append(fs, f)
		case ECPub:
			e, err := GetECAddress(adr.Secret)
			if err != nil {
				return nil, nil, err
			}
			es = append(es, e)
		default:
			return nil, nil, fmt.Errorf("%s is not a valid address", adr.Public)
		}
	}

	return fs, es, nil
}

func FetchECAddress(ecpub string) (*ECAddress, error) {
	if AddressStringType(ecpub) != ECPub {
		return nil, fmt.Errorf(
			"%s is not an Entry Credit Public Address", ecpub)
	}
	params := new(addressRequest)
	params.Address = ecpub

	req := NewJSON2Request("address", APICounter(), params)
	resp, err := walletRequest(req)
	if err != nil {
		return nil, err
	}
	if resp.Error != nil {
		return nil, resp.Error
	}

	r := new(addressResponse)
	if err := json.Unmarshal(resp.JSONResult(), r); err != nil {
		return nil, err
	}

	return GetECAddress(r.Secret)
}

func FetchFactoidAddress(fctpub string) (*FactoidAddress, error) {
	if AddressStringType(fctpub) != FactoidPub {
		return nil, fmt.Errorf("%s is not a Factoid Address", fctpub)
	}
	params := new(addressRequest)
	params.Address = fctpub

	req := NewJSON2Request("address", APICounter(), params)
	resp, err := walletRequest(req)
	if err != nil {
		return nil, err
	}
	if resp.Error != nil {
		return nil, resp.Error
	}

	r := new(addressResponse)
	if err := json.Unmarshal(resp.JSONResult(), r); err != nil {
		return nil, err
	}

	return GetFactoidAddress(r.Secret)
}

func FetchIdentityKey(pub string) (*IdentityKey, error) {
	params := new(addressRequest)
	params.Address = pub

	req := NewJSON2Request("identity-key", APICounter(), params)
	resp, err := walletRequest(req)
	if err != nil {
		return nil, err
	}
	if resp.Error != nil {
		return nil, resp.Error
	}

	r := new(addressResponse)
	if err := json.Unmarshal(resp.JSONResult(), r); err != nil {
		return nil, err
	}

	return GetIdentityKey(r.Secret)
}

func GetWalletHeight() (uint32, error) {
	req := NewJSON2Request("get-height", APICounter(), nil)
	resp, err := walletRequest(req)
	if err != nil {
		return 0, err
	}
	if resp.Error != nil {
		return 0, resp.Error
	}

	r := new(heightResponse)
	if err := json.Unmarshal(resp.JSONResult(), r); err != nil {
		return 0, err
	}

	return uint32(r.Height), nil
}

type addressResponse struct {
	Public string `json:"public"`
	Secret string `json:"secret"`
}

type multiAddressResponse struct {
	Addresses []*addressResponse `json:"addresses"`
}

type composeEntryRequest struct {
	Entry Entry  `json:"entry"`
	ECPub string `json:"ecpub"`
	Force bool   `json:"force"`
}

type composeChainRequest struct {
	Chain Chain  `json:"chain"`
	ECPub string `json:"ecpub"`
	Force bool   `json:"force"`
}

type composeEntryResponse struct {
	Commit *JSON2Request `json:"commit"`
	Reveal *JSON2Request `json:"reveal"`
}

func WalletComposeChainCommitReveal(chain *Chain, ecPub string, force bool) (*JSON2Request, *JSON2Request, error) {
	params := new(composeChainRequest)
	params.Chain = *chain
	params.ECPub = ecPub
	params.Force = force

	req := NewJSON2Request("compose-chain", APICounter(), params)
	resp, err := walletRequest(req)
	if err != nil {
		return nil, nil, err
	}
	if resp.Error != nil {
		return nil, nil, resp.Error
	}

	r := new(composeEntryResponse)
	if err := json.Unmarshal(resp.JSONResult(), r); err != nil {
		return nil, nil, err
	}

	return r.Commit, r.Reveal, nil
}

func WalletComposeEntryCommitReveal(entry *Entry, ecPub string, force bool) (*JSON2Request, *JSON2Request, error) {
	params := new(composeEntryRequest)
	params.Entry = *entry
	params.ECPub = ecPub
	params.Force = force

	req := NewJSON2Request("compose-entry", APICounter(), params)
	resp, err := walletRequest(req)
	if err != nil {
		return nil, nil, err
	}
	if resp.Error != nil {
		return nil, nil, resp.Error
	}

	r := new(composeEntryResponse)
	if err := json.Unmarshal(resp.JSONResult(), r); err != nil {
		return nil, nil, err
	}

	return r.Commit, r.Reveal, nil
}

type heightResponse struct {
	Height int64 `json:"height"`
}
<|endoftext|>


<commit_before>// Alexandria
//
// Copyright (C) 2015,2016  Colin Benner
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"fmt"
	"html/template"
	"io/ioutil"
	"net/http"
	"os"
	"runtime/pprof"

	flag "github.com/ogier/pflag"

	. "github.com/yzhs/alexandria"
)

const (
	LISTEN_ON   = "127.0.0.1:41665"
	MAX_RESULTS = 100
)

// Generate a HTML file describing the size of the library.
func printStats() string {
	stats := ComputeStatistics()
	n := stats.Num()
	size := float32(stats.Size()) / 1024.0
	return fmt.Sprintf("The library contains %v scrolls with a total size of %.1f kiB.\n", n, size)
}

// Send the statistics page to the client.
func statsHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, printStats())
}

// Handle the edit-link, causing the browser to open that scroll in an editor.
func editHandler(w http.ResponseWriter, r *http.Request) {
	headers := w.Header()
	headers["Content-Type"] = []string{"application/x-alexandria-edit"}
	id := r.FormValue("id")
	fmt.Fprintf(w, id)
}

// Serve the search page.
func mainHandler(w http.ResponseWriter, r *http.Request) {
	html, err := loadHTMLTemplate("main")
	if err != nil {
		fmt.Fprintf(w, "%v", err)
		return
	}
	fmt.Fprintln(w, string(html))
}

func loadHTMLTemplate(name string) ([]byte, error) {
	return ioutil.ReadFile(Config.TemplateDirectory + name + ".html")
}

type result struct {
	Query        string
	Matches      []Scroll
	NumMatches   int
	TotalMatches int
}

func renderTemplate(w http.ResponseWriter, templateFile string, resultData result) {
	t, err := template.ParseFiles(Config.TemplateDirectory + templateFile + ".html")
	if err != nil {
		fmt.Fprintf(w, "Error: %v", err)
		return
	}
	err = t.Execute(w, resultData)
	if err != nil {
		fmt.Fprintf(w, "Error: %v", err)
	}
}

func min(a, b int) int {
	if a > b {
		return b
	}
	return a
}

// Handle a query and serve the results.
func queryHandler(w http.ResponseWriter, r *http.Request) {
	query := r.FormValue("q")
	if query == "" {
		mainHandler(w, r)
		return
	}
	results, err := FindScrolls(query)
	if err != nil {
		panic(err)
	}
	numMatches := len(results.Ids)
	data := result{Query: query, NumMatches: numMatches, Matches: results.Ids[:min(20, numMatches)],
		TotalMatches: results.Total}
	renderTemplate(w, "search", data)
}

func serveDirectory(prefix string, directory string) {
	http.Handle(prefix, http.StripPrefix(prefix, http.FileServer(http.Dir(directory))))
}

func main() {
	var profile, version bool
	flag.BoolVarP(&version, "version", "v", false, "\tShow version")
	flag.BoolVar(&profile, "profile", false, "\tEnable profiler")
	flag.Parse()

	InitConfig()
	Config.MaxResults = MAX_RESULTS

	if profile {
		f, err := os.Create("alexandria.prof")
		if err != nil {
			panic(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	// TODO run GenerateIndex() when there is something new

	if version {
		fmt.Println(NAME, VERSION)
		return
	}

	http.HandleFunc("/", mainHandler)
	http.HandleFunc("/stats", statsHandler)
	http.HandleFunc("/search", queryHandler)
	http.HandleFunc("/alexandria.edit", editHandler)
	serveDirectory("/images/", Config.CacheDirectory)
	serveDirectory("/static/", Config.TemplateDirectory+"static")
	err := http.ListenAndServe(LISTEN_ON, nil)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: %v", err)
	}
}
<commit_msg>Move HTML templates to templates/html<commit_after>// Alexandria
//
// Copyright (C) 2015,2016  Colin Benner
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"fmt"
	"html/template"
	"io/ioutil"
	"net/http"
	"os"
	"runtime/pprof"

	flag "github.com/ogier/pflag"

	. "github.com/yzhs/alexandria"
)

const (
	LISTEN_ON   = "127.0.0.1:41665"
	MAX_RESULTS = 100
)

// Generate a HTML file describing the size of the library.
func printStats() string {
	stats := ComputeStatistics()
	n := stats.Num()
	size := float32(stats.Size()) / 1024.0
	return fmt.Sprintf("The library contains %v scrolls with a total size of %.1f kiB.\n", n, size)
}

// Send the statistics page to the client.
func statsHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, printStats())
}

// Handle the edit-link, causing the browser to open that scroll in an editor.
func editHandler(w http.ResponseWriter, r *http.Request) {
	headers := w.Header()
	headers["Content-Type"] = []string{"application/x-alexandria-edit"}
	id := r.FormValue("id")
	fmt.Fprintf(w, id)
}

// Serve the search page.
func mainHandler(w http.ResponseWriter, r *http.Request) {
	html, err := loadHTMLTemplate("main")
	if err != nil {
		fmt.Fprintf(w, "%v", err)
		return
	}
	fmt.Fprintln(w, string(html))
}

func loadHTMLTemplate(name string) ([]byte, error) {
	return ioutil.ReadFile(Config.TemplateDirectory + "html/" + name + ".html")
}

type result struct {
	Query        string
	Matches      []Scroll
	NumMatches   int
	TotalMatches int
}

func renderTemplate(w http.ResponseWriter, templateFile string, resultData result) {
	t, err := template.ParseFiles(Config.TemplateDirectory + "html/" + templateFile + ".html")
	if err != nil {
		fmt.Fprintf(w, "Error: %v", err)
		return
	}
	err = t.Execute(w, resultData)
	if err != nil {
		fmt.Fprintf(w, "Error: %v", err)
	}
}

func min(a, b int) int {
	if a > b {
		return b
	}
	return a
}

// Handle a query and serve the results.
func queryHandler(w http.ResponseWriter, r *http.Request) {
	query := r.FormValue("q")
	if query == "" {
		mainHandler(w, r)
		return
	}
	results, err := FindScrolls(query)
	if err != nil {
		panic(err)
	}
	numMatches := len(results.Ids)
	data := result{Query: query, NumMatches: numMatches, Matches: results.Ids[:min(20, numMatches)],
		TotalMatches: results.Total}
	renderTemplate(w, "search", data)
}

func serveDirectory(prefix string, directory string) {
	http.Handle(prefix, http.StripPrefix(prefix, http.FileServer(http.Dir(directory))))
}

func main() {
	var profile, version bool
	flag.BoolVarP(&version, "version", "v", false, "\tShow version")
	flag.BoolVar(&profile, "profile", false, "\tEnable profiler")
	flag.Parse()

	InitConfig()
	Config.MaxResults = MAX_RESULTS

	if profile {
		f, err := os.Create("alexandria.prof")
		if err != nil {
			panic(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	// TODO run GenerateIndex() when there is something new

	if version {
		fmt.Println(NAME, VERSION)
		return
	}

	http.HandleFunc("/", mainHandler)
	http.HandleFunc("/stats", statsHandler)
	http.HandleFunc("/search", queryHandler)
	http.HandleFunc("/alexandria.edit", editHandler)
	serveDirectory("/images/", Config.CacheDirectory)
	serveDirectory("/static/", Config.TemplateDirectory+"static")
	err := http.ListenAndServe(LISTEN_ON, nil)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: %v", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"fmt"
	"sync"
	"sync/atomic"
	"time"

	blog "github.com/letsencrypt/boulder/log"
	"github.com/prometheus/client_golang/prometheus"
)

var (
	// errStat is a prometheus counter vector tracking the number of errors
	// experienced by the janitor during operation sliced by a table label and a
	// type label. Examples of possible type labels include "getWork" and
	// "deleteResource".
	errStat = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "janitor_errors",
			Help: "Number of errors by type the boulder-janitor has experienced.",
		},
		[]string{"table", "type"})
	// deletedStat is a prometheus counter vector tracking the number of rows
	// deleted by the janitor, sliced by a table label.
	deletedStat = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "janitor_deletions",
			Help: "Number of deletions by table the boulder-janitor has performed.",
		},
		[]string{"table"})
	// workStat is a prometheus gauge vector tracking the number of rows found
	// during a batchedJob's getWork stage and queued into the work channel sliced
	// by a table label.
	workStat = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "janitor_workbatch",
			Help: "Number of items of work by table the boulder-janitor queued for deletion.",
		},
		[]string{"table"})
)

// batchedDBJob is a struct abstracting the common properties of a long running
// cleanup job based on cursoring across a database table's auto incrementing
// primary key.
type batchedDBJob struct {
	db  janitorDB
	log blog.Logger
	// table is the name of the table that this job cleans up.
	table string
	// purgeBefore indicates the cut-off for the the resoruce being cleaned up by
	// the job. Rows that older than now - purgeBefore are deleted.
	purgeBefore time.Time
	// workSleep is a duration that the job will sleep between getWork() calls
	// when no new work is found. If not provided, defaults to a minute.
	workSleep time.Duration
	// batchSize indicates how many database rows of work should be returned per query.
	batchSize int64
	// maxDPS optionally indicates a maximum rate of deletes to run per second.
	maxDPS int
	// parallelism controls how many independent go routines will be performing
	// cleanup deletes.
	parallelism int
	// workQuery is the parameterized SQL query that is used to find more work. It will be provided three parameters:
	//   * :startID - the primary key value to start the work query from.
	//   * :cutoff  - the purgeBefore date used to control which rows are old enough to be deleted.
	//   * :limit   - the batchSize value. Only this many rows should be returned by the query.
	workQuery string
}

// getWork reads work into the provided work channel starting at the startID by
// using the batchedDBJob's configured work query, purgeBefore, and batchSize.
// If there is no error the last primary key ID written to the work channel will
// be returned, otherwise an error result is returned.
func (j batchedDBJob) getWork(work chan<- int64, startID int64) (int64, error) {
	var idBatch []int64
	_, err := j.db.Select(
		&idBatch,
		j.workQuery,
		map[string]interface{}{
			"startID": startID,
			"cutoff":  j.purgeBefore,
			"limit":   j.batchSize,
		},
	)
	if err != nil && err != sql.ErrNoRows {
		return 0, err
	}
	rowsFound := len(idBatch)
	workStat.WithLabelValues(j.table).Set(float64(rowsFound))
	if rowsFound == 0 {
		return startID, nil
	}
	var lastID int64
	for _, v := range idBatch {
		work <- v
		lastID = v
	}
	return lastID, nil
}

// cleanResource uses the configured level of parallelism to run go routines
// that read ID values from the work channel and delete the corresponding table
// rows. If the batchedDBJob configures a maxDPS rate then it will be enforced by
// synchronizing the delete operations on a ticker based on the maxDPS.
// cleanResource will block until all of the worker go routines complete.
func (j batchedDBJob) cleanResource(work <-chan int64) {
	wg := new(sync.WaitGroup)
	deleted := int64(0)

	var ticker *time.Ticker
	if j.maxDPS > 0 {
		ticker = time.NewTicker(
			time.Duration(float64(time.Second) / float64(j.maxDPS)))
	}

	for i := 0; i < j.parallelism; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for id := range work {
				if ticker != nil {
					<-ticker.C
				}
				if err := j.deleteResource(id); err != nil {
					j.log.Errf(
						"error deleting ID %d from table %q: %s",
						id, j.table, err)
					errStat.WithLabelValues(j.table, "deleteResource").Inc()
				}
				_ = atomic.AddInt64(&deleted, 1)
			}
		}()
	}

	wg.Wait()
	j.log.Infof(
		"deleted a total of %d rows from table %q",
		deleted, j.table)
}

// deleteResource peforms a delete of the given ID from the batchedDBJob's
// table or returns an error.
func (j batchedDBJob) deleteResource(id int64) error {
	// NOTE(@cpu): We throw away the sql.Result here without checking the rows
	// affected because the query is always specific to the ID auto-increment
	// primary key. If there are multiple rows with the same primary key MariaDB
	// has failed us deeply.
	query := fmt.Sprintf(`DELETE FROM %s WHERE id = ?`, j.table)
	if _, err := j.db.Exec(query, id); err != nil {
		return err
	}
	j.log.Debugf("deleted ID %d in table %q", id, j.table)
	deletedStat.WithLabelValues(j.table).Inc()
	return nil
}

// RunForever starts a go routine that will run forever getting work with
// getWork and deleting rows with cleanResource.
func (j batchedDBJob) RunForever() {
	var id int64
	work := make(chan int64)

	go func() {
		for {
			lastID, err := j.getWork(work, id)
			if err != nil {
				j.log.Err(err.Error())
				errStat.WithLabelValues(j.table, "getWork").Inc()
				time.Sleep(time.Millisecond * 500)
				continue
			} else if lastID == id {
				j.log.Debugf(
					"made no new progress on table %q. Sleeping for a minute",
					j.table)
				if j.workSleep.Seconds() == 0 {
					time.Sleep(time.Minute)
				} else {
					time.Sleep(j.workSleep)
				}
			}
			id = lastID
		}
	}()

	j.cleanResource(work)
}
<commit_msg>janitor: add more context to getWork Err log. (#4398)<commit_after>package main

import (
	"database/sql"
	"fmt"
	"sync"
	"sync/atomic"
	"time"

	blog "github.com/letsencrypt/boulder/log"
	"github.com/prometheus/client_golang/prometheus"
)

var (
	// errStat is a prometheus counter vector tracking the number of errors
	// experienced by the janitor during operation sliced by a table label and a
	// type label. Examples of possible type labels include "getWork" and
	// "deleteResource".
	errStat = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "janitor_errors",
			Help: "Number of errors by type the boulder-janitor has experienced.",
		},
		[]string{"table", "type"})
	// deletedStat is a prometheus counter vector tracking the number of rows
	// deleted by the janitor, sliced by a table label.
	deletedStat = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "janitor_deletions",
			Help: "Number of deletions by table the boulder-janitor has performed.",
		},
		[]string{"table"})
	// workStat is a prometheus gauge vector tracking the number of rows found
	// during a batchedJob's getWork stage and queued into the work channel sliced
	// by a table label.
	workStat = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "janitor_workbatch",
			Help: "Number of items of work by table the boulder-janitor queued for deletion.",
		},
		[]string{"table"})
)

// batchedDBJob is a struct abstracting the common properties of a long running
// cleanup job based on cursoring across a database table's auto incrementing
// primary key.
type batchedDBJob struct {
	db  janitorDB
	log blog.Logger
	// table is the name of the table that this job cleans up.
	table string
	// purgeBefore indicates the cut-off for the the resoruce being cleaned up by
	// the job. Rows that older than now - purgeBefore are deleted.
	purgeBefore time.Time
	// workSleep is a duration that the job will sleep between getWork() calls
	// when no new work is found. If not provided, defaults to a minute.
	workSleep time.Duration
	// batchSize indicates how many database rows of work should be returned per query.
	batchSize int64
	// maxDPS optionally indicates a maximum rate of deletes to run per second.
	maxDPS int
	// parallelism controls how many independent go routines will be performing
	// cleanup deletes.
	parallelism int
	// workQuery is the parameterized SQL query that is used to find more work. It will be provided three parameters:
	//   * :startID - the primary key value to start the work query from.
	//   * :cutoff  - the purgeBefore date used to control which rows are old enough to be deleted.
	//   * :limit   - the batchSize value. Only this many rows should be returned by the query.
	workQuery string
}

// getWork reads work into the provided work channel starting at the startID by
// using the batchedDBJob's configured work query, purgeBefore, and batchSize.
// If there is no error the last primary key ID written to the work channel will
// be returned, otherwise an error result is returned.
func (j batchedDBJob) getWork(work chan<- int64, startID int64) (int64, error) {
	var idBatch []int64
	_, err := j.db.Select(
		&idBatch,
		j.workQuery,
		map[string]interface{}{
			"startID": startID,
			"cutoff":  j.purgeBefore,
			"limit":   j.batchSize,
		},
	)
	if err != nil && err != sql.ErrNoRows {
		return 0, err
	}
	rowsFound := len(idBatch)
	workStat.WithLabelValues(j.table).Set(float64(rowsFound))
	if rowsFound == 0 {
		return startID, nil
	}
	var lastID int64
	for _, v := range idBatch {
		work <- v
		lastID = v
	}
	return lastID, nil
}

// cleanResource uses the configured level of parallelism to run go routines
// that read ID values from the work channel and delete the corresponding table
// rows. If the batchedDBJob configures a maxDPS rate then it will be enforced by
// synchronizing the delete operations on a ticker based on the maxDPS.
// cleanResource will block until all of the worker go routines complete.
func (j batchedDBJob) cleanResource(work <-chan int64) {
	wg := new(sync.WaitGroup)
	deleted := int64(0)

	var ticker *time.Ticker
	if j.maxDPS > 0 {
		ticker = time.NewTicker(
			time.Duration(float64(time.Second) / float64(j.maxDPS)))
	}

	for i := 0; i < j.parallelism; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for id := range work {
				if ticker != nil {
					<-ticker.C
				}
				if err := j.deleteResource(id); err != nil {
					j.log.Errf(
						"error deleting ID %d from table %q: %s",
						id, j.table, err)
					errStat.WithLabelValues(j.table, "deleteResource").Inc()
				}
				_ = atomic.AddInt64(&deleted, 1)
			}
		}()
	}

	wg.Wait()
	j.log.Infof(
		"deleted a total of %d rows from table %q",
		deleted, j.table)
}

// deleteResource peforms a delete of the given ID from the batchedDBJob's
// table or returns an error.
func (j batchedDBJob) deleteResource(id int64) error {
	// NOTE(@cpu): We throw away the sql.Result here without checking the rows
	// affected because the query is always specific to the ID auto-increment
	// primary key. If there are multiple rows with the same primary key MariaDB
	// has failed us deeply.
	query := fmt.Sprintf(`DELETE FROM %s WHERE id = ?`, j.table)
	if _, err := j.db.Exec(query, id); err != nil {
		return err
	}
	j.log.Debugf("deleted ID %d in table %q", id, j.table)
	deletedStat.WithLabelValues(j.table).Inc()
	return nil
}

// RunForever starts a go routine that will run forever getting work with
// getWork and deleting rows with cleanResource.
func (j batchedDBJob) RunForever() {
	var id int64
	work := make(chan int64)

	go func() {
		for {
			lastID, err := j.getWork(work, id)
			if err != nil {
				j.log.Errf("error getting work for %q from ID %d: %s",
					j.table, id, err.Error())
				errStat.WithLabelValues(j.table, "getWork").Inc()
				time.Sleep(time.Millisecond * 500)
				continue
			} else if lastID == id {
				j.log.Debugf(
					"made no new progress on table %q. Sleeping for a minute",
					j.table)
				if j.workSleep.Seconds() == 0 {
					time.Sleep(time.Minute)
				} else {
					time.Sleep(j.workSleep)
				}
			}
			id = lastID
		}
	}()

	j.cleanResource(work)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/Symantec/Dominator/lib/constants"
	"github.com/Symantec/Dominator/lib/filegen"
	"github.com/Symantec/Dominator/lib/filegen/httpd"
	"github.com/Symantec/Dominator/lib/filegen/util"
	"github.com/Symantec/Dominator/lib/fsutil"
	"github.com/Symantec/Dominator/lib/logbuf"
	"github.com/Symantec/Dominator/lib/srpc/setupserver"
	"github.com/Symantec/tricorder/go/tricorder"
	"log"
	"os"
	"syscall"
)

var (
	configFile = flag.String("configFile", "/var/lib/filegen-server/config",
		"Name of file containing the configuration")
	permitInsecureMode = flag.Bool("permitInsecureMode", false,
		"If true, run in insecure mode. This gives remote access to all")
	portNum = flag.Uint("portNum", constants.BasicFileGenServerPortNumber,
		"Port number to allocate and listen on for HTTP/RPC")
)

func printUsage() {
	fmt.Fprintln(os.Stderr,
		"Usage: filegen-server [flags...] directory...")
	fmt.Fprintln(os.Stderr, "Common flags:")
	flag.PrintDefaults()
	fmt.Fprintln(os.Stderr, "directory: tree of source files")
}

func main() {
	flag.Usage = printUsage
	flag.Parse()
	tricorder.RegisterFlags()
	if os.Geteuid() == 0 {
		fmt.Fprintln(os.Stderr, "Do not run the filegen server as root")
		os.Exit(1)
	}
	circularBuffer := logbuf.New()
	logger := log.New(circularBuffer, "", log.LstdFlags)
	if err := setupserver.SetupTls(); err != nil {
		logger.Println(err)
		circularBuffer.Flush()
		if !*permitInsecureMode {
			os.Exit(1)
		}
	}
	manager := filegen.New(logger)
	if *configFile != "" {
		if err := util.LoadConfiguration(manager, *configFile); err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
		ch := fsutil.WatchFile(*configFile, nil)
		(<-ch).Close() // Drain the first event.
		go func() {
			<-ch
			err := syscall.Exec(os.Args[0], os.Args, os.Environ())
			if err != nil {
				logger.Printf("Unable to Exec:%s: %s\n", os.Args[0], err)
			}
		}()
	}
	httpd.AddHtmlWriter(manager)
	httpd.AddHtmlWriter(circularBuffer)
	for _, pathname := range flag.Args() {
		if err := registerSourceDirectory(manager, pathname, "/"); err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
	}
	if err := httpd.StartServer(*portNum, manager, false); err != nil {
		fmt.Fprintf(os.Stderr, "Unable to create http server: %s\n", err)
		os.Exit(1)
	}
}
<commit_msg>Switch filegen-server to lib/log/serverlogger package.<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/Symantec/Dominator/lib/constants"
	"github.com/Symantec/Dominator/lib/filegen"
	"github.com/Symantec/Dominator/lib/filegen/httpd"
	"github.com/Symantec/Dominator/lib/filegen/util"
	"github.com/Symantec/Dominator/lib/fsutil"
	"github.com/Symantec/Dominator/lib/log/serverlogger"
	"github.com/Symantec/Dominator/lib/srpc/setupserver"
	"github.com/Symantec/tricorder/go/tricorder"
	"os"
	"syscall"
)

var (
	configFile = flag.String("configFile", "/var/lib/filegen-server/config",
		"Name of file containing the configuration")
	permitInsecureMode = flag.Bool("permitInsecureMode", false,
		"If true, run in insecure mode. This gives remote access to all")
	portNum = flag.Uint("portNum", constants.BasicFileGenServerPortNumber,
		"Port number to allocate and listen on for HTTP/RPC")
)

func printUsage() {
	fmt.Fprintln(os.Stderr, "Usage: filegen-server [flags...] directory...")
	fmt.Fprintln(os.Stderr, "Common flags:")
	flag.PrintDefaults()
	fmt.Fprintln(os.Stderr, "directory: tree of source files")
}

func main() {
	flag.Usage = printUsage
	flag.Parse()
	tricorder.RegisterFlags()
	if os.Geteuid() == 0 {
		fmt.Fprintln(os.Stderr, "Do not run the filegen server as root")
		os.Exit(1)
	}
	logger := serverlogger.New("")
	if err := setupserver.SetupTls(); err != nil {
		if *permitInsecureMode {
			logger.Println(err)
		} else {
			logger.Fatalln(err)
		}
	}
	manager := filegen.New(logger)
	if *configFile != "" {
		if err := util.LoadConfiguration(manager, *configFile); err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
		ch := fsutil.WatchFile(*configFile, nil)
		(<-ch).Close() // Drain the first event.
		go func() {
			<-ch
			err := syscall.Exec(os.Args[0], os.Args, os.Environ())
			if err != nil {
				logger.Printf("Unable to Exec:%s: %s\n", os.Args[0], err)
			}
		}()
	}
	httpd.AddHtmlWriter(manager)
	httpd.AddHtmlWriter(logger)
	for _, pathname := range flag.Args() {
		if err := registerSourceDirectory(manager, pathname, "/"); err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
	}
	if err := httpd.StartServer(*portNum, manager, false); err != nil {
		fmt.Fprintf(os.Stderr, "Unable to create http server: %s\n", err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package gowhoson

import (
	"context"
	"errors"
	"fmt"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"

	"google.golang.org/grpc"

	"github.com/tai-ga/gowhoson/whoson"
	"github.com/urfave/cli"
)

func signalHandler(ch <-chan os.Signal, wg *sync.WaitGroup, c *cli.Context, ctx context.Context, f func()) {
	defer wg.Done()

	for {
		select {
		case <-ctx.Done():
			return
		case s := <-ch:
			switch s {
			case syscall.SIGHUP:
				err := whoson.LogWriter.Reopen()
				if err != nil {
					panic(err)
				}
			default:
				f()
				time.AfterFunc(time.Second*8, func() {
					displayError(c.App.ErrWriter, errors.New("Clean shutdown took too long, forcing exit"))
					os.Exit(0)
				})
			}
		}
	}
}

func splitHostPort(hostPort string) (host string, port int, err error) {
	host, p, err := net.SplitHostPort(hostPort)
	if err != nil {
		return "", 0, err
	}
	port, err = strconv.Atoi(p)
	if err != nil {
		return "", 0, err
	}
	return
}

func cmdServerValidate(c *cli.Context) (*whoson.ServerConfig, error) {
	config := c.App.Metadata["config"].(*whoson.ServerConfig)
	if c.String("loglevel") != "" {
		config.Loglevel = c.String("loglevel")
		switch config.Loglevel {
		case "debug", "info", "warn", "error", "dpanic", "panic", "fatal":
		default:
			return nil, fmt.Errorf("\"--loglevel %s\" not support loglevel", config.Loglevel)
		}
	}
	if c.String("log") != "" {
		config.Log = c.String("log")
	}
	if c.Int("serverid") != 0 {
		config.ServerID = c.Int("serverid")
	}
	if c.Bool("expvar") != false {
		config.Expvar = c.Bool("expvar")
	}
	if c.String("tcp") != "" {
		config.TCP = c.String("tcp")
		if config.TCP != "nostart" {
			host, _, err := splitHostPort(config.TCP)
			if err != nil {
				return nil, err
			}
			if net.ParseIP(host) == nil {
				return nil, fmt.Errorf("\"--tcp %s\" parse error", config.TCP)
			}
		}
	}
	if c.String("udp") != "" {
		config.UDP = c.String("udp")
		if config.UDP != "nostart" {
			host, _, err := splitHostPort(config.UDP)
			if err != nil {
				return nil, err
			}
			if net.ParseIP(host) == nil {
				return nil, fmt.Errorf("\"--udp %s\" parse error", config.UDP)
			}
		}
	}
	return config, nil
}

func cmdServer(c *cli.Context) error {
	var err error
	config, err := cmdServerValidate(c)
	if err != nil {
		displayError(c.App.ErrWriter, err)
		return err
	}

	wg := new(sync.WaitGroup)
	sigChan := make(chan os.Signal, 1)
	defer close(sigChan)

	whoson.NewMainStoreEnableSyncRemote()
	err = whoson.NewLogger(config.Log, config.Loglevel)
	if err != nil {
		displayError(c.App.ErrWriter, err)
		return err
	}
	whoson.Log("info", fmt.Sprintf("ServerID:%d", config.ServerID), nil, nil)
	whoson.NewIDGenerator(uint(config.ServerID))

	var con *net.UDPConn
	if config.UDP != "nostart" {
		host, port, err := splitHostPort(config.UDP)
		if err != nil {
			displayError(c.App.ErrWriter, err)
			return err
		}
		addrudp := net.UDPAddr{
			Port: port,
			IP:   net.ParseIP(host),
		}
		con, err = net.ListenUDP("udp", &addrudp)
		if err != nil {
			displayError(c.App.ErrWriter, err)
			return err
		}
		wg.Add(1)
		go func() {
			defer wg.Done()
			whoson.ServeUDP(con)
		}()
	}

	var lis *net.TCPListener
	if config.TCP != "nostart" {
		host, port, err := splitHostPort(config.TCP)
		if err != nil {
			displayError(c.App.ErrWriter, err)
			return err
		}
		addrtcp := net.TCPAddr{
			Port: port,
			IP:   net.ParseIP(host),
		}
		lis, err = net.ListenTCP("tcp", &addrtcp)
		if err != nil {
			displayError(c.App.ErrWriter, err)
			return err
		}
		wg.Add(1)
		go func() {
			defer wg.Done()
			whoson.ServeTCP(lis)
		}()
	}

	var lishttp net.Listener
	lishttp, err = runExpvar(config, wg, c)
	if err != nil {
		return err
	}

	var g *grpc.Server
	var lisgrpc net.Listener
	g = grpc.NewServer()
	lisgrpc, err = runGrpc(g, config, wg, c)
	if err != nil {
		return err
	}

	ctx, ctxCancel := context.WithCancel(context.Background())
	wg.Add(1)
	go func() {
		defer wg.Done()
		whoson.RunExpireChecker(ctx)
	}()

	wg.Add(1)
	go func() {
		defer wg.Done()
		hosts := strings.Split(config.SyncRemote, ",")
		whoson.RunSyncRemote(ctx, hosts)
	}()

	wg.Add(1)
	signal.Notify(sigChan, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
	go signalHandler(sigChan, wg, c, ctx, func() {
		defer ctxCancel()
		if config.UDP != "nostart" {
			con.Close()
		}
		if config.TCP != "nostart" {
			lis.Close()
		}
		if config.Expvar {
			lishttp.Close()
		}
		lisgrpc.Close()
		g.Stop()
	})

	wg.Wait()
	return nil
}

func getListener(c *cli.Context, host string) (net.Listener, error) {
	l, err := net.Listen("tcp", host)
	if err != nil {
		displayError(c.App.ErrWriter, err)
		return nil, err
	}
	return l, nil
}

func runExpvar(config *whoson.ServerConfig, wg *sync.WaitGroup, c *cli.Context) (net.Listener, error) {
	var lishttp net.Listener
	var err error
	if config.Expvar {
		if lishttp, err = getListener(c, ":8080"); err != nil {
			return nil, err
		}
		wg.Add(1)
		go func() {
			defer wg.Done()
			http.Serve(lishttp, nil)
		}()
	}
	return lishttp, nil
}

func runGrpc(g *grpc.Server, config *whoson.ServerConfig, wg *sync.WaitGroup, c *cli.Context) (net.Listener, error) {
	var lisgrpc net.Listener
	var err error
	if lisgrpc, err = getListener(c, config.GRPCPort); err != nil {
		return nil, err
	}
	wg.Add(1)
	go func() {
		defer wg.Done()
		whoson.RegisterSyncServer(g, &whoson.Sync{})
		g.Serve(lisgrpc)
	}()
	return lisgrpc, nil
}
<commit_msg>Fix argument order of the context<commit_after>package gowhoson

import (
	"context"
	"errors"
	"fmt"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"

	"google.golang.org/grpc"

	"github.com/tai-ga/gowhoson/whoson"
	"github.com/urfave/cli"
)

func signalHandler(ctx context.Context, ch <-chan os.Signal, wg *sync.WaitGroup, c *cli.Context, f func()) {
	defer wg.Done()

	for {
		select {
		case <-ctx.Done():
			return
		case s := <-ch:
			switch s {
			case syscall.SIGHUP:
				err := whoson.LogWriter.Reopen()
				if err != nil {
					panic(err)
				}
			default:
				f()
				time.AfterFunc(time.Second*8, func() {
					displayError(c.App.ErrWriter, errors.New("Clean shutdown took too long, forcing exit"))
					os.Exit(0)
				})
			}
		}
	}
}

func splitHostPort(hostPort string) (host string, port int, err error) {
	host, p, err := net.SplitHostPort(hostPort)
	if err != nil {
		return "", 0, err
	}
	port, err = strconv.Atoi(p)
	if err != nil {
		return "", 0, err
	}
	return
}

func cmdServerValidate(c *cli.Context) (*whoson.ServerConfig, error) {
	config := c.App.Metadata["config"].(*whoson.ServerConfig)
	if c.String("loglevel") != "" {
		config.Loglevel = c.String("loglevel")
		switch config.Loglevel {
		case "debug", "info", "warn", "error", "dpanic", "panic", "fatal":
		default:
			return nil, fmt.Errorf("\"--loglevel %s\" not support loglevel", config.Loglevel)
		}
	}
	if c.String("log") != "" {
		config.Log = c.String("log")
	}
	if c.Int("serverid") != 0 {
		config.ServerID = c.Int("serverid")
	}
	if c.Bool("expvar") != false {
		config.Expvar = c.Bool("expvar")
	}
	if c.String("tcp") != "" {
		config.TCP = c.String("tcp")
		if config.TCP != "nostart" {
			host, _, err := splitHostPort(config.TCP)
			if err != nil {
				return nil, err
			}
			if net.ParseIP(host) == nil {
				return nil, fmt.Errorf("\"--tcp %s\" parse error", config.TCP)
			}
		}
	}
	if c.String("udp") != "" {
		config.UDP = c.String("udp")
		if config.UDP != "nostart" {
			host, _, err := splitHostPort(config.UDP)
			if err != nil {
				return nil, err
			}
			if net.ParseIP(host) == nil {
				return nil, fmt.Errorf("\"--udp %s\" parse error", config.UDP)
			}
		}
	}
	return config, nil
}

func cmdServer(c *cli.Context) error {
	var err error
	config, err := cmdServerValidate(c)
	if err != nil {
		displayError(c.App.ErrWriter, err)
		return err
	}

	wg := new(sync.WaitGroup)
	sigChan := make(chan os.Signal, 1)
	defer close(sigChan)

	whoson.NewMainStoreEnableSyncRemote()
	err = whoson.NewLogger(config.Log, config.Loglevel)
	if err != nil {
		displayError(c.App.ErrWriter, err)
		return err
	}
	whoson.Log("info", fmt.Sprintf("ServerID:%d", config.ServerID), nil, nil)
	whoson.NewIDGenerator(uint(config.ServerID))

	var con *net.UDPConn
	if config.UDP != "nostart" {
		host, port, err := splitHostPort(config.UDP)
		if err != nil {
			displayError(c.App.ErrWriter, err)
			return err
		}
		addrudp := net.UDPAddr{
			Port: port,
			IP:   net.ParseIP(host),
		}
		con, err = net.ListenUDP("udp", &addrudp)
		if err != nil {
			displayError(c.App.ErrWriter, err)
			return err
		}
		wg.Add(1)
		go func() {
			defer wg.Done()
			whoson.ServeUDP(con)
		}()
	}

	var lis *net.TCPListener
	if config.TCP != "nostart" {
		host, port, err := splitHostPort(config.TCP)
		if err != nil {
			displayError(c.App.ErrWriter, err)
			return err
		}
		addrtcp := net.TCPAddr{
			Port: port,
			IP:   net.ParseIP(host),
		}
		lis, err = net.ListenTCP("tcp", &addrtcp)
		if err != nil {
			displayError(c.App.ErrWriter, err)
			return err
		}
		wg.Add(1)
		go func() {
			defer wg.Done()
			whoson.ServeTCP(lis)
		}()
	}

	var lishttp net.Listener
	lishttp, err = runExpvar(config, wg, c)
	if err != nil {
		return err
	}

	var g *grpc.Server
	var lisgrpc net.Listener
	g = grpc.NewServer()
	lisgrpc, err = runGrpc(g, config, wg, c)
	if err != nil {
		return err
	}

	ctx, ctxCancel := context.WithCancel(context.Background())
	wg.Add(1)
	go func() {
		defer wg.Done()
		whoson.RunExpireChecker(ctx)
	}()

	wg.Add(1)
	go func() {
		defer wg.Done()
		hosts := strings.Split(config.SyncRemote, ",")
		whoson.RunSyncRemote(ctx, hosts)
	}()

	wg.Add(1)
	signal.Notify(sigChan, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
	go signalHandler(ctx, sigChan, wg, c, func() {
		defer ctxCancel()
		if config.UDP != "nostart" {
			con.Close()
		}
		if config.TCP != "nostart" {
			lis.Close()
		}
		if config.Expvar {
			lishttp.Close()
		}
		lisgrpc.Close()
		g.Stop()
	})

	wg.Wait()
	return nil
}

func getListener(c *cli.Context, host string) (net.Listener, error) {
	l, err := net.Listen("tcp", host)
	if err != nil {
		displayError(c.App.ErrWriter, err)
		return nil, err
	}
	return l, nil
}

func runExpvar(config *whoson.ServerConfig, wg *sync.WaitGroup, c *cli.Context) (net.Listener, error) {
	var lishttp net.Listener
	var err error
	if config.Expvar {
		if lishttp, err = getListener(c, ":8080"); err != nil {
			return nil, err
		}
		wg.Add(1)
		go func() {
			defer wg.Done()
			http.Serve(lishttp, nil)
		}()
	}
	return lishttp, nil
}

func runGrpc(g *grpc.Server, config *whoson.ServerConfig, wg *sync.WaitGroup, c *cli.Context) (net.Listener, error) {
	var lisgrpc net.Listener
	var err error
	if lisgrpc, err = getListener(c, config.GRPCPort); err != nil {
		return nil, err
	}
	wg.Add(1)
	go func() {
		defer wg.Done()
		whoson.RegisterSyncServer(g, &whoson.Sync{})
		g.Serve(lisgrpc)
	}()
	return lisgrpc, nil
}
<|endoftext|>


<commit_before>package process

// Logic for this file is largely based on:
// https://github.com/jarib/childprocess/blob/783f7a00a1678b5d929062564ef5ae76822dfd62/lib/childprocess/unix/process.rb

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"regexp"
	"sync"
	"sync/atomic"
	"syscall"
	"time"

	"github.com/buildkite/agent/logger"
	"github.com/mattn/go-shellwords"
)

type Process struct {
	Pid        int
	PTY        bool
	Timestamp  bool
	Script     string
	Env        []string
	ExitStatus string

	buffer bytes.Buffer

	command *exec.Cmd

	// This callback is called when the process offically starts
	StartCallback func()

	// For every line in the process output, this callback will be called
	// with the contents of the line if its filter returns true.
	LineCallback       func(string)
	LinePreProcessor   func(string) string
	LineCallbackFilter func(string) bool

	// Running is stored as an int32 so we can use atomic operations to
	// set/get it (it's accessed by multiple goroutines)
	running int32
}

// If you change header parsing here make sure to change it in the
// buildkite.com frontend logic, too

var headerExpansionRegex = regexp.MustCompile("^(?:\\^\\^\\^\\s+\\+\\+\\+)$")

func (p *Process) Start() error {
	args, err := shellwords.Parse(p.Script)
	if err != nil {
		return err
	}

	p.command = exec.Command(args[0], args[1:]...)

	// Copy the current processes ENV and merge in the new ones. We do this
	// so the sub process gets PATH and stuff. We merge our path in over
	// the top of the current one so the ENV from Buildkite and the agent
	// take precedence over the agent
	currentEnv := os.Environ()
	p.command.Env = append(currentEnv, p.Env...)

	var waitGroup sync.WaitGroup

	lineReaderPipe, lineWriterPipe := io.Pipe()

	var multiWriter io.Writer
	if p.Timestamp {
		multiWriter = io.MultiWriter(lineWriterPipe)
	} else {
		multiWriter = io.MultiWriter(&p.buffer, lineWriterPipe)
	}

	// Toggle between running in a pty
	if p.PTY {
		pty, err := StartPTY(p.command)
		if err != nil {
			p.ExitStatus = "1"
			return err
		}

		p.Pid = p.command.Process.Pid
		p.setRunning(true)

		waitGroup.Add(1)

		go func() {
			logger.Debug("[Process] Starting to copy PTY to the buffer")

			// Copy the pty to our buffer. This will block until it
			// EOF's or something breaks.
			_, err = io.Copy(multiWriter, pty)
			if e, ok := err.(*os.PathError); ok && e.Err == syscall.EIO {
				// We can safely ignore this error, because
				// it's just the PTY telling us that it closed
				// successfully.  See:
				// https://github.com/buildkite/agent/pull/34#issuecomment-46080419
				err = nil
			}

			if err != nil {
				logger.Error("[Process] PTY output copy failed with error: %T: %v", err, err)
			} else {
				logger.Debug("[Process] PTY has finished being copied to the buffer")
			}

			waitGroup.Done()
		}()
	} else {
		p.command.Stdout = multiWriter
		p.command.Stderr = multiWriter
		p.command.Stdin = nil

		err := p.command.Start()
		if err != nil {
			p.ExitStatus = "1"
			return err
		}

		p.Pid = p.command.Process.Pid
		p.setRunning(true)
	}

	logger.Info("[Process] Process is running with PID: %d", p.Pid)

	// Add the line callback routine to the waitGroup
	waitGroup.Add(1)

	go func() {
		logger.Debug("[LineScanner] Starting to read lines")

		reader := bufio.NewReader(lineReaderPipe)

		var appending []byte
		var lineCallbackWaitGroup sync.WaitGroup

		for {
			line, isPrefix, err := reader.ReadLine()
			if err != nil {
				if err == io.EOF {
					logger.Debug("[LineScanner] Encountered EOF")
					break
				}

				logger.Error("[LineScanner] Failed to read: (%T: %v)", err, err)
			}

			// If isPrefix is true, that means we've got a really
			// long line incoming, and we'll keep appending to it
			// until isPrefix is false (which means the long line
			// has ended.
			if isPrefix && appending == nil {
				logger.Debug("[LineScanner] Line is too long to read, going to buffer it until it finishes")
				// bufio.ReadLine returns a slice which is only valid until the next invocation
				// since it points to its own internal buffer array. To accumulate the entire
				// result we make a copy of the first prefix, and insure there is spare capacity
				// for future appends to minimize the need for resizing on append.
				appending = make([]byte, len(line), (cap(line))*2)
				copy(appending, line)

				continue
			}

			// Should we be appending?
			if appending != nil {
				appending = append(appending, line...)

				// No more isPrefix! Line is finished!
				if !isPrefix {
					logger.Debug("[LineScanner] Finished buffering long line")
					line = appending

					// Reset appending back to nil
					appending = nil
				} else {
					continue
				}
			}

			// If we're timestamping this main thread will take
			// the hit of running the regex so we can build up
			// the timestamped buffer without breaking headers,
			// otherwise we let the goroutines take the perf hit.

			checkedForCallback := false
			lineHasCallback := false
			lineString := p.LinePreProcessor(string(line))

			// Create the prefixed buffer
			if p.Timestamp {
				lineHasCallback = p.LineCallbackFilter(lineString)
				checkedForCallback = true
				if lineHasCallback || headerExpansionRegex.MatchString(lineString) {
					// Don't timestamp special lines (e.g. header)
					p.buffer.WriteString(fmt.Sprintf("%s\n", line))
				} else {
					currentTime := time.Now().UTC().Format(time.RFC3339)
					p.buffer.WriteString(fmt.Sprintf("[%s] %s\n", currentTime, line))
				}
			}

			if lineHasCallback || !checkedForCallback {
				lineCallbackWaitGroup.Add(1)
				go func(line string) {
					defer lineCallbackWaitGroup.Done()
					if (checkedForCallback && lineHasCallback) || p.LineCallbackFilter(lineString) {
						p.LineCallback(line)
					}
				}(lineString)
			}
		}

		// We need to make sure all the line callbacks have finish before
		// finish up the process
		logger.Debug("[LineScanner] Waiting for callbacks to finish")
		lineCallbackWaitGroup.Wait()

		logger.Debug("[LineScanner] Finished")
		waitGroup.Done()
	}()

	// Call the StartCallback
	go p.StartCallback()

	// Wait until the process has finished. The returned error is nil if the command runs,
	// has no problems copying stdin, stdout, and stderr, and exits with a zero exit status.
	waitResult := p.command.Wait()

	// Close the line writer pipe
	lineWriterPipe.Close()

	// The process is no longer running at this point
	p.setRunning(false)

	// Find the exit status of the script
	p.ExitStatus = getExitStatus(waitResult)

	logger.Info("Process with PID: %d finished with Exit Status: %s", p.Pid, p.ExitStatus)

	// Sometimes (in docker containers) io.Copy never seems to finish. This is a mega
	// hack around it. If it doesn't finish after 1 second, just continue.
	logger.Debug("[Process] Waiting for routines to finish")
	err = timeoutWait(&waitGroup)
	if err != nil {
		logger.Debug("[Process] Timed out waiting for wait group: (%T: %v)", err, err)
	}

	// No error occurred so we can return nil
	return nil
}

func (p *Process) Output() string {
	return p.buffer.String()
}

func (p *Process) Kill() error {
	// Send a sigterm
	err := p.signal(syscall.SIGTERM)
	if err != nil {
		return err
	}

	// Make a channel that we'll use as a timeout
	c := make(chan int, 1)
	checking := true

	// Start a routine that checks to see if the process
	// is still alive.
	go func() {
		for checking {
			logger.Debug("[Process] Checking to see if PID: %d is still alive", p.Pid)

			foundProcess, err := os.FindProcess(p.Pid)

			// Can't find the process at all
			if err != nil {
				logger.Debug("[Process] Could not find process with PID: %d", p.Pid)

				break
			}

			// We have some information about the process
			if foundProcess != nil {
				processState, err := foundProcess.Wait()

				if err != nil || processState.Exited() {
					logger.Debug("[Process] Process with PID: %d has exited.", p.Pid)

					break
				}
			}

			// Retry in a moment
			sleepTime := time.Duration(1 * time.Second)
			time.Sleep(sleepTime)
		}

		c <- 1
	}()

	// Timeout this process after 3 seconds
	select {
	case _ = <-c:
		// Was successfully terminated
	case <-time.After(10 * time.Second):
		// Stop checking in the routine above
		checking = false

		// Forcefully kill the thing
		err = p.signal(syscall.SIGKILL)

		if err != nil {
			return err
		}
	}

	return nil
}

func (p *Process) signal(sig os.Signal) error {
	if p.command != nil && p.command.Process != nil {
		logger.Debug("[Process] Sending signal: %s to PID: %d", sig.String(), p.Pid)

		err := p.command.Process.Signal(sig)
		if err != nil {
			logger.Error("[Process] Failed to send signal: %s to PID: %d (%T: %v)", sig.String(), p.Pid, err, err)
			return err
		}
	} else {
		logger.Debug("[Process] No process to signal yet")
	}

	return nil
}

// Returns whether or not the process is running
func (p *Process) IsRunning() bool {
	return atomic.LoadInt32(&p.running) != 0
}

// Sets the running flag of the process
func (p *Process) setRunning(r bool) {
	// Use the atomic package to avoid race conditions when setting the
	// `running` value from multiple routines
	if r {
		atomic.StoreInt32(&p.running, 1)
	} else {
		atomic.StoreInt32(&p.running, 0)
	}
}

// https://github.com/hnakamur/commango/blob/fe42b1cf82bf536ce7e24dceaef6656002e03743/os/executil/executil.go#L29
// TODO: Can this be better?
func getExitStatus(waitResult error) string {
	exitStatus := -1

	if waitResult != nil {
		if err, ok := waitResult.(*exec.ExitError); ok {
			if s, ok := err.Sys().(syscall.WaitStatus); ok {
				exitStatus = s.ExitStatus()
			} else {
				logger.Error("[Process] Unimplemented for system where exec.ExitError.Sys() is not syscall.WaitStatus.")
			}
		} else {
			logger.Error("[Process] Unexpected error type in getExitStatus: %#v", waitResult)
		}
	} else {
		exitStatus = 0
	}

	return fmt.Sprintf("%d", exitStatus)
}

func timeoutWait(waitGroup *sync.WaitGroup) error {
	// Make a chanel that we'll use as a timeout
	c := make(chan int, 1)

	// Start waiting for the routines to finish
	go func() {
		waitGroup.Wait()
		c <- 1
	}()

	select {
	case _ = <-c:
		return nil
	case <-time.After(10 * time.Second):
		return errors.New("Timeout")
	}

	return nil
}
<commit_msg>fallback to TASKKILL on windows<commit_after>package process

// Logic for this file is largely based on:
// https://github.com/jarib/childprocess/blob/783f7a00a1678b5d929062564ef5ae76822dfd62/lib/childprocess/unix/process.rb

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"regexp"
	"runtime"
	"strconv"
	"sync"
	"sync/atomic"
	"syscall"
	"time"

	"github.com/buildkite/agent/logger"
	"github.com/mattn/go-shellwords"
)

type Process struct {
	Pid        int
	PTY        bool
	Timestamp  bool
	Script     string
	Env        []string
	ExitStatus string

	buffer bytes.Buffer

	command *exec.Cmd

	// This callback is called when the process offically starts
	StartCallback func()

	// For every line in the process output, this callback will be called
	// with the contents of the line if its filter returns true.
	LineCallback       func(string)
	LinePreProcessor   func(string) string
	LineCallbackFilter func(string) bool

	// Running is stored as an int32 so we can use atomic operations to
	// set/get it (it's accessed by multiple goroutines)
	running int32
}

// If you change header parsing here make sure to change it in the
// buildkite.com frontend logic, too

var headerExpansionRegex = regexp.MustCompile("^(?:\\^\\^\\^\\s+\\+\\+\\+)$")

func (p *Process) Start() error {
	args, err := shellwords.Parse(p.Script)
	if err != nil {
		return err
	}

	p.command = exec.Command(args[0], args[1:]...)

	// Copy the current processes ENV and merge in the new ones. We do this
	// so the sub process gets PATH and stuff. We merge our path in over
	// the top of the current one so the ENV from Buildkite and the agent
	// take precedence over the agent
	currentEnv := os.Environ()
	p.command.Env = append(currentEnv, p.Env...)

	var waitGroup sync.WaitGroup

	lineReaderPipe, lineWriterPipe := io.Pipe()

	var multiWriter io.Writer
	if p.Timestamp {
		multiWriter = io.MultiWriter(lineWriterPipe)
	} else {
		multiWriter = io.MultiWriter(&p.buffer, lineWriterPipe)
	}

	// Toggle between running in a pty
	if p.PTY {
		pty, err := StartPTY(p.command)
		if err != nil {
			p.ExitStatus = "1"
			return err
		}

		p.Pid = p.command.Process.Pid
		p.setRunning(true)

		waitGroup.Add(1)

		go func() {
			logger.Debug("[Process] Starting to copy PTY to the buffer")

			// Copy the pty to our buffer. This will block until it
			// EOF's or something breaks.
			_, err = io.Copy(multiWriter, pty)
			if e, ok := err.(*os.PathError); ok && e.Err == syscall.EIO {
				// We can safely ignore this error, because
				// it's just the PTY telling us that it closed
				// successfully.  See:
				// https://github.com/buildkite/agent/pull/34#issuecomment-46080419
				err = nil
			}

			if err != nil {
				logger.Error("[Process] PTY output copy failed with error: %T: %v", err, err)
			} else {
				logger.Debug("[Process] PTY has finished being copied to the buffer")
			}

			waitGroup.Done()
		}()
	} else {
		p.command.Stdout = multiWriter
		p.command.Stderr = multiWriter
		p.command.Stdin = nil

		err := p.command.Start()
		if err != nil {
			p.ExitStatus = "1"
			return err
		}

		p.Pid = p.command.Process.Pid
		p.setRunning(true)
	}

	logger.Info("[Process] Process is running with PID: %d", p.Pid)

	// Add the line callback routine to the waitGroup
	waitGroup.Add(1)

	go func() {
		logger.Debug("[LineScanner] Starting to read lines")

		reader := bufio.NewReader(lineReaderPipe)

		var appending []byte
		var lineCallbackWaitGroup sync.WaitGroup

		for {
			line, isPrefix, err := reader.ReadLine()
			if err != nil {
				if err == io.EOF {
					logger.Debug("[LineScanner] Encountered EOF")
					break
				}

				logger.Error("[LineScanner] Failed to read: (%T: %v)", err, err)
			}

			// If isPrefix is true, that means we've got a really
			// long line incoming, and we'll keep appending to it
			// until isPrefix is false (which means the long line
			// has ended.
			if isPrefix && appending == nil {
				logger.Debug("[LineScanner] Line is too long to read, going to buffer it until it finishes")
				// bufio.ReadLine returns a slice which is only valid until the next invocation
				// since it points to its own internal buffer array. To accumulate the entire
				// result we make a copy of the first prefix, and insure there is spare capacity
				// for future appends to minimize the need for resizing on append.
				appending = make([]byte, len(line), (cap(line))*2)
				copy(appending, line)

				continue
			}

			// Should we be appending?
			if appending != nil {
				appending = append(appending, line...)

				// No more isPrefix! Line is finished!
				if !isPrefix {
					logger.Debug("[LineScanner] Finished buffering long line")
					line = appending

					// Reset appending back to nil
					appending = nil
				} else {
					continue
				}
			}

			// If we're timestamping this main thread will take
			// the hit of running the regex so we can build up
			// the timestamped buffer without breaking headers,
			// otherwise we let the goroutines take the perf hit.

			checkedForCallback := false
			lineHasCallback := false
			lineString := p.LinePreProcessor(string(line))

			// Create the prefixed buffer
			if p.Timestamp {
				lineHasCallback = p.LineCallbackFilter(lineString)
				checkedForCallback = true
				if lineHasCallback || headerExpansionRegex.MatchString(lineString) {
					// Don't timestamp special lines (e.g. header)
					p.buffer.WriteString(fmt.Sprintf("%s\n", line))
				} else {
					currentTime := time.Now().UTC().Format(time.RFC3339)
					p.buffer.WriteString(fmt.Sprintf("[%s] %s\n", currentTime, line))
				}
			}

			if lineHasCallback || !checkedForCallback {
				lineCallbackWaitGroup.Add(1)
				go func(line string) {
					defer lineCallbackWaitGroup.Done()
					if (checkedForCallback && lineHasCallback) || p.LineCallbackFilter(lineString) {
						p.LineCallback(line)
					}
				}(lineString)
			}
		}

		// We need to make sure all the line callbacks have finish before
		// finish up the process
		logger.Debug("[LineScanner] Waiting for callbacks to finish")
		lineCallbackWaitGroup.Wait()

		logger.Debug("[LineScanner] Finished")
		waitGroup.Done()
	}()

	// Call the StartCallback
	go p.StartCallback()

	// Wait until the process has finished. The returned error is nil if the command runs,
	// has no problems copying stdin, stdout, and stderr, and exits with a zero exit status.
	waitResult := p.command.Wait()

	// Close the line writer pipe
	lineWriterPipe.Close()

	// The process is no longer running at this point
	p.setRunning(false)

	// Find the exit status of the script
	p.ExitStatus = getExitStatus(waitResult)

	logger.Info("Process with PID: %d finished with Exit Status: %s", p.Pid, p.ExitStatus)

	// Sometimes (in docker containers) io.Copy never seems to finish. This is a mega
	// hack around it. If it doesn't finish after 1 second, just continue.
	logger.Debug("[Process] Waiting for routines to finish")
	err = timeoutWait(&waitGroup)
	if err != nil {
		logger.Debug("[Process] Timed out waiting for wait group: (%T: %v)", err, err)
	}

	// No error occurred so we can return nil
	return nil
}

func (p *Process) Output() string {
	return p.buffer.String()
}

func (p *Process) Kill() error {
	// Send a sigterm
	var err error
	if runtime.GOOS == "windows" {
		err = exec.Command("CMD", "/C", "TASKKILL", "/F", "/PID", strconv.Itoa(p.Pid)).Run()
	} else {
		err = p.signal(syscall.SIGTERM)
	}
	if err != nil {
		return err
	}

	// Make a channel that we'll use as a timeout
	c := make(chan int, 1)
	checking := true

	// Start a routine that checks to see if the process
	// is still alive.
	go func() {
		for checking {
			logger.Debug("[Process] Checking to see if PID: %d is still alive", p.Pid)

			foundProcess, err := os.FindProcess(p.Pid)

			// Can't find the process at all
			if err != nil {
				logger.Debug("[Process] Could not find process with PID: %d", p.Pid)

				break
			}

			// We have some information about the process
			if foundProcess != nil {
				processState, err := foundProcess.Wait()

				if err != nil || processState.Exited() {
					logger.Debug("[Process] Process with PID: %d has exited.", p.Pid)

					break
				}
			}

			// Retry in a moment
			sleepTime := time.Duration(1 * time.Second)
			time.Sleep(sleepTime)
		}

		c <- 1
	}()

	// Timeout this process after 3 seconds
	select {
	case _ = <-c:
		// Was successfully terminated
	case <-time.After(10 * time.Second):
		// Stop checking in the routine above
		checking = false

		// Forcefully kill the thing
		err = p.signal(syscall.SIGKILL)

		if err != nil {
			return err
		}
	}

	return nil
}

func (p *Process) signal(sig os.Signal) error {
	if p.command != nil && p.command.Process != nil {
		logger.Debug("[Process] Sending signal: %s to PID: %d", sig.String(), p.Pid)

		err := p.command.Process.Signal(sig)
		if err != nil {
			logger.Error("[Process] Failed to send signal: %s to PID: %d (%T: %v)", sig.String(), p.Pid, err, err)
			return err
		}
	} else {
		logger.Debug("[Process] No process to signal yet")
	}

	return nil
}

// Returns whether or not the process is running
func (p *Process) IsRunning() bool {
	return atomic.LoadInt32(&p.running) != 0
}

// Sets the running flag of the process
func (p *Process) setRunning(r bool) {
	// Use the atomic package to avoid race conditions when setting the
	// `running` value from multiple routines
	if r {
		atomic.StoreInt32(&p.running, 1)
	} else {
		atomic.StoreInt32(&p.running, 0)
	}
}

// https://github.com/hnakamur/commango/blob/fe42b1cf82bf536ce7e24dceaef6656002e03743/os/executil/executil.go#L29
// TODO: Can this be better?
func getExitStatus(waitResult error) string {
	exitStatus := -1

	if waitResult != nil {
		if err, ok := waitResult.(*exec.ExitError); ok {
			if s, ok := err.Sys().(syscall.WaitStatus); ok {
				exitStatus = s.ExitStatus()
			} else {
				logger.Error("[Process] Unimplemented for system where exec.ExitError.Sys() is not syscall.WaitStatus.")
			}
		} else {
			logger.Error("[Process] Unexpected error type in getExitStatus: %#v", waitResult)
		}
	} else {
		exitStatus = 0
	}

	return fmt.Sprintf("%d", exitStatus)
}

func timeoutWait(waitGroup *sync.WaitGroup) error {
	// Make a chanel that we'll use as a timeout
	c := make(chan int, 1)

	// Start waiting for the routines to finish
	go func() {
		waitGroup.Wait()
		c <- 1
	}()

	select {
	case _ = <-c:
		return nil
	case <-time.After(10 * time.Second):
		return errors.New("Timeout")
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"github.com/mpreath/netcalc/pkg/network"
	"github.com/mpreath/netcalc/pkg/network/networknode"
	"github.com/mpreath/netcalc/pkg/utils"
	"log"
	"net/http"
	"sort"
	"strconv"
	"strings"
)

type Response struct {
	Status    string      `json:"status"`
	Error     string      `json:"error,omitempty"`
	ErrorCode int         `json:"error_code,omitempty"`
	Data      interface{} `json:"data,omitempty"`
}

func Info(w http.ResponseWriter, r *http.Request) {

	ipAddress, err := utils.ParseAddress(r.URL.Query().Get("address"))
	if err != nil {
		writeErrorResponse(w, err)
		return
	}

	subnetMask, err := utils.ParseAddress(r.URL.Query().Get("mask"))
	if err != nil {
		writeErrorResponse(w, err)
		return
	}

	net, err := network.New(ipAddress, subnetMask)
	if err != nil {
		writeErrorResponse(w, err)
		return
	}

	writeJsonResponse(w, http.StatusOK, net)

}

func Subnet(w http.ResponseWriter, r *http.Request) {

	ipAddress, err := utils.ParseAddress(r.URL.Query().Get("address"))
	if err != nil {
		writeErrorResponse(w, err)
		return
	}

	subnetMask, err := utils.ParseAddress(r.URL.Query().Get("mask"))
	if err != nil {
		writeErrorResponse(w, err)
		return
	}

	hostCount, err := strconv.Atoi(r.URL.Query().Get("hostCount"))
	if err != nil {
		writeErrorResponse(w, err)
		return
	}

	networkCount, err := strconv.Atoi(r.URL.Query().Get("networkCount"))
	if err != nil {
		writeErrorResponse(w, err)
		return
	}

	net, err := network.New(ipAddress, subnetMask)
	if err != nil {
		writeErrorResponse(w, err)
		return
	}

	node := networknode.New(net)

	if hostCount > 0 {
		err = networknode.SplitToHostCount(node, hostCount)
		if err != nil {
			log.Fatal(err)
		}

	} else if networkCount > 0 {
		err = networknode.SplitToNetCount(node, networkCount)
		if err != nil {
			log.Fatal(err)
		}
	}

	writeJsonResponse(w, http.StatusOK, node.Flatten())
}

func Summarize(w http.ResponseWriter, r *http.Request) {
	var networkList []*network.Network

	err := json.NewDecoder(r.Body).Decode(&networkList)
	if err != nil {
		writeErrorResponse(w, err)
		return
	}
	summarizedNetwork, err := network.SummarizeNetworks(networkList)
	if err != nil {
		writeErrorResponse(w, err)
		return
	}

	writeJsonResponse(w, http.StatusOK, summarizedNetwork)
}

func Vlsm(w http.ResponseWriter, r *http.Request) {

	ipAddress, err := utils.ParseAddress(r.URL.Query().Get("address"))
	if err != nil {
		writeErrorResponse(w, err)
		return
	}

	subnetMask, err := utils.ParseAddress(r.URL.Query().Get("mask"))
	if err != nil {
		writeErrorResponse(w, err)
		return
	}

	net, err := network.New(ipAddress, subnetMask)
	if err != nil {
		writeErrorResponse(w, err)
		return
	}

	node := networknode.New(net)

	vlsmArgs := strings.Split(r.URL.Query().Get("vlsmList"), ",")
	var vlsmList = make([]int, len(vlsmArgs))
	for idx, val := range vlsmArgs {
		vlsmList[idx], err = strconv.Atoi(val)
		if err != nil {
			writeErrorResponse(w, err)
			return
		}
	}
	sort.Slice(vlsmList, func(i, j int) bool {
		return vlsmList[i] < vlsmList[j]
	})

	for _, vlsm := range vlsmList {
		err = networknode.SplitToVlsmCount(node, vlsm)

		if err != nil {
			writeErrorResponse(w, err)
			return
		}
	}

	writeJsonResponse(w, http.StatusOK, node.Flatten())
}

func writeJsonResponse(w http.ResponseWriter, status int, data interface{}) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(status)
	enc := json.NewEncoder(w)
	enc.SetIndent("", "  ")
	err := enc.Encode(Response{
		Status: "ok",
		Data:   data,
	})
	if err != nil {
		log.Println(err)
		return
	}
}

func writeErrorResponse(w http.ResponseWriter, err error) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(http.StatusInternalServerError)
	enc := json.NewEncoder(w)
	enc.SetIndent("", "  ")
	err = enc.Encode(Response{
		Status:    "error",
		Error:     err.Error(),
		ErrorCode: http.StatusInternalServerError,
	})
	if err != nil {
		log.Println(err)
		return
	}
}
<commit_msg>fixed error handling in subnet endpoint<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/mpreath/netcalc/pkg/network"
	"github.com/mpreath/netcalc/pkg/network/networknode"
	"github.com/mpreath/netcalc/pkg/utils"
	"log"
	"net/http"
	"sort"
	"strconv"
	"strings"
)

type Response struct {
	Status    string      `json:"status"`
	Error     string      `json:"error,omitempty"`
	ErrorCode int         `json:"error_code,omitempty"`
	Data      interface{} `json:"data,omitempty"`
}

func Info(w http.ResponseWriter, r *http.Request) {

	ipAddress, err := utils.ParseAddress(r.URL.Query().Get("address"))
	if err != nil {
		writeErrorResponse(w, err)
		return
	}

	subnetMask, err := utils.ParseAddress(r.URL.Query().Get("mask"))
	if err != nil {
		writeErrorResponse(w, err)
		return
	}

	net, err := network.New(ipAddress, subnetMask)
	if err != nil {
		writeErrorResponse(w, err)
		return
	}

	writeJsonResponse(w, http.StatusOK, net)

}

func Subnet(w http.ResponseWriter, r *http.Request) {

	ipAddress, err := utils.ParseAddress(r.URL.Query().Get("address"))
	if err != nil {
		writeErrorResponse(w, err)
		return
	}

	subnetMask, err := utils.ParseAddress(r.URL.Query().Get("mask"))
	if err != nil {
		writeErrorResponse(w, err)
		return
	}

	hostCount, _ := strconv.Atoi(r.URL.Query().Get("hosts"))
	networkCount, _ := strconv.Atoi(r.URL.Query().Get("networks"))

	net, err := network.New(ipAddress, subnetMask)
	if err != nil {
		writeErrorResponse(w, err)
		return
	}

	node := networknode.New(net)

	if hostCount > 0 {
		err = networknode.SplitToHostCount(node, hostCount)
		if err != nil {
			log.Fatal(err)
		}

	} else if networkCount > 0 {
		err = networknode.SplitToNetCount(node, networkCount)
		if err != nil {
			log.Fatal(err)
		}
	} else {
		writeErrorResponse(w, fmt.Errorf("no valid hosts or networks value provided"))
		return
	}

	writeJsonResponse(w, http.StatusOK, node.Flatten())
}

func Summarize(w http.ResponseWriter, r *http.Request) {
	var networkList []*network.Network

	err := json.NewDecoder(r.Body).Decode(&networkList)
	if err != nil {
		writeErrorResponse(w, err)
		return
	}
	summarizedNetwork, err := network.SummarizeNetworks(networkList)
	if err != nil {
		writeErrorResponse(w, err)
		return
	}

	writeJsonResponse(w, http.StatusOK, summarizedNetwork)
}

func Vlsm(w http.ResponseWriter, r *http.Request) {

	ipAddress, err := utils.ParseAddress(r.URL.Query().Get("address"))
	if err != nil {
		writeErrorResponse(w, err)
		return
	}

	subnetMask, err := utils.ParseAddress(r.URL.Query().Get("mask"))
	if err != nil {
		writeErrorResponse(w, err)
		return
	}

	net, err := network.New(ipAddress, subnetMask)
	if err != nil {
		writeErrorResponse(w, err)
		return
	}

	node := networknode.New(net)

	vlsmArgs := strings.Split(r.URL.Query().Get("vlsmList"), ",")
	var vlsmList = make([]int, len(vlsmArgs))
	for idx, val := range vlsmArgs {
		vlsmList[idx], err = strconv.Atoi(val)
		if err != nil {
			writeErrorResponse(w, err)
			return
		}
	}
	sort.Slice(vlsmList, func(i, j int) bool {
		return vlsmList[i] < vlsmList[j]
	})

	for _, vlsm := range vlsmList {
		err = networknode.SplitToVlsmCount(node, vlsm)

		if err != nil {
			writeErrorResponse(w, err)
			return
		}
	}

	writeJsonResponse(w, http.StatusOK, node.Flatten())
}

func writeJsonResponse(w http.ResponseWriter, status int, data interface{}) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(status)
	enc := json.NewEncoder(w)
	enc.SetIndent("", "  ")
	err := enc.Encode(Response{
		Status: "ok",
		Data:   data,
	})
	if err != nil {
		log.Println(err)
		return
	}
}

func writeErrorResponse(w http.ResponseWriter, err error) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(http.StatusInternalServerError)
	enc := json.NewEncoder(w)
	enc.SetIndent("", "  ")
	err = enc.Encode(Response{
		Status:    "error",
		Error:     err.Error(),
		ErrorCode: http.StatusInternalServerError,
	})
	if err != nil {
		log.Println(err)
		return
	}
}
<|endoftext|>


<commit_before>package program

// Version can be requested through the command line with:
//
//     c2go -v
//
// See https://github.com/elliotchance/c2go/wiki/Release-Process
const Version = "v0.21.10 Zinc 2018-02-14"
<commit_msg>Bump version: v0.21.11 Zinc 2018-02-21<commit_after>package program

// Version can be requested through the command line with:
//
//     c2go -v
//
// See https://github.com/elliotchance/c2go/wiki/Release-Process
const Version = "v0.21.11 Zinc 2018-02-21"
<|endoftext|>


<commit_before>package program

// Version can be requested through the command line with:
//
//     c2go -v
//
// See https://github.com/elliotchance/c2go/wiki/Release-Process
const Version = "v0.25.7 Dubnium 2018-12-03"
<commit_msg>Bump version: v0.25.8 Dubnium 2018-12-04<commit_after>package program

// Version can be requested through the command line with:
//
//     c2go -v
//
// See https://github.com/elliotchance/c2go/wiki/Release-Process
const Version = "v0.25.8 Dubnium 2018-12-04"
<|endoftext|>