<commit_before>package capacitor

import (
	"fmt"
	"github.com/mathcunha/CloudCapacitor/sync2"
	"log"
	"strings"
)

const (
	Conservative = "conservative"
	Pessimist    = "pessimist"
	Optimist     = "optimist"
)

type ExecInfo struct {
	execs int
	path  string
	it    int
}

type NodeExec struct {
	nodes NodesInfo
	ExecInfo
}

type Heuristic interface {
	Exec(mode string, slo float32, wkls []string)
}

//Execute all configurations and workloads without infer
type BrutalForce struct {
	c *Capacitor
}

//Find the shortest path to Mark all configurations and workloads
type ShortestPath struct {
	c     *Capacitor
	slo   float32
	it    int
	maxIt int
}

//the policies proposed at thesis
type Policy struct {
	c           *Capacitor
	levelPolicy string
	wklPolicy   string
}

func NewPolicy(c *Capacitor, levelPolicy string, wklPolicy string) (h *Policy) {
	return &(Policy{c, levelPolicy, wklPolicy})
}

func NewShortestPath(c *Capacitor) (h *ShortestPath) {
	h = new(ShortestPath)
	h.c = c
	return
}

func (bf *BrutalForce) Exec(mode string, slo float32, wkls []string) {
	mapa := bf.c.Dspace.CapacityBy(mode)
	for _, nodes := range *mapa {
		for _, node := range nodes {
			for _, conf := range node.Configs {
				for _, wkl := range wkls {
					result := bf.c.Executor.Execute(*conf, wkl)
					log.Printf("%v x %v ? %v \n", *conf, wkl, result.SLO <= slo)
				}
			}
		}
	}
}

func (h *ShortestPath) Exec(mode string, slo float32, wkls []string) {
	mapa := h.c.Dspace.CapacityBy(mode)
	h.slo = slo
	for _, nodes := range *mapa {
		h.ExecCategory(wkls, nodes)
	}
}

func (h *Policy) Exec(mode string, slo float32, wkls []string) {
	dspace := h.c.Dspace.CapacityBy(mode)

	execs := 0

	//map to store the results by category
	dspaceInfo := make(map[string]NodesInfo)

	for cat, nodes := range *dspace {
		log.Printf("[Policy.Exec] Category:%v\n", cat)
		dspaceInfo[cat] = buildMatrix(wkls, nodes)

		wkl := h.selectWorkload(false, dspace, "", cat)
		level := h.selectCapacityLevel(false, dspace, "", cat)
		nodesInfo := dspaceInfo[cat]

		key := getMatrixKey(nodes.NodeByLevel(level).ID, wkl)
		nodeInfo := nodesInfo.matrix[key]

		//Process main loop, basically there will be no blank space
		for h.c.HasMore(&nodesInfo) {
			result := h.c.Executor.Execute(*nodeInfo.Configs[0], nodeInfo.WKL)
			log.Printf("[Policy.Exec] Node: %v - Result :%v\n", nodeInfo, result)
			execs++
			metSLO := result.SLO <= slo
			(&nodesInfo).Mark(key, metSLO, execs)

			log.Printf("[Policy.Exec] loop :%v\n", nodesInfo)
			//execute all equivalents
			equivalent := nodeInfo.Node.Equivalents()
			for _, node := range equivalent {
				key = getMatrixKey(node.ID, wkl)
				nodeInfo = nodesInfo.matrix[key]
				if !(nodeInfo.When != -1) {
					result = h.c.Executor.Execute(*nodeInfo.Configs[0], nodeInfo.WKL)
					log.Printf("[Policy.Exec] Node: %v - Result :%v\n", nodeInfo, result)
					execs++
					metSLO = result.SLO <= slo
					(&nodesInfo).Mark(key, metSLO, execs)
				}
			}
			//execute all equivalents
			level = h.selectCapacityLevel(metSLO, dspace, key, cat)

			//select workload
			if level == -1 {
				wkl = h.selectCapacityLevel(metSLO, dspace, key, cat)
			}

			//select other starting point

			break
		}
	}
}

func HasMore(c *Capacitor, dspaceInfo map[string]NodesInfo) (hasMore bool) {
	hasMore = true
	for _, nodes := range dspaceInfo {
		hasMore = c.HasMore(&nodes) || hasMore
	}
	return
}

func (p *Policy) selectWorkload(metSLO bool, mapa *map[string]Nodes, key string, cat string) (wklID int) {
	if "" == key {
		return 0
	}
	//TODO
	return -1
}

func (p *Policy) selectCapacityLevel(metSLO bool, mapa *map[string]Nodes, key string, cat string) (level int) {
	if "" == key {
		return 1
	}
	//TODO
	return -1
}

func (h *ShortestPath) ExecCategory(wkls []string, nodes Nodes) {
	numConfigs := 0
	for _, node := range nodes {
		numConfigs = numConfigs + len(node.Configs)
	}
	h.maxIt = len(wkls) * numConfigs

	nexts := []NodeExec{NodeExec{buildMatrix(wkls, nodes), ExecInfo{0, "", 0}}}

	for i := 0; i <= h.maxIt; i++ {
		wg := sync2.NewBlockWaitGroup(100000)
		chBest := make(chan ExecInfo)

		wg.Add(1)
		go func() {
			defer wg.Done()
			nexts = h.findShortestPath(nexts, wg, chBest, h.maxIt)
		}()

		go func() {
			wg.Wait()
			close(chBest)
		}()

		best := h.GetBest(chBest)

		if best.execs != -1 {
			PrintExecPath(best, wkls, nodes)
			return
		}

	}

}

func PrintExecPath(winner ExecInfo, wkls []string, nodes Nodes) {
	path := strings.Split(winner.path, "->")
	str := ""
	execs := 0
	for _, key := range path {
		ID, cWKL := splitMatrixKey(key)
		if cWKL != -1 {
			node := nodes.NodeByID(ID)
			str = fmt.Sprintf("%vWorkload:%v, Configs:%v\n", str, wkls[cWKL], node.Configs)
			execs = execs + len(node.Configs)
		}
	}
	str = fmt.Sprintf("%vTotal Execs:%v", str, execs)
	log.Printf(str)
}

func (h *ShortestPath) findShortestPath(current []NodeExec, wg *sync2.BlockWaitGroup, chBest chan ExecInfo, numConfigs int) (nexts []NodeExec) {
	nexts = *(new([]NodeExec))
	lessNodes := numConfigs
	for _, ex := range current {
		for key, node := range ex.nodes.matrix {
			if !(node.When != -1) {
				cNodes := ex.nodes.Clone()
				nExecs := ex.execs
				var result Result
				for _, conf := range node.Configs {
					nExecs = nExecs + 1
					result = h.c.Executor.Execute(*conf, node.WKL)
					cNodes.Mark(key, result.SLO <= h.slo, nExecs)

				}
				nPath := fmt.Sprintf("%v%v->", ex.path, key)

				if nodesLeft := h.c.NodesLeft(cNodes); nodesLeft != 0 {
					if lessNodes == nodesLeft {
						nexts = append(nexts, NodeExec{*cNodes, ExecInfo{nExecs, nPath, ex.it + 1}})
					}
					if lessNodes > nodesLeft {
						lessNodes = nodesLeft
						nexts = []NodeExec{NodeExec{*cNodes, ExecInfo{nExecs, nPath, ex.it + 1}}}
					}
				} else {
					//All executions!
					wg.Add(1)
					go func() {
						defer wg.Done()
						chBest <- ExecInfo{nExecs, nPath, -1}
					}()
					//return nil
				}
			}
		}
	}

	return nexts
}

func (h *ShortestPath) GetBest(chBest chan ExecInfo) (best ExecInfo) {
	best = ExecInfo{-1, "", -1}
	for {
		execInfo, more := <-chBest
		if more {
			best = execInfo
		} else {
			break
		}
	}

	return best
}
<commit_msg>time to make some tests<commit_after>package capacitor

import (
	"fmt"
	"github.com/mathcunha/CloudCapacitor/sync2"
	"log"
	"sort"
	"strings"
)

const (
	Conservative = "conservative"
	Pessimistic  = "pessimistic"
	Optimistic   = "optimistic"
)

type ExecInfo struct {
	execs int
	path  string
	it    int
}

type NodeExec struct {
	nodes NodesInfo
	ExecInfo
}

type Heuristic interface {
	Exec(mode string, slo float32, wkls []string)
}

//Execute all configurations and workloads without infer
type BrutalForce struct {
	c *Capacitor
}

//Find the shortest path to Mark all configurations and workloads
type ShortestPath struct {
	c     *Capacitor
	slo   float32
	it    int
	maxIt int
}

//the policies proposed at thesis
type Policy struct {
	c           *Capacitor
	levelPolicy string
	wklPolicy   string
}

func NewPolicy(c *Capacitor, levelPolicy string, wklPolicy string) (h *Policy) {
	return &(Policy{c, levelPolicy, wklPolicy})
}

func NewShortestPath(c *Capacitor) (h *ShortestPath) {
	h = new(ShortestPath)
	h.c = c
	return
}

func (bf *BrutalForce) Exec(mode string, slo float32, wkls []string) {
	mapa := bf.c.Dspace.CapacityBy(mode)
	for _, nodes := range *mapa {
		for _, node := range nodes {
			for _, conf := range node.Configs {
				for _, wkl := range wkls {
					result := bf.c.Executor.Execute(*conf, wkl)
					log.Printf("%v x %v ? %v \n", *conf, wkl, result.SLO <= slo)
				}
			}
		}
	}
}

func (h *ShortestPath) Exec(mode string, slo float32, wkls []string) {
	mapa := h.c.Dspace.CapacityBy(mode)
	h.slo = slo
	for _, nodes := range *mapa {
		h.ExecCategory(wkls, nodes)
	}
}

func (h *Policy) Exec(mode string, slo float32, wkls []string) {
	dspace := h.c.Dspace.CapacityBy(mode)

	execs := 0

	//map to store the results by category
	dspaceInfo := make(map[string]NodesInfo)

	for cat, nodes := range *dspace {
		log.Printf("[Policy.Exec] Category:%v\n", cat)
		dspaceInfo[cat] = buildMatrix(wkls, nodes)

		nodesInfo := dspaceInfo[cat]
		level, wkl := h.selectStartingPoint(&nodesInfo, &nodes)

		key := getMatrixKey(nodes.NodeByLevel(level).ID, wkl)
		nodeInfo := nodesInfo.matrix[key]

		//Process main loop, basically there will be no blank space
		for h.c.HasMore(&nodesInfo) {
			result := h.c.Executor.Execute(*nodeInfo.Configs[0], nodeInfo.WKL)
			log.Printf("[Policy.Exec] Node: %v - Result :%v\n", nodeInfo, result)
			execs++
			metSLO := result.SLO <= slo
			(&nodesInfo).Mark(key, metSLO, execs)

			log.Printf("[Policy.Exec] loop :%v\n", nodesInfo)
			//execute all equivalents
			equivalent := nodeInfo.Node.Equivalents()
			for _, node := range equivalent {
				key = getMatrixKey(node.ID, wkl)
				nodeInfo = nodesInfo.matrix[key]
				if !(nodeInfo.When != -1) {
					result = h.c.Executor.Execute(*nodeInfo.Configs[0], nodeInfo.WKL)
					log.Printf("[Policy.Exec] Node: %v - Result :%v\n", nodeInfo, result)
					execs++
					metSLO = result.SLO <= slo
					(&nodesInfo).Mark(key, metSLO, execs)
				}
			}
			//select capacity level
			oldLevel := level
			level = h.selectCapacityLevel(&nodesInfo, key, &nodes)

			//select workload
			if level == -1 {
				level = oldLevel
				wkl = h.selectWorkload(&nodesInfo, key)
			}

			//select other starting point
			if wkl == -1 {
				level, wkl = h.selectStartingPoint(&nodesInfo, &nodes)
			}

			break
		}
	}
}

func HasMore(c *Capacitor, dspaceInfo map[string]NodesInfo) (hasMore bool) {
	hasMore = true
	for _, nodes := range dspaceInfo {
		hasMore = c.HasMore(&nodes) || hasMore
	}
	return
}

func (p *Policy) selectStartingPoint(nodesInfo *NodesInfo, nodes *Nodes) (level int, wkl int) {
	for level = 1; level <= nodesInfo.levels; level++ {
		for wkl = 0; wkl < nodesInfo.workloads; wkl++ {
			nodeInfo := nodesInfo.matrix[getMatrixKey(nodes.NodeByLevel(level).ID, wkl)]
			if nodeInfo.When == -1 {
				return
			} else {
				//Same height, but, possibly, different lowers and highers
				equivalents := nodeInfo.Node.Equivalents()
				for _, node := range equivalents {
					nodeInfo := nodesInfo.matrix[getMatrixKey(node.ID, wkl)]
					if nodeInfo.When == -1 {
						return
					}
				}
			}
		}
	}
	return
}

func (p *Policy) selectWorkload(nodesInfo *NodesInfo, key string) (wklID int) {
	_, wkls := p.buildWorkloadList(key, nodesInfo)
	wklID = -1
	if len(wkls) == 0 {
		return
	}

	switch p.wklPolicy {
	case Conservative:
		wklID = wkls[len(wkls)/2]
	case Pessimistic:
		wklID = wkls[0]
	case Optimistic:
		wklID = wkls[len(wkls)-1]
	}
	return
}

func (p *Policy) selectCapacityLevel(nodesInfo *NodesInfo, key string, nodes *Nodes) (level int) {
	_, levels := p.buildCapacityLevelList(key, nodesInfo, nodes)
	level = -1
	if len(levels) == 0 {
		return
	}

	switch p.levelPolicy {
	case Conservative:
		level = levels[len(levels)/2]
	case Pessimistic:
		level = levels[0]
	case Optimistic:
		level = levels[len(levels)-1]

	}
	return
}

//Workloads availables in the current capacity level
func (p *Policy) buildWorkloadList(key string, nodesInfo *NodesInfo) (wkl int, wkls []int) {
	wkls = make([]int, 0, nodesInfo.workloads)
	ID, wkl := splitMatrixKey(key)
	for i := 0; i < nodesInfo.workloads; i++ {
		nodeInfo := nodesInfo.matrix[getMatrixKey(ID, i)]
		if nodeInfo.When == -1 {
			wkls = append(wkls, i)
		}
	}
	sort.Ints(wkls)
	return
}

//capacity levels availables in the current workload
func (p *Policy) buildCapacityLevelList(key string, nodesInfo *NodesInfo, nodes *Nodes) (ID string, levels []int) {
	levels = make([]int, 0, nodesInfo.levels)
	ID, wkl := splitMatrixKey(key)
	for i := 1; i <= nodesInfo.levels; i++ {
		nodeInfo := nodesInfo.matrix[getMatrixKey(nodes.NodeByLevel(i).ID, wkl)]
		if nodeInfo.When == -1 {
			levels = append(levels, i)
		} else {
			//Same height, but, possibly, different lowers and highers
			equivalents := nodeInfo.Node.Equivalents()
			for _, node := range equivalents {
				nodeInfo := nodesInfo.matrix[getMatrixKey(node.ID, wkl)]
				if nodeInfo.When == -1 {
					levels = append(levels, i)
					break
				}
			}
		}
	}
	sort.Ints(levels)
	return
}

func (h *ShortestPath) ExecCategory(wkls []string, nodes Nodes) {
	numConfigs := 0
	for _, node := range nodes {
		numConfigs = numConfigs + len(node.Configs)
	}
	h.maxIt = len(wkls) * numConfigs

	nexts := []NodeExec{NodeExec{buildMatrix(wkls, nodes), ExecInfo{0, "", 0}}}

	for i := 0; i <= h.maxIt; i++ {
		wg := sync2.NewBlockWaitGroup(100000)
		chBest := make(chan ExecInfo)

		wg.Add(1)
		go func() {
			defer wg.Done()
			nexts = h.findShortestPath(nexts, wg, chBest, h.maxIt)
		}()

		go func() {
			wg.Wait()
			close(chBest)
		}()

		best := h.GetBest(chBest)

		if best.execs != -1 {
			PrintExecPath(best, wkls, nodes)
			return
		}

	}

}

func PrintExecPath(winner ExecInfo, wkls []string, nodes Nodes) {
	path := strings.Split(winner.path, "->")
	str := ""
	execs := 0
	for _, key := range path {
		ID, cWKL := splitMatrixKey(key)
		if cWKL != -1 {
			node := nodes.NodeByID(ID)
			str = fmt.Sprintf("%vWorkload:%v, Configs:%v\n", str, wkls[cWKL], node.Configs)
			execs = execs + len(node.Configs)
		}
	}
	str = fmt.Sprintf("%vTotal Execs:%v", str, execs)
	log.Printf(str)
}

func (h *ShortestPath) findShortestPath(current []NodeExec, wg *sync2.BlockWaitGroup, chBest chan ExecInfo, numConfigs int) (nexts []NodeExec) {
	nexts = *(new([]NodeExec))
	lessNodes := numConfigs
	for _, ex := range current {
		for key, node := range ex.nodes.matrix {
			if !(node.When != -1) {
				cNodes := ex.nodes.Clone()
				nExecs := ex.execs
				var result Result
				for _, conf := range node.Configs {
					nExecs = nExecs + 1
					result = h.c.Executor.Execute(*conf, node.WKL)
					cNodes.Mark(key, result.SLO <= h.slo, nExecs)

				}
				nPath := fmt.Sprintf("%v%v->", ex.path, key)

				if nodesLeft := h.c.NodesLeft(cNodes); nodesLeft != 0 {
					if lessNodes == nodesLeft {
						nexts = append(nexts, NodeExec{*cNodes, ExecInfo{nExecs, nPath, ex.it + 1}})
					}
					if lessNodes > nodesLeft {
						lessNodes = nodesLeft
						nexts = []NodeExec{NodeExec{*cNodes, ExecInfo{nExecs, nPath, ex.it + 1}}}
					}
				} else {
					//All executions!
					wg.Add(1)
					go func() {
						defer wg.Done()
						chBest <- ExecInfo{nExecs, nPath, -1}
					}()
					//return nil
				}
			}
		}
	}

	return nexts
}

func (h *ShortestPath) GetBest(chBest chan ExecInfo) (best ExecInfo) {
	best = ExecInfo{-1, "", -1}
	for {
		execInfo, more := <-chBest
		if more {
			best = execInfo
		} else {
			break
		}
	}

	return best
}
<|endoftext|>


<commit_before>package circonusgometrics

// abstracted in preparation of separate circonus-api-go package

import (
	"encoding/json"
	"fmt"
)

type CheckBundleConfig struct {
	AsyncMetrics  bool   `json:"async_metrics"`
	Secret        string `json:"secret"`
	SubmissionUrl string `json:"submission_url"`
}

type CheckBundleMetric struct {
	Name   string `json:"name"`
	Type   string `json:"type"`
	Units  string `json:"units"`
	Status string `json:"status"`
}

type CheckBundle struct {
	CheckUUIDs         []string            `json:"_check_uuids,omitempty"`
	Checks             []string            `json:"_checks,omitempty"`
	Cid                string              `json:"_cid,omitempty"`
	Created            int                 `json:"_created,omitempty"`
	LastModified       int                 `json:"_last_modified,omitempty"`
	LastModifedBy      string              `json:"_last_modifed_by,omitempty"`
	ReverseConnectUrls []string            `json:"_reverse_connection_urls,omitempty"`
	Brokers            []string            `json:"brokers"`
	Config             CheckBundleConfig   `json:"config"`
	DisplayName        string              `json:"display_name"`
	Metrics            []CheckBundleMetric `json:"metrics"`
	MetricLimit        int                 `json:"metric_limit"`
	Notes              string              `json:"notes"`
	Period             int                 `json:"period"`
	Status             string              `json:"status"`
	Tags               []string            `json:"tags"`
	Target             string              `json:"target"`
	Timeout            int                 `json:"timeout"`
	Type               string              `json:"type"`
}

// Use Circonus API to retrieve a check bundle by ID
func (m *CirconusMetrics) fetchCheckBundleById(id int) (*CheckBundle, error) {
	cid := fmt.Sprintf("/check_bundle/%d", id)
	return m.fetchCheckBundleByCid(cid)
}

// Use Circonus API to retrieve a check bundle by CID
func (m *CirconusMetrics) fetchCheckBundleByCid(cid string) (*CheckBundle, error) {
	result, err := m.apiCall("GET", cid, nil)
	if err != nil {
		return nil, err
	}

	checkBundle := new(CheckBundle)
	json.Unmarshal(result, checkBundle)

	return checkBundle, nil
}

// Use Circonus API to search for a check bundle
func (m *CirconusMetrics) searchCheckBundles(searchCriteria string) ([]CheckBundle, error) {
	apiPath := fmt.Sprintf("/v2/check_bundle?search=%s", searchCriteria)

	response, err := m.apiCall("GET", apiPath, nil)
	if err != nil {
		return nil, fmt.Errorf("API call error %+v", response)
	}

	var results []CheckBundle
	err = json.Unmarshal(response, &results)
	if err != nil {
		return nil, fmt.Errorf("Error parsing JSON response %+v", err)
	}

	return results, nil
}

// Use Circonus API to create a check bundle
func (m *CirconusMetrics) createCheckBundle(config CheckBundle) (*CheckBundle, error) {
	cfgJson, err := json.Marshal(config)
	if err != nil {
		return nil, err
	}

	response, err := m.apiCall("POST", "/v2/check_bundle", cfgJson)
	if err != nil {
		return nil, err
	}

	checkBundle := new(CheckBundle)
	err = json.Unmarshal(response, checkBundle)
	if err != nil {
		return nil, err
	}

	return checkBundle, nil
}

// Use Circonus API to update a check bundle
func (m *CirconusMetrics) updateCheckBundle(config *CheckBundle) (*CheckBundle, error) {
	cfgJson, err := json.Marshal(config)
	if err != nil {
		return nil, err
	}

	fmt.Printf("%s\n", string(cfgJson))

	response, err := m.apiCall("PUT", config.Cid, cfgJson)
	if err != nil {
		return nil, err
	}

	checkBundle := new(CheckBundle)
	err = json.Unmarshal(response, checkBundle)
	if err != nil {
		return nil, err
	}

	return checkBundle, nil
}
<commit_msg>Update error messages<commit_after>package circonusgometrics

// abstracted in preparation of separate circonus-api-go package

import (
	"encoding/json"
	"fmt"
)

type CheckBundleConfig struct {
	AsyncMetrics  bool   `json:"async_metrics"`
	Secret        string `json:"secret"`
	SubmissionUrl string `json:"submission_url"`
}

type CheckBundleMetric struct {
	Name   string `json:"name"`
	Type   string `json:"type"`
	Units  string `json:"units"`
	Status string `json:"status"`
}

type CheckBundle struct {
	CheckUUIDs         []string            `json:"_check_uuids,omitempty"`
	Checks             []string            `json:"_checks,omitempty"`
	Cid                string              `json:"_cid,omitempty"`
	Created            int                 `json:"_created,omitempty"`
	LastModified       int                 `json:"_last_modified,omitempty"`
	LastModifedBy      string              `json:"_last_modifed_by,omitempty"`
	ReverseConnectUrls []string            `json:"_reverse_connection_urls,omitempty"`
	Brokers            []string            `json:"brokers"`
	Config             CheckBundleConfig   `json:"config"`
	DisplayName        string              `json:"display_name"`
	Metrics            []CheckBundleMetric `json:"metrics"`
	MetricLimit        int                 `json:"metric_limit"`
	Notes              string              `json:"notes"`
	Period             int                 `json:"period"`
	Status             string              `json:"status"`
	Tags               []string            `json:"tags"`
	Target             string              `json:"target"`
	Timeout            int                 `json:"timeout"`
	Type               string              `json:"type"`
}

// Use Circonus API to retrieve a check bundle by ID
func (m *CirconusMetrics) fetchCheckBundleById(id int) (*CheckBundle, error) {
	cid := fmt.Sprintf("/check_bundle/%d", id)
	return m.fetchCheckBundleByCid(cid)
}

// Use Circonus API to retrieve a check bundle by CID
func (m *CirconusMetrics) fetchCheckBundleByCid(cid string) (*CheckBundle, error) {
	result, err := m.apiCall("GET", cid, nil)
	if err != nil {
		return nil, err
	}

	checkBundle := new(CheckBundle)
	json.Unmarshal(result, checkBundle)

	return checkBundle, nil
}

// Use Circonus API to search for a check bundle
func (m *CirconusMetrics) searchCheckBundles(searchCriteria string) ([]CheckBundle, error) {
	apiPath := fmt.Sprintf("/v2/check_bundle?search=%s", searchCriteria)

	response, err := m.apiCall("GET", apiPath, nil)
	if err != nil {
		return nil, fmt.Errorf("[ERROR] API call error %+v", response)
	}

	var results []CheckBundle
	err = json.Unmarshal(response, &results)
	if err != nil {
		return nil, fmt.Errorf("[ERROR] Parsing JSON response %+v", err)
	}

	return results, nil
}

// Use Circonus API to create a check bundle
func (m *CirconusMetrics) createCheckBundle(config CheckBundle) (*CheckBundle, error) {
	cfgJson, err := json.Marshal(config)
	if err != nil {
		return nil, err
	}

	response, err := m.apiCall("POST", "/v2/check_bundle", cfgJson)
	if err != nil {
		return nil, err
	}

	checkBundle := new(CheckBundle)
	err = json.Unmarshal(response, checkBundle)
	if err != nil {
		return nil, err
	}

	return checkBundle, nil
}

// Use Circonus API to update a check bundle
func (m *CirconusMetrics) updateCheckBundle(config *CheckBundle) (*CheckBundle, error) {
	cfgJson, err := json.Marshal(config)
	if err != nil {
		return nil, err
	}

	fmt.Printf("%s\n", string(cfgJson))

	response, err := m.apiCall("PUT", config.Cid, cfgJson)
	if err != nil {
		return nil, err
	}

	checkBundle := new(CheckBundle)
	err = json.Unmarshal(response, checkBundle)
	if err != nil {
		return nil, err
	}

	return checkBundle, nil
}
<|endoftext|>


<commit_before>package checkers

/*
Checker board class
For example:

  abcdefgh
 ..........
8| # # # #|8
7|# # # # |7
6| # # # #|6
5|# # # # |5
4| # # # #|4
3|# # # # |3
2| # # # #|2
1|# # # # |1
 ''''''''''
  abcdefgh

Here a1 has position (0,0) and h8 is on (7,7)
*/
type Board struct {
	cells [][]*Checker
}

func NewBoard(size int) *Board {
	if size < 0 {
		return nil
	}
	cells := make([][]*Checker, size)
	for i := range cells {
		cells[i] = make([]*Checker, size)
	}
	return &Board{cells: cells}
}

func (b Board) Size() int {
	return len(b.cells)
}

func (b *Board) placeChecker(x, y int, c *Checker) {
	b.cells[y][x] = c
	if c != nil {
		c.setPosition(x, y)
	}
}

func (b *Board) takeChecker(x, y int) *Checker {
	c := b.cells[y][x]
	b.cells[y][x] = nil
	return c
}

func (b *Board) moveChecker(from, to Point) {
	if from == to {
		return
	}
	if !b.ContainsPos(from.X, from.Y) || !b.ContainsPos(to.X, to.Y) {
		return
	}
	c := b.takeChecker(from.X, from.Y)
	b.placeChecker(to.X, to.Y, c)
}

func (b Board) GetChecker(x, y int) *Checker {
	return b.cells[y][x]
}

func (b Board) IsEmpty(x, y int) bool {
	return b.cells[y][x] == nil
}

func (b Board) IsBlackSquare(pos Point) bool {
	return pos.Manhattan()%2 == 0
}

func (b Board) IsWhiteSquare(pos Point) bool {
	return pos.Manhattan()%2 == 1
}

func (b Board) ContainsPos(x, y int) bool {
	fieldSize := b.Size()
	return x >= 0 && y >= 0 && x < fieldSize && y < fieldSize
}

func (b Board) LastRowIndex() int {
	return b.Size() - 1
}

func (b Board) LastColumnIndex() int {
	return b.Size() - 1
}
<commit_msg>Add success result for place/move checker Board methods<commit_after>package checkers

/*
Checker board class
For example:

  abcdefgh
 ..........
8| # # # #|8
7|# # # # |7
6| # # # #|6
5|# # # # |5
4| # # # #|4
3|# # # # |3
2| # # # #|2
1|# # # # |1
 ''''''''''
  abcdefgh

Here a1 has position (0,0) and h8 is on (7,7)
*/
type Board struct {
	cells [][]*Checker
}

func NewBoard(size int) *Board {
	if size < 0 {
		return nil
	}
	cells := make([][]*Checker, size)
	for i := range cells {
		cells[i] = make([]*Checker, size)
	}
	return &Board{cells: cells}
}

func (b Board) Size() int {
	return len(b.cells)
}

func (b *Board) placeChecker(x, y int, c *Checker) bool {
	if !b.ContainsPos(x, y) {
		return false
	}

	b.cells[y][x] = c
	if c != nil {
		c.setPosition(x, y)
	}
	return true
}

func (b *Board) takeChecker(x, y int) *Checker {
	c := b.cells[y][x]
	b.cells[y][x] = nil
	return c
}

func (b *Board) moveChecker(from, to Point) bool {
	if from == to {
		return false
	}
	if !b.ContainsPos(from.X, from.Y) || !b.ContainsPos(to.X, to.Y) {
		return false
	}
	c := b.takeChecker(from.X, from.Y)
	return b.placeChecker(to.X, to.Y, c)
}

func (b Board) GetChecker(x, y int) *Checker {
	return b.cells[y][x]
}

func (b Board) IsEmpty(x, y int) bool {
	return b.cells[y][x] == nil
}

func (b Board) IsBlackSquare(pos Point) bool {
	return pos.Manhattan()%2 == 0
}

func (b Board) IsWhiteSquare(pos Point) bool {
	return pos.Manhattan()%2 == 1
}

func (b Board) ContainsPos(x, y int) bool {
	fieldSize := b.Size()
	return x >= 0 && y >= 0 && x < fieldSize && y < fieldSize
}

func (b Board) LastRowIndex() int {
	return b.Size() - 1
}

func (b Board) LastColumnIndex() int {
	return b.Size() - 1
}
<|endoftext|>


<commit_before>package v1alpha1

import (
	"fmt"
	"time"

	"github.com/golang/glog"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func (i *IssuerStatus) ACMEStatus() *ACMEIssuerStatus {
	if i.ACME == nil {
		i.ACME = &ACMEIssuerStatus{}
	}
	return i.ACME
}

func (a *ACMEIssuerDNS01Config) Provider(name string) (*ACMEIssuerDNS01Provider, error) {
	for _, p := range a.Providers {
		if p.Name == name {
			return &(*&p), nil
		}
	}
	return nil, fmt.Errorf("provider '%s' not found", name)
}

func (a *ACMECertificateConfig) ConfigForDomain(domain string) ACMECertificateDomainConfig {
	for _, cfg := range a.Config {
		for _, d := range cfg.Domains {
			if d == domain {
				return cfg
			}
		}
	}
	return ACMECertificateDomainConfig{}
}

func (c *CertificateStatus) ACMEStatus() *CertificateACMEStatus {
	if c.ACME == nil {
		c.ACME = &CertificateACMEStatus{}
	}
	return c.ACME
}

func (c *CertificateACMEStatus) SaveAuthorization(a ACMEDomainAuthorization) {
	for i, auth := range c.Authorizations {
		if auth.Domain == a.Domain {
			c.Authorizations[i] = a
			return
		}
	}
	c.Authorizations = append(c.Authorizations, a)
}

func (iss *Issuer) HasCondition(condition IssuerCondition) bool {
	if len(iss.Status.Conditions) == 0 {
		return false
	}
	for _, cond := range iss.Status.Conditions {
		if condition.Type == cond.Type && condition.Status == cond.Status {
			return true
		}
	}
	return false
}

func (iss *Issuer) UpdateStatusCondition(conditionType IssuerConditionType, status ConditionStatus, reason, message string) {
	newCondition := IssuerCondition{
		Type:    conditionType,
		Status:  status,
		Reason:  reason,
		Message: message,
	}

	t := time.Now()

	if len(iss.Status.Conditions) == 0 {
		glog.Infof("Setting lastTransitionTime for Issuer %q condition %q to %v", iss.Name, conditionType, t)
		newCondition.LastTransitionTime = metav1.NewTime(t)
		iss.Status.Conditions = []IssuerCondition{newCondition}
	} else {
		for i, cond := range iss.Status.Conditions {
			if cond.Type == conditionType {
				if cond.Status != newCondition.Status {
					glog.Infof("Found status change for Issuer %q condition %q: %q -> %q; setting lastTransitionTime to %v", iss.Name, conditionType, cond.Status, status, t)
					newCondition.LastTransitionTime = metav1.NewTime(t)
				} else {
					newCondition.LastTransitionTime = cond.LastTransitionTime
				}

				iss.Status.Conditions[i] = newCondition
				break
			}
		}
	}
}

func (crt *Certificate) HasCondition(condition CertificateCondition) bool {
	if len(crt.Status.Conditions) == 0 {
		return false
	}
	for _, cond := range crt.Status.Conditions {
		if condition.Type == cond.Type && condition.Status == cond.Status {
			return true
		}
	}
	return false
}

func (crt *Certificate) UpdateStatusCondition(conditionType CertificateConditionType, status ConditionStatus, reason, message string) {
	newCondition := CertificateCondition{
		Type:    conditionType,
		Status:  status,
		Reason:  reason,
		Message: message,
	}

	t := time.Now()

	if len(crt.Status.Conditions) == 0 {
		glog.Infof("Setting lastTransitionTime for Certificate %q condition %q to %v", crt.Name, conditionType, t)
		newCondition.LastTransitionTime = metav1.NewTime(t)
		crt.Status.Conditions = []CertificateCondition{newCondition}
	} else {
		for i, cond := range crt.Status.Conditions {
			if cond.Type == conditionType {
				if cond.Status != newCondition.Status {
					glog.Infof("Found status change for Certificate %q condition %q: %q -> %q; setting lastTransitionTime to %v", crt.Name, conditionType, cond.Status, status, t)
					newCondition.LastTransitionTime = metav1.NewTime(t)
				} else {
					newCondition.LastTransitionTime = cond.LastTransitionTime
				}

				crt.Status.Conditions[i] = newCondition
				break
			}
		}
	}
}
<commit_msg>Add ClusterIssuer helpers. Add GenericIssuer interface.<commit_after>package v1alpha1

import (
	"fmt"
	"time"

	"github.com/golang/glog"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
)

func (i *IssuerStatus) ACMEStatus() *ACMEIssuerStatus {
	if i.ACME == nil {
		i.ACME = &ACMEIssuerStatus{}
	}
	return i.ACME
}

func (a *ACMEIssuerDNS01Config) Provider(name string) (*ACMEIssuerDNS01Provider, error) {
	for _, p := range a.Providers {
		if p.Name == name {
			return &(*&p), nil
		}
	}
	return nil, fmt.Errorf("provider '%s' not found", name)
}

func (a *ACMECertificateConfig) ConfigForDomain(domain string) ACMECertificateDomainConfig {
	for _, cfg := range a.Config {
		for _, d := range cfg.Domains {
			if d == domain {
				return cfg
			}
		}
	}
	return ACMECertificateDomainConfig{}
}

func (c *CertificateStatus) ACMEStatus() *CertificateACMEStatus {
	if c.ACME == nil {
		c.ACME = &CertificateACMEStatus{}
	}
	return c.ACME
}

func (c *CertificateACMEStatus) SaveAuthorization(a ACMEDomainAuthorization) {
	for i, auth := range c.Authorizations {
		if auth.Domain == a.Domain {
			c.Authorizations[i] = a
			return
		}
	}
	c.Authorizations = append(c.Authorizations, a)
}

func (iss *Issuer) HasCondition(condition IssuerCondition) bool {
	if len(iss.Status.Conditions) == 0 {
		return false
	}
	for _, cond := range iss.Status.Conditions {
		if condition.Type == cond.Type && condition.Status == cond.Status {
			return true
		}
	}
	return false
}

func (iss *Issuer) UpdateStatusCondition(conditionType IssuerConditionType, status ConditionStatus, reason, message string) {
	newCondition := IssuerCondition{
		Type:    conditionType,
		Status:  status,
		Reason:  reason,
		Message: message,
	}

	t := time.Now()

	if len(iss.Status.Conditions) == 0 {
		glog.Infof("Setting lastTransitionTime for Issuer %q condition %q to %v", iss.Name, conditionType, t)
		newCondition.LastTransitionTime = metav1.NewTime(t)
		iss.Status.Conditions = []IssuerCondition{newCondition}
	} else {
		for i, cond := range iss.Status.Conditions {
			if cond.Type == conditionType {
				if cond.Status != newCondition.Status {
					glog.Infof("Found status change for Issuer %q condition %q: %q -> %q; setting lastTransitionTime to %v", iss.Name, conditionType, cond.Status, status, t)
					newCondition.LastTransitionTime = metav1.NewTime(t)
				} else {
					newCondition.LastTransitionTime = cond.LastTransitionTime
				}

				iss.Status.Conditions[i] = newCondition
				break
			}
		}
	}
}

func (iss *ClusterIssuer) UpdateStatusCondition(conditionType IssuerConditionType, status ConditionStatus, reason, message string) {
	newCondition := IssuerCondition{
		Type:    conditionType,
		Status:  status,
		Reason:  reason,
		Message: message,
	}

	t := time.Now()

	if len(iss.Status.Conditions) == 0 {
		glog.Infof("Setting lastTransitionTime for ClusterIssuer %q condition %q to %v", iss.Name, conditionType, t)
		newCondition.LastTransitionTime = metav1.NewTime(t)
		iss.Status.Conditions = []IssuerCondition{newCondition}
	} else {
		for i, cond := range iss.Status.Conditions {
			if cond.Type == conditionType {
				if cond.Status != newCondition.Status {
					glog.Infof("Found status change for ClusterIssuer %q condition %q: %q -> %q; setting lastTransitionTime to %v", iss.Name, conditionType, cond.Status, status, t)
					newCondition.LastTransitionTime = metav1.NewTime(t)
				} else {
					newCondition.LastTransitionTime = cond.LastTransitionTime
				}

				iss.Status.Conditions[i] = newCondition
				break
			}
		}
	}
}

func (crt *Certificate) HasCondition(condition CertificateCondition) bool {
	if len(crt.Status.Conditions) == 0 {
		return false
	}
	for _, cond := range crt.Status.Conditions {
		if condition.Type == cond.Type && condition.Status == cond.Status {
			return true
		}
	}
	return false
}

func (crt *Certificate) UpdateStatusCondition(conditionType CertificateConditionType, status ConditionStatus, reason, message string) {
	newCondition := CertificateCondition{
		Type:    conditionType,
		Status:  status,
		Reason:  reason,
		Message: message,
	}

	t := time.Now()

	if len(crt.Status.Conditions) == 0 {
		glog.Infof("Setting lastTransitionTime for Certificate %q condition %q to %v", crt.Name, conditionType, t)
		newCondition.LastTransitionTime = metav1.NewTime(t)
		crt.Status.Conditions = []CertificateCondition{newCondition}
	} else {
		for i, cond := range crt.Status.Conditions {
			if cond.Type == conditionType {
				if cond.Status != newCondition.Status {
					glog.Infof("Found status change for Certificate %q condition %q: %q -> %q; setting lastTransitionTime to %v", crt.Name, conditionType, cond.Status, status, t)
					newCondition.LastTransitionTime = metav1.NewTime(t)
				} else {
					newCondition.LastTransitionTime = cond.LastTransitionTime
				}

				crt.Status.Conditions[i] = newCondition
				break
			}
		}
	}
}

type GenericIssuer interface {
	runtime.Object
	GetObjectMeta() *metav1.ObjectMeta
	GetSpec() *IssuerSpec
	GetStatus() *IssuerStatus
	UpdateStatusCondition(conditionType IssuerConditionType, status ConditionStatus, reason, message string)
	Copy() GenericIssuer
}

var _ GenericIssuer = &Issuer{}
var _ GenericIssuer = &ClusterIssuer{}

func (c *ClusterIssuer) GetObjectMeta() *metav1.ObjectMeta {
	return &c.ObjectMeta
}
func (c *ClusterIssuer) GetSpec() *IssuerSpec {
	return &c.Spec
}
func (c *ClusterIssuer) GetStatus() *IssuerStatus {
	return &c.Status
}
func (c *ClusterIssuer) SetSpec(spec IssuerSpec) {
	c.Spec = spec
}
func (c *ClusterIssuer) SetStatus(status IssuerStatus) {
	c.Status = status
}
func (c *ClusterIssuer) Copy() GenericIssuer {
	return c.DeepCopy()
}
func (c *Issuer) GetObjectMeta() *metav1.ObjectMeta {
	return &c.ObjectMeta
}
func (c *Issuer) GetSpec() *IssuerSpec {
	return &c.Spec
}
func (c *Issuer) GetStatus() *IssuerStatus {
	return &c.Status
}
func (c *Issuer) SetSpec(spec IssuerSpec) {
	c.Spec = spec
}
func (c *Issuer) SetStatus(status IssuerStatus) {
	c.Status = status
}
func (c *Issuer) Copy() GenericIssuer {
	return c.DeepCopy()
}
<|endoftext|>


<commit_before>package sso

import (
	"bytes"
	"crypto/sha256"
	"encoding/base64"
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"net/url"
	"testing"
	"time"

	"github.com/skygeario/skygear-server/pkg/core/auth/authinfo"
	"github.com/skygeario/skygear-server/pkg/core/auth/session"
	coreconfig "github.com/skygeario/skygear-server/pkg/core/config"
	"github.com/skygeario/skygear-server/pkg/core/db"
	coreTime "github.com/skygeario/skygear-server/pkg/core/time"
	"github.com/skygeario/skygear-server/pkg/core/validation"

	"github.com/skygeario/skygear-server/pkg/auth/dependency/authnsession"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/hook"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/mfa"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/principal"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/principal/oauth"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/principal/password"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/sso"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/userprofile"

	. "github.com/skygeario/skygear-server/pkg/core/skytest"
	. "github.com/smartystreets/goconvey/convey"
)

func codeVerifierToCodeChallenge(codeVerifier string) string {
	sha256Arr := sha256.Sum256([]byte(codeVerifier))
	sha256Slice := sha256Arr[:]
	codeChallenge := base64.RawURLEncoding.EncodeToString(sha256Slice)
	return codeChallenge
}

func TestAuthResultHandler(t *testing.T) {
	stateJWTSecret := "secret"
	providerName := "mock"
	providerUserID := "mock_user_id"

	Convey("AuthResultHandler", t, func() {
		sh := &AuthResultHandler{}
		sh.TxContext = db.NewMockTxContext()
		oauthConfig := &coreconfig.OAuthConfiguration{
			StateJWTSecret: stateJWTSecret,
		}
		providerConfig := coreconfig.OAuthProviderConfiguration{
			ID:           providerName,
			Type:         "google",
			ClientID:     "mock_client_id",
			ClientSecret: "mock_client_secret",
		}
		mockProvider := sso.MockSSOProvider{
			RedirectURIs: []string{
				"http://localhost",
			},
			URLPrefix:      &url.URL{Scheme: "https", Host: "api.example.com"},
			BaseURL:        "http://mock/auth",
			OAuthConfig:    oauthConfig,
			ProviderConfig: providerConfig,
			UserInfo: sso.ProviderUserInfo{
				ID:    providerUserID,
				Email: "mock@example.com",
			},
		}
		sh.SSOProvider = &mockProvider
		mockOAuthProvider := oauth.NewMockProvider([]*oauth.Principal{
			&oauth.Principal{
				ID:             "john.doe.id",
				UserID:         "john.doe.id",
				ProviderType:   "google",
				ProviderKeys:   map[string]interface{}{},
				ProviderUserID: providerUserID,
			},
		})
		sh.OAuthAuthProvider = mockOAuthProvider
		authInfoStore := authinfo.NewMockStoreWithAuthInfoMap(
			map[string]authinfo.AuthInfo{
				"john.doe.id": authinfo.AuthInfo{
					ID: "john.doe.id",
				},
			},
		)
		sh.AuthInfoStore = authInfoStore
		sessionProvider := session.NewMockProvider()
		sessionWriter := session.NewMockWriter()
		userProfileStore := userprofile.NewMockUserProfileStore()
		sh.UserProfileStore = userProfileStore
		one := 1
		loginIDsKeys := []coreconfig.LoginIDKeyConfiguration{
			coreconfig.LoginIDKeyConfiguration{Key: "email", Maximum: &one},
		}
		allowedRealms := []string{password.DefaultRealm}
		passwordAuthProvider := password.NewMockProviderWithPrincipalMap(
			loginIDsKeys,
			allowedRealms,
			map[string]password.Principal{},
		)
		identityProvider := principal.NewMockIdentityProvider(sh.OAuthAuthProvider, passwordAuthProvider)
		sh.IdentityProvider = identityProvider
		hookProvider := hook.NewMockProvider()
		sh.HookProvider = hookProvider
		timeProvider := &coreTime.MockProvider{TimeNowUTC: time.Date(2006, 1, 2, 15, 4, 5, 0, time.UTC)}
		sh.TimeProvider = timeProvider
		mfaStore := mfa.NewMockStore(timeProvider)
		mfaConfiguration := &coreconfig.MFAConfiguration{
			Enabled:     false,
			Enforcement: coreconfig.MFAEnforcementOptional,
		}
		mfaSender := mfa.NewMockSender()
		mfaProvider := mfa.NewProvider(mfaStore, mfaConfiguration, timeProvider, mfaSender)
		sh.AuthnSessionProvider = authnsession.NewMockProvider(
			mfaConfiguration,
			timeProvider,
			mfaProvider,
			authInfoStore,
			sessionProvider,
			sessionWriter,
			identityProvider,
			hookProvider,
			userProfileStore,
		)
		validator := validation.NewValidator("http://v2.skygear.io")
		validator.AddSchemaFragments(
			AuthResultRequestSchema,
		)
		sh.Validator = validator

		Convey("invalid code verifier", func() {
			codeVerifier := "code_verifier"
			codeChallenge := "nonsense"
			code := &sso.SkygearAuthorizationCode{
				Action:              "login",
				CodeChallenge:       codeChallenge,
				UserID:              "john.doe.id",
				PrincipalID:         "john.doe.id",
				SessionCreateReason: "login",
			}
			encodedCode, err := mockProvider.EncodeSkygearAuthorizationCode(*code)
			So(err, ShouldBeNil)

			reqBody := map[string]interface{}{
				"authorization_code": encodedCode,
				"code_verifier":      codeVerifier,
			}
			reqBodyBytes, err := json.Marshal(reqBody)
			So(err, ShouldBeNil)

			req, _ := http.NewRequest("POST", "", bytes.NewReader(reqBodyBytes))
			req.Header.Set("Content-Type", "application/json")
			recorder := httptest.NewRecorder()
			sh.ServeHTTP(recorder, req)

			So(recorder.Result().StatusCode, ShouldEqual, 401)
			So(recorder.Body.Bytes(), ShouldEqualJSON, `
			{
				"error": {
					"code": 401,
					"info": {
						"cause": {
							"kind": "InvalidCodeVerifier"
						}
					},
					"message": "invalid code verifier",
					"name": "Unauthorized",
					"reason": "SSOFailed"
				}
			}
			`)

		})

		Convey("action = login", func() {
			codeVerifier := "code_verifier"
			codeChallenge := codeVerifierToCodeChallenge(codeVerifier)
			code := &sso.SkygearAuthorizationCode{
				Action:              "login",
				CodeChallenge:       codeChallenge,
				UserID:              "john.doe.id",
				PrincipalID:         "john.doe.id",
				SessionCreateReason: "login",
			}
			encodedCode, err := mockProvider.EncodeSkygearAuthorizationCode(*code)
			So(err, ShouldBeNil)

			reqBody := map[string]interface{}{
				"authorization_code": encodedCode,
				"code_verifier":      codeVerifier,
			}
			reqBodyBytes, err := json.Marshal(reqBody)
			So(err, ShouldBeNil)

			req, _ := http.NewRequest("POST", "", bytes.NewReader(reqBodyBytes))
			req.Header.Set("Content-Type", "application/json")
			recorder := httptest.NewRecorder()
			sh.ServeHTTP(recorder, req)

			So(recorder.Result().StatusCode, ShouldEqual, 200)
			So(recorder.Body.Bytes(), ShouldEqualJSON, `
			{
				"result": {
					"access_token": "access-token-john.doe.id-john.doe.id-0",
					"identity": {
						"claims": null,
						"id": "john.doe.id",
						"provider_keys": {},
						"provider_type": "google",
						"provider_user_id": "mock_user_id",
						"raw_profile": null,
						"type": "oauth"
					},
					"session_id": "john.doe.id-john.doe.id-0",
					"user": {
						"created_at": "0001-01-01T00:00:00Z",
						"id": "john.doe.id",
						"is_disabled": false,
						"is_manually_verified": false,
						"is_verified": false,
						"metadata": {},
						"verify_info": {}
					}
				}
			}
			`)
		})

		Convey("action = link", func() {
			codeVerifier := "code_verifier"
			codeChallenge := codeVerifierToCodeChallenge(codeVerifier)
			code := &sso.SkygearAuthorizationCode{
				Action:        "link",
				CodeChallenge: codeChallenge,
				UserID:        "john.doe.id",
				PrincipalID:   "john.doe.id",
			}
			encodedCode, err := mockProvider.EncodeSkygearAuthorizationCode(*code)
			So(err, ShouldBeNil)

			reqBody := map[string]interface{}{
				"authorization_code": encodedCode,
				"code_verifier":      codeVerifier,
			}
			reqBodyBytes, err := json.Marshal(reqBody)
			So(err, ShouldBeNil)

			req, _ := http.NewRequest("POST", "", bytes.NewReader(reqBodyBytes))
			req.Header.Set("Content-Type", "application/json")
			recorder := httptest.NewRecorder()
			sh.ServeHTTP(recorder, req)

			So(recorder.Result().StatusCode, ShouldEqual, 200)
			So(recorder.Body.Bytes(), ShouldEqualJSON, `
			{
				"result": {
					"user": {
						"created_at": "0001-01-01T00:00:00Z",
						"id": "john.doe.id",
						"is_disabled": false,
						"is_manually_verified": false,
						"is_verified": false,
						"metadata": {},
						"verify_info": {}
					}
				}
			}
			`)
		})
	})
}
<commit_msg>Unbreak auth_result_test<commit_after>package sso

import (
	"bytes"
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"net/url"
	"testing"

	. "github.com/smartystreets/goconvey/convey"

	"github.com/skygeario/skygear-server/pkg/auth/dependency/hook"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/sso"
	coreconfig "github.com/skygeario/skygear-server/pkg/core/config"
	"github.com/skygeario/skygear-server/pkg/core/db"
	. "github.com/skygeario/skygear-server/pkg/core/skytest"
	"github.com/skygeario/skygear-server/pkg/core/validation"
)

func TestAuthResultHandler(t *testing.T) {
	stateJWTSecret := "secret"
	providerName := "mock"
	providerUserID := "mock_user_id"

	Convey("AuthResultHandler", t, func() {
		sh := &AuthResultHandler{}
		sh.TxContext = db.NewMockTxContext()
		oauthConfig := &coreconfig.OAuthConfiguration{
			StateJWTSecret: stateJWTSecret,
		}
		providerConfig := coreconfig.OAuthProviderConfiguration{
			ID:           providerName,
			Type:         "google",
			ClientID:     "mock_client_id",
			ClientSecret: "mock_client_secret",
		}
		mockProvider := sso.MockSSOProvider{
			RedirectURIs: []string{
				"http://localhost",
			},
			URLPrefix:      &url.URL{Scheme: "https", Host: "api.example.com"},
			BaseURL:        "http://mock/auth",
			OAuthConfig:    oauthConfig,
			ProviderConfig: providerConfig,
			UserInfo: sso.ProviderUserInfo{
				ID:    providerUserID,
				Email: "mock@example.com",
			},
		}
		sh.SSOProvider = &mockProvider
		hookProvider := hook.NewMockProvider()
		sh.HookProvider = hookProvider
		validator := validation.NewValidator("http://v2.skygear.io")
		validator.AddSchemaFragments(
			AuthResultRequestSchema,
		)
		sh.Validator = validator
		sh.AuthnSessionProvider = &MockAuthnSessionProvider{}

		Convey("invalid code verifier", func() {
			codeVerifier := "code_verifier"
			codeChallenge := "nonsense"
			code := &sso.SkygearAuthorizationCode{
				Action:              "login",
				CodeChallenge:       codeChallenge,
				UserID:              "john.doe.id",
				PrincipalID:         "john.doe.id",
				SessionCreateReason: "login",
			}
			encodedCode, err := mockProvider.EncodeSkygearAuthorizationCode(*code)
			So(err, ShouldBeNil)

			reqBody := map[string]interface{}{
				"authorization_code": encodedCode,
				"code_verifier":      codeVerifier,
			}
			reqBodyBytes, err := json.Marshal(reqBody)
			So(err, ShouldBeNil)

			req, _ := http.NewRequest("POST", "", bytes.NewReader(reqBodyBytes))
			req.Header.Set("Content-Type", "application/json")
			recorder := httptest.NewRecorder()
			sh.ServeHTTP(recorder, req)

			So(recorder.Result().StatusCode, ShouldEqual, 401)
			So(recorder.Body.Bytes(), ShouldEqualJSON, `
			{
				"error": {
					"code": 401,
					"info": {
						"cause": {
							"kind": "InvalidCodeVerifier"
						}
					},
					"message": "invalid code verifier",
					"name": "Unauthorized",
					"reason": "SSOFailed"
				}
			}
			`)
		})
	})
}
<|endoftext|>


<commit_before>/*
Copyright 2021 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package genericclioptions

import (
	"net/http"
	"strings"

	"github.com/google/uuid"
	"github.com/spf13/cobra"
)

const (
	kubectlCommandHeader = "X-Kubectl-Command"
	kubectlSessionHeader = "X-Kubectl-Session"
)

// CommandHeaderRoundTripper adds a layer around the standard
// round tripper to add Request headers before delegation. Implements
// the go standard library "http.RoundTripper" interface.
type CommandHeaderRoundTripper struct {
	Delegate http.RoundTripper
	Headers  map[string]string
}

// CommandHeaderRoundTripper adds Request headers before delegating to standard
// round tripper. These headers are kubectl command headers which
// detail the kubectl command. See SIG CLI KEP 859:
//   https://github.com/kubernetes/enhancements/tree/master/keps/sig-cli/859-kubectl-headers
func (c *CommandHeaderRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
	for header, value := range c.Headers {
		req.Header.Set(header, value)
	}
	return c.Delegate.RoundTrip(req)
}

// ParseCommandHeaders fills in a map of X-Headers into the CommandHeaderRoundTripper. These
// headers are then filled into each request. For details on X-Headers see:
//   https://github.com/kubernetes/enhancements/tree/master/keps/sig-cli/859-kubectl-headers
// Each call overwrites the previously parsed command headers (not additive).
// TODO(seans3): Parse/add flags removing PII from flag values.
func (c *CommandHeaderRoundTripper) ParseCommandHeaders(cmd *cobra.Command, args []string) {
	if cmd == nil {
		return
	}
	// Overwrites previously parsed command headers (headers not additive).
	c.Headers = map[string]string{}
	// Session identifier to aggregate multiple Requests from single kubectl command.
	uid := uuid.New().String()
	c.Headers[kubectlSessionHeader] = uid
	// Iterate up the hierarchy of commands from the leaf command to create
	// the full command string. Example: kubectl create secret generic
	cmdStrs := []string{}
	for cmd.HasParent() {
		parent := cmd.Parent()
		currName := strings.TrimSpace(cmd.Name())
		cmdStrs = append([]string{currName}, cmdStrs...)
		cmd = parent
	}
	currName := strings.TrimSpace(cmd.Name())
	cmdStrs = append([]string{currName}, cmdStrs...)
	if len(cmdStrs) > 0 {
		c.Headers[kubectlCommandHeader] = strings.Join(cmdStrs, " ")
	}
}
<commit_msg>Changes headers to IETF standard<commit_after>/*
Copyright 2021 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package genericclioptions

import (
	"net/http"
	"strings"

	"github.com/google/uuid"
	"github.com/spf13/cobra"
)

const (
	kubectlCommandHeader = "Kubectl-Command"
	kubectlSessionHeader = "Kubectl-Session"
)

// CommandHeaderRoundTripper adds a layer around the standard
// round tripper to add Request headers before delegation. Implements
// the go standard library "http.RoundTripper" interface.
type CommandHeaderRoundTripper struct {
	Delegate http.RoundTripper
	Headers  map[string]string
}

// CommandHeaderRoundTripper adds Request headers before delegating to standard
// round tripper. These headers are kubectl command headers which
// detail the kubectl command. See SIG CLI KEP 859:
//   https://github.com/kubernetes/enhancements/tree/master/keps/sig-cli/859-kubectl-headers
func (c *CommandHeaderRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
	for header, value := range c.Headers {
		req.Header.Set(header, value)
	}
	return c.Delegate.RoundTrip(req)
}

// ParseCommandHeaders fills in a map of X-Headers into the CommandHeaderRoundTripper. These
// headers are then filled into each request. For details on X-Headers see:
//   https://github.com/kubernetes/enhancements/tree/master/keps/sig-cli/859-kubectl-headers
// Each call overwrites the previously parsed command headers (not additive).
// TODO(seans3): Parse/add flags removing PII from flag values.
func (c *CommandHeaderRoundTripper) ParseCommandHeaders(cmd *cobra.Command, args []string) {
	if cmd == nil {
		return
	}
	// Overwrites previously parsed command headers (headers not additive).
	c.Headers = map[string]string{}
	// Session identifier to aggregate multiple Requests from single kubectl command.
	uid := uuid.New().String()
	c.Headers[kubectlSessionHeader] = uid
	// Iterate up the hierarchy of commands from the leaf command to create
	// the full command string. Example: kubectl create secret generic
	cmdStrs := []string{}
	for cmd.HasParent() {
		parent := cmd.Parent()
		currName := strings.TrimSpace(cmd.Name())
		cmdStrs = append([]string{currName}, cmdStrs...)
		cmd = parent
	}
	currName := strings.TrimSpace(cmd.Name())
	cmdStrs = append([]string{currName}, cmdStrs...)
	if len(cmdStrs) > 0 {
		c.Headers[kubectlCommandHeader] = strings.Join(cmdStrs, " ")
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The TestGrid Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package naiveanalyzer

import (
	summarypb "github.com/GoogleCloudPlatform/testgrid/pb/summary"
	"github.com/GoogleCloudPlatform/testgrid/pkg/summarizer/common"
	"github.com/golang/protobuf/ptypes/timestamp"
)

const analyzerName = "naiveanalyzer"

// IntString is for sorting, primarily intended for map[string]int as implemented below
type IntString struct {
	s string
	i int
}

// NaiveAnalyzer implements functions that calculate flakiness as a ratio of failed tests to total tests
type NaiveAnalyzer struct {
}

// GetFlakiness returns a HealthinessInfo message with data to display flakiness as a ratio of failed tests
// to total tests
func (na *NaiveAnalyzer) GetFlakiness(gridMetrics []*common.GridMetrics, minRuns int, startDate int, endDate int, tab string) *summarypb.HealthinessInfo {
	testInfoList := []*summarypb.TestInfo{}
	for _, test := range gridMetrics {
		testInfo, success := calculateNaiveFlakiness(test, minRuns)
		if !success {
			continue
		}
		// TODO (itsazhuhere@): Introduce name parsing into test name and env
		testInfo.DisplayName = test.Name
		testInfoList = append(testInfoList, testInfo)
	}
	// Populate Healthiness with above calculated information
	healthiness := createHealthiness(startDate, endDate, testInfoList)
	return healthiness
}

func createHealthiness(startDate int, endDate int, testInfoList []*summarypb.TestInfo) *summarypb.HealthinessInfo {
	healthiness := &summarypb.HealthinessInfo{
		Start: intToTimestamp(startDate),
		End:   intToTimestamp(endDate),
		Tests: testInfoList,
	}

	var averageFlakiness float32
	for _, testInfo := range healthiness.Tests {
		averageFlakiness += testInfo.Flakiness
	}
	totalTests := int32(len(healthiness.Tests))
	if totalTests > 0 {
		healthiness.AverageFlakiness = averageFlakiness / float32(totalTests)
	}
	return healthiness
}

func calculateNaiveFlakiness(test *common.GridMetrics, minRuns int) (*summarypb.TestInfo, bool) {
	failedCount := int32(test.Failed)
	totalCount := int32(test.Passed) + int32(test.Failed)
	totalCountWithInfra := totalCount + int32(test.FailedInfraCount)
	if totalCount < int32(minRuns) {
		return &summarypb.TestInfo{}, false
	}
	flakiness := 100 * float32(failedCount) / float32(totalCount)
	testInfo := &summarypb.TestInfo{
		Flakiness:          flakiness,
		TotalNonInfraRuns:  totalCount,
		TotalRunsWithInfra: totalCountWithInfra,
		PassedNonInfraRuns: int32(test.Passed),
		FailedNonInfraRuns: int32(test.Failed),
		FailedInfraRuns:    int32(test.FailedInfraCount),
	}
	return testInfo, true

}

func intToTimestamp(seconds int) *timestamp.Timestamp {
	timestamp := &timestamp.Timestamp{
		Seconds: int64(seconds),
	}
	return timestamp
}
<commit_msg>Avoid potential divide by zero operations<commit_after>/*
Copyright 2020 The TestGrid Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package naiveanalyzer

import (
	summarypb "github.com/GoogleCloudPlatform/testgrid/pb/summary"
	"github.com/GoogleCloudPlatform/testgrid/pkg/summarizer/common"
	"github.com/golang/protobuf/ptypes/timestamp"
)

const analyzerName = "naiveanalyzer"

// IntString is for sorting, primarily intended for map[string]int as implemented below
type IntString struct {
	s string
	i int
}

// NaiveAnalyzer implements functions that calculate flakiness as a ratio of failed tests to total tests
type NaiveAnalyzer struct {
}

// GetFlakiness returns a HealthinessInfo message with data to display flakiness as a ratio of failed tests
// to total tests
func (na *NaiveAnalyzer) GetFlakiness(gridMetrics []*common.GridMetrics, minRuns int, startDate int, endDate int, tab string) *summarypb.HealthinessInfo {
	testInfoList := []*summarypb.TestInfo{}
	for _, test := range gridMetrics {
		testInfo, success := calculateNaiveFlakiness(test, minRuns)
		if !success {
			continue
		}
		// TODO (itsazhuhere@): Introduce name parsing into test name and env
		testInfo.DisplayName = test.Name
		testInfoList = append(testInfoList, testInfo)
	}
	// Populate Healthiness with above calculated information
	healthiness := createHealthiness(startDate, endDate, testInfoList)
	return healthiness
}

func createHealthiness(startDate int, endDate int, testInfoList []*summarypb.TestInfo) *summarypb.HealthinessInfo {
	healthiness := &summarypb.HealthinessInfo{
		Start: intToTimestamp(startDate),
		End:   intToTimestamp(endDate),
		Tests: testInfoList,
	}

	var averageFlakiness float32
	for _, testInfo := range healthiness.Tests {
		averageFlakiness += testInfo.Flakiness
	}
	totalTests := int32(len(healthiness.Tests))
	if totalTests > 0 {
		healthiness.AverageFlakiness = averageFlakiness / float32(totalTests)
	}
	return healthiness
}

func calculateNaiveFlakiness(test *common.GridMetrics, minRuns int) (*summarypb.TestInfo, bool) {
	failedCount := int32(test.Failed)
	totalCount := int32(test.Passed) + int32(test.Failed)
	totalCountWithInfra := totalCount + int32(test.FailedInfraCount)
	if totalCount <= 0 || totalCount < int32(minRuns) {
		return &summarypb.TestInfo{}, false
	}
	flakiness := 100 * float32(failedCount) / float32(totalCount)
	testInfo := &summarypb.TestInfo{
		Flakiness:          flakiness,
		TotalNonInfraRuns:  totalCount,
		TotalRunsWithInfra: totalCountWithInfra,
		PassedNonInfraRuns: int32(test.Passed),
		FailedNonInfraRuns: int32(test.Failed),
		FailedInfraRuns:    int32(test.FailedInfraCount),
	}
	return testInfo, true

}

func intToTimestamp(seconds int) *timestamp.Timestamp {
	timestamp := &timestamp.Timestamp{
		Seconds: int64(seconds),
	}
	return timestamp
}
<|endoftext|>


<commit_before>package main

import "fmt"

// A Word represents a coefficient of a WordPoly.
// TODO(akalin): Use uintptr instead.
type Word uint32

// The size of Word in bits.
const WORD_BITS = 32

// A WordPoly represents a polynomial with Word coefficients.
//
// The zero value for a WordPoly represents the zero polynomial.
type WordPoly struct {
	coeffs []Word
}

// Only polynomials built with the same value of N and R may be used
// together in one of the functions below.

// Builds a new WordPoly representing the zero polynomial
// mod (N, X^R - 1). R must fit into an int.
func NewWordPoly(N, R Word) *WordPoly {
	return &WordPoly{make([]Word, R)}
}

// Sets p to X^k + a mod (N, X^R - 1).
func (p *WordPoly) Set(a, k, N Word) {
	R := len(p.coeffs)
	p.coeffs[0] = a % N
	for i := 1; i < R; i++ {
		p.coeffs[i] = 0
	}
	p.coeffs[int(k%Word(R))] = 1
}

// Returns whether p has the same coefficients as q.
func (p *WordPoly) Eq(q *WordPoly) bool {
	R := len(p.coeffs)
	for i := 0; i < R; i++ {
		if p.coeffs[i] != q.coeffs[i] {
			return false
		}
	}
	return true
}

// Sets p to the product of p and q mod (N, X^R - 1). tmp must not
// alias p or q.
func (p *WordPoly) mul(q *WordPoly, N Word, tmp *WordPoly) {
	R := len(tmp.coeffs)
	for i := 0; i < R; i++ {
		tmp.coeffs[i] = 0
	}

	for i := 0; i < R; i++ {
		for j := 0; j < R; j++ {
			k := (i + j) % R
			// TODO(akalin): Handle overflow here when we
			// change Word to uintptr.
			e := uint64(p.coeffs[i]) * uint64(q.coeffs[j])
			e %= uint64(N)
			e += uint64(tmp.coeffs[k])
			e %= uint64(N)
			tmp.coeffs[k] = Word(e)
		}
	}
	p.coeffs, tmp.coeffs = tmp.coeffs, p.coeffs
}

// Sets p to p^N mod (N, X^R - 1), where R is the size of p. N must be
// positive, and tmp1 and tmp2 must not alias each other or p.
func (p *WordPoly) Pow(N Word, tmp1, tmp2 *WordPoly) {
	R := len(p.coeffs)
	for i := 0; i < R; i++ {
		tmp1.coeffs[i] = p.coeffs[i]
	}

	// Find N's highest set bit.
	i := WORD_BITS - 1
	for ; (i >= 0) && ((N & (1 << uint(i))) == 0); i-- {
	}

	for i--; i >= 0; i-- {
		tmp1.mul(tmp1, N, tmp2)
		if (N & (1 << uint(i))) != 0 {
			tmp1.mul(p, N, tmp2)
		}
	}
	p.coeffs, tmp1.coeffs = tmp1.coeffs, p.coeffs
}

// fmt.Formatter implementation.
func (p *WordPoly) Format(f fmt.State, c rune) {
	i := len(p.coeffs) - 1
	for ; i >= 0 && p.coeffs[i] == 0; i-- {
	}

	if i < 0 {
		fmt.Fprint(f, "0")
		return
	}

	// Formats coeff*x^deg.
	formatNonZeroMonomial := func(f fmt.State, c rune, coeff, deg Word) {
		if coeff != 1 || deg == 0 {
			fmt.Fprint(f, coeff)
		}
		if deg != 0 {
			fmt.Fprint(f, "x")
			if deg > 1 {
				fmt.Fprint(f, "^", deg)
			}
		}
	}

	formatNonZeroMonomial(f, c, p.coeffs[i], Word(i))

	for i--; i >= 0; i-- {
		if p.coeffs[i] != 0 {
			fmt.Fprint(f, " + ")
			formatNonZeroMonomial(f, c, p.coeffs[i], Word(i))
		}
	}
}
<commit_msg>Avoid modulo operation in inner loop of WordPoly.mul()<commit_after>package main

import "fmt"

// A Word represents a coefficient of a WordPoly.
// TODO(akalin): Use uintptr instead.
type Word uint32

// The size of Word in bits.
const WORD_BITS = 32

// A WordPoly represents a polynomial with Word coefficients.
//
// The zero value for a WordPoly represents the zero polynomial.
type WordPoly struct {
	coeffs []Word
}

// Only polynomials built with the same value of N and R may be used
// together in one of the functions below.

// Builds a new WordPoly representing the zero polynomial
// mod (N, X^R - 1). R must fit into an int.
func NewWordPoly(N, R Word) *WordPoly {
	return &WordPoly{make([]Word, R)}
}

// Sets p to X^k + a mod (N, X^R - 1).
func (p *WordPoly) Set(a, k, N Word) {
	R := len(p.coeffs)
	p.coeffs[0] = a % N
	for i := 1; i < R; i++ {
		p.coeffs[i] = 0
	}
	p.coeffs[int(k%Word(R))] = 1
}

// Returns whether p has the same coefficients as q.
func (p *WordPoly) Eq(q *WordPoly) bool {
	R := len(p.coeffs)
	for i := 0; i < R; i++ {
		if p.coeffs[i] != q.coeffs[i] {
			return false
		}
	}
	return true
}

// Sets p to the product of p and q mod (N, X^R - 1). tmp must not
// alias p or q.
func (p *WordPoly) mul(q *WordPoly, N Word, tmp *WordPoly) {
	R := len(tmp.coeffs)
	for i := 0; i < R; i++ {
		tmp.coeffs[i] = 0
	}

	// Optimized and unrolled version of the following loop:
	//
	//   for i, j < R {
	//     tmp_{(i + j) % R} += (p_i * q_j) % N
	//   }
	for i := 0; i < R; i++ {
		for j := 0; j < R-i; j++ {
			k := i + j
			// TODO(akalin): Handle overflow here when we
			// change Word to uintptr.
			e := uint64(p.coeffs[i]) * uint64(q.coeffs[j])
			e %= uint64(N)
			e += uint64(tmp.coeffs[k])
			e %= uint64(N)
			tmp.coeffs[k] = Word(e)
		}
		for j := R - i; j < R; j++ {
			k := j - (R - i)
			// Duplicate of loop above.
			e := uint64(p.coeffs[i]) * uint64(q.coeffs[j])
			e %= uint64(N)
			e += uint64(tmp.coeffs[k])
			e %= uint64(N)
			tmp.coeffs[k] = Word(e)
		}
	}

	p.coeffs, tmp.coeffs = tmp.coeffs, p.coeffs
}

// Sets p to p^N mod (N, X^R - 1), where R is the size of p. N must be
// positive, and tmp1 and tmp2 must not alias each other or p.
func (p *WordPoly) Pow(N Word, tmp1, tmp2 *WordPoly) {
	R := len(p.coeffs)
	for i := 0; i < R; i++ {
		tmp1.coeffs[i] = p.coeffs[i]
	}

	// Find N's highest set bit.
	i := WORD_BITS - 1
	for ; (i >= 0) && ((N & (1 << uint(i))) == 0); i-- {
	}

	for i--; i >= 0; i-- {
		tmp1.mul(tmp1, N, tmp2)
		if (N & (1 << uint(i))) != 0 {
			tmp1.mul(p, N, tmp2)
		}
	}
	p.coeffs, tmp1.coeffs = tmp1.coeffs, p.coeffs
}

// fmt.Formatter implementation.
func (p *WordPoly) Format(f fmt.State, c rune) {
	i := len(p.coeffs) - 1
	for ; i >= 0 && p.coeffs[i] == 0; i-- {
	}

	if i < 0 {
		fmt.Fprint(f, "0")
		return
	}

	// Formats coeff*x^deg.
	formatNonZeroMonomial := func(f fmt.State, c rune, coeff, deg Word) {
		if coeff != 1 || deg == 0 {
			fmt.Fprint(f, coeff)
		}
		if deg != 0 {
			fmt.Fprint(f, "x")
			if deg > 1 {
				fmt.Fprint(f, "^", deg)
			}
		}
	}

	formatNonZeroMonomial(f, c, p.coeffs[i], Word(i))

	for i--; i >= 0; i-- {
		if p.coeffs[i] != 0 {
			fmt.Fprint(f, " + ")
			formatNonZeroMonomial(f, c, p.coeffs[i], Word(i))
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/alphagov/govuk_crawler_worker/http_crawler"
	"github.com/alphagov/govuk_crawler_worker/queue"
	"github.com/alphagov/govuk_crawler_worker/ttl_hash_set"
	"github.com/alphagov/govuk_crawler_worker/util"
	"github.com/streadway/amqp"
)

const NotRecentlyCrawled int = 0
const AlreadyCrawled int = -1

func ReadFromQueue(
	inboundChannel <-chan amqp.Delivery,
	rootURL *url.URL,
	ttlHashSet *ttl_hash_set.TTLHashSet,
	blacklistPaths []string,
	crawlerThreads int,
) chan *CrawlerMessageItem {
	outboundChannel := make(chan *CrawlerMessageItem, crawlerThreads)

	readLoop := func(
		inbound <-chan amqp.Delivery,
		outbound chan<- *CrawlerMessageItem,
		ttlHashSet *ttl_hash_set.TTLHashSet,
		blacklistPaths []string,
	) {
		for item := range inbound {
			start := time.Now()
			message := NewCrawlerMessageItem(item, rootURL, blacklistPaths)

			if message.IsBlacklisted() {
				item.Ack(false)
				log.Infoln("URL is blacklisted (acknowledging):", message.URL())
				continue
			}

			crawlCount, err := ttlHashSet.Get(message.URL())
			if err != nil {
				item.Reject(true)
				log.Errorln("Couldn't check existence of (rejecting):", message.URL(), err)
				continue
			}

			if crawlCount == AlreadyCrawled {
				log.Infoln("URL read from queue already crawled:", message.URL())
				if err = item.Ack(false); err != nil {
					log.Errorln("Ack failed (ReadFromQueue): ", message.URL())
				}
				continue
			}

			outbound <- message

			util.StatsDTiming("read_from_queue", start, time.Now())
		}
	}

	go readLoop(inboundChannel, outboundChannel, ttlHashSet, blacklistPaths)

	return outboundChannel
}

func CrawlURL(
	ttlHashSet *ttl_hash_set.TTLHashSet,
	crawlChannel <-chan *CrawlerMessageItem,
	crawler *http_crawler.Crawler,
	crawlerThreads int,
	maxCrawlRetries int,
) <-chan *CrawlerMessageItem {
	if crawlerThreads < 1 {
		panic("cannot start a negative or zero number of crawler threads")
	}

	extractChannel := make(chan *CrawlerMessageItem, 2)

	crawlLoop := func(
		ttlHashSet *ttl_hash_set.TTLHashSet,
		crawl <-chan *CrawlerMessageItem,
		extract chan<- *CrawlerMessageItem,
		crawler *http_crawler.Crawler,
		maxCrawlRetries int,
	) {
		for item := range crawl {
			start := time.Now()
			u, err := url.Parse(item.URL())
			if err != nil {
				item.Reject(false)
				log.Warningln("Couldn't crawl, invalid URL (rejecting):", item.URL(), err)
				continue
			}
			log.Infoln("Crawling URL:", u)

			crawlCount, err := ttlHashSet.Get(u.String())
			if err != nil {
				item.Reject(false)
				log.Errorln("Couldn't confirm existence of URL (rejecting):", u.String(), err)
				continue
			}

			if crawlCount == maxCrawlRetries {
				item.Reject(false)
				log.Errorf("Aborting crawl of URL which has been retried %d times (rejecting): %s", maxCrawlRetries, u.String())
				continue
			}

			body, err := crawler.Crawl(u)
			if err != nil {
				switch err {
				case http_crawler.RetryRequest5XXError, http_crawler.RetryRequest429Error:
					switch err {
					case http_crawler.RetryRequest5XXError:
						ttlHashSet.Incr(u.String())
					case http_crawler.RetryRequest429Error:
						sleepTime := 5 * time.Second

						// Back off from crawling for a few seconds.
						log.Warningln("Sleeping for: ", sleepTime, " seconds. Received 429 HTTP status")
						time.Sleep(sleepTime)
					}

					item.Reject(true)
					log.Warningln("Couldn't crawl (requeueing):", u.String(), err)
				case http_crawler.RedirectError:
					setErr := ttlHashSet.Set(u.String(), AlreadyCrawled)
					if setErr != nil {
						log.Errorln("Couldn't mark item as already crawled:", u.String(), setErr)
					}

					item.Reject(false)
					log.Warningln("Couldn't crawl (rejecting):", u.String(), err)
				default:
					item.Reject(false)
					log.Warningln("Couldn't crawl (rejecting):", u.String(), err)
				}

				continue
			}

			item.HTMLBody = body

			if item.IsHTML() {
				extract <- item
			} else {
				if err = item.Ack(false); err != nil {
					log.Errorln("Ack failed (CrawlURL): ", item.URL())
				}

				err = ttlHashSet.Set(item.URL(), AlreadyCrawled)
				if err != nil {
					log.Errorln("Couldn't mark item as already crawled:", item.URL(), err)
				}
			}

			util.StatsDTiming("crawl_url", start, time.Now())
		}
	}

	for i := 1; i <= crawlerThreads; i++ {
		go crawlLoop(ttlHashSet, crawlChannel, extractChannel, crawler, maxCrawlRetries)
	}

	return extractChannel
}

func WriteItemToDisk(basePath string, crawlChannel <-chan *CrawlerMessageItem) <-chan *CrawlerMessageItem {
	extractChannel := make(chan *CrawlerMessageItem, 2)

	writeLoop := func(
		crawl <-chan *CrawlerMessageItem,
		extract chan<- *CrawlerMessageItem,
	) {
		for item := range crawl {
			start := time.Now()
			relativeFilePath, err := item.RelativeFilePath()

			if err != nil {
				item.Reject(false)
				log.Errorln("Couldn't write to disk (rejecting):", err)
				continue
			}

			filePath := filepath.Join(basePath, relativeFilePath)
			basePath := filepath.Dir(filePath)
			err = os.MkdirAll(basePath, 0755)

			if err != nil {
				item.Reject(false)
				log.Errorln("Couldn't write to disk (rejecting):", filePath, err)
				continue
			}

			err = ioutil.WriteFile(filePath, item.HTMLBody, 0644)

			if err != nil {
				item.Reject(false)
				log.Errorln("Couldn't write to disk (rejecting):", filePath, err)
				continue
			}

			log.Infoln("Wrote URL body to disk for:", item.URL())
			extract <- item

			util.StatsDTiming("write_to_disk", start, time.Now())
		}
	}

	go writeLoop(crawlChannel, extractChannel)

	return extractChannel
}

func ExtractURLs(extractChannel <-chan *CrawlerMessageItem) (<-chan string, <-chan *CrawlerMessageItem) {
	publishChannel := make(chan string, 100)
	acknowledgeChannel := make(chan *CrawlerMessageItem, 1)

	extractLoop := func(
		extract <-chan *CrawlerMessageItem,
		publish chan<- string,
		acknowledge chan<- *CrawlerMessageItem,
	) {
		for item := range extract {
			start := time.Now()
			urls, err := item.ExtractURLs()
			if err != nil {
				item.Reject(false)
				log.Errorln("ExtractURLs (rejecting):", string(item.Body), err)

				continue
			}

			log.Infoln("Extracted URLs:", len(urls))

			for _, u := range urls {
				publish <- u.String()
			}

			acknowledge <- item

			util.StatsDTiming("extract_urls", start, time.Now())
		}
	}

	go extractLoop(extractChannel, publishChannel, acknowledgeChannel)

	return publishChannel, acknowledgeChannel
}

func PublishURLs(ttlHashSet *ttl_hash_set.TTLHashSet, queueManager *queue.QueueManager, publish <-chan string) {
	for url := range publish {
		start := time.Now()
		crawlCount, err := ttlHashSet.Get(url)

		if err != nil {
			log.Errorln("Couldn't check existence of URL:", url, err)
			continue
		}

		if crawlCount == AlreadyCrawled {
			log.Infoln("URL extracted from page already crawled:", url)
		} else if crawlCount == NotRecentlyCrawled {
			err = queueManager.Publish("#", "text/plain", url)
			if err != nil {
				log.Fatalln("Delivery failed:", url, err)
			}
		}

		util.StatsDGauge("publish_urls", int64(len(publish)))
		util.StatsDTiming("publish_urls", start, time.Now())
	}
}

func AcknowledgeItem(inbound <-chan *CrawlerMessageItem, ttlHashSet *ttl_hash_set.TTLHashSet) {
	for item := range inbound {
		start := time.Now()
		url := item.URL()

		err := ttlHashSet.Set(url, AlreadyCrawled)
		if err != nil {
			item.Reject(false)
			log.Errorln("Acknowledge failed (rejecting):", url, err)
			continue
		}

		if err = item.Ack(false); err != nil {
			log.Errorln("Ack failed (AcknowledgeItem): ", item.URL())
		}
		log.Infoln("Acknowledged:", url)

		util.StatsDTiming("acknowledge_item", start, time.Now())
	}
}
<commit_msg>Use fallthrough keyword to reduce duplication<commit_after>package main

import (
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/alphagov/govuk_crawler_worker/http_crawler"
	"github.com/alphagov/govuk_crawler_worker/queue"
	"github.com/alphagov/govuk_crawler_worker/ttl_hash_set"
	"github.com/alphagov/govuk_crawler_worker/util"
	"github.com/streadway/amqp"
)

const NotRecentlyCrawled int = 0
const AlreadyCrawled int = -1

func ReadFromQueue(
	inboundChannel <-chan amqp.Delivery,
	rootURL *url.URL,
	ttlHashSet *ttl_hash_set.TTLHashSet,
	blacklistPaths []string,
	crawlerThreads int,
) chan *CrawlerMessageItem {
	outboundChannel := make(chan *CrawlerMessageItem, crawlerThreads)

	readLoop := func(
		inbound <-chan amqp.Delivery,
		outbound chan<- *CrawlerMessageItem,
		ttlHashSet *ttl_hash_set.TTLHashSet,
		blacklistPaths []string,
	) {
		for item := range inbound {
			start := time.Now()
			message := NewCrawlerMessageItem(item, rootURL, blacklistPaths)

			if message.IsBlacklisted() {
				item.Ack(false)
				log.Infoln("URL is blacklisted (acknowledging):", message.URL())
				continue
			}

			crawlCount, err := ttlHashSet.Get(message.URL())
			if err != nil {
				item.Reject(true)
				log.Errorln("Couldn't check existence of (rejecting):", message.URL(), err)
				continue
			}

			if crawlCount == AlreadyCrawled {
				log.Infoln("URL read from queue already crawled:", message.URL())
				if err = item.Ack(false); err != nil {
					log.Errorln("Ack failed (ReadFromQueue): ", message.URL())
				}
				continue
			}

			outbound <- message

			util.StatsDTiming("read_from_queue", start, time.Now())
		}
	}

	go readLoop(inboundChannel, outboundChannel, ttlHashSet, blacklistPaths)

	return outboundChannel
}

func CrawlURL(
	ttlHashSet *ttl_hash_set.TTLHashSet,
	crawlChannel <-chan *CrawlerMessageItem,
	crawler *http_crawler.Crawler,
	crawlerThreads int,
	maxCrawlRetries int,
) <-chan *CrawlerMessageItem {
	if crawlerThreads < 1 {
		panic("cannot start a negative or zero number of crawler threads")
	}

	extractChannel := make(chan *CrawlerMessageItem, 2)

	crawlLoop := func(
		ttlHashSet *ttl_hash_set.TTLHashSet,
		crawl <-chan *CrawlerMessageItem,
		extract chan<- *CrawlerMessageItem,
		crawler *http_crawler.Crawler,
		maxCrawlRetries int,
	) {
		for item := range crawl {
			start := time.Now()
			u, err := url.Parse(item.URL())
			if err != nil {
				item.Reject(false)
				log.Warningln("Couldn't crawl, invalid URL (rejecting):", item.URL(), err)
				continue
			}
			log.Infoln("Crawling URL:", u)

			crawlCount, err := ttlHashSet.Get(u.String())
			if err != nil {
				item.Reject(false)
				log.Errorln("Couldn't confirm existence of URL (rejecting):", u.String(), err)
				continue
			}

			if crawlCount == maxCrawlRetries {
				item.Reject(false)
				log.Errorf("Aborting crawl of URL which has been retried %d times (rejecting): %s", maxCrawlRetries, u.String())
				continue
			}

			body, err := crawler.Crawl(u)
			if err != nil {
				switch err {
				case http_crawler.RetryRequest5XXError, http_crawler.RetryRequest429Error:
					switch err {
					case http_crawler.RetryRequest5XXError:
						ttlHashSet.Incr(u.String())
					case http_crawler.RetryRequest429Error:
						sleepTime := 5 * time.Second

						// Back off from crawling for a few seconds.
						log.Warningln("Sleeping for: ", sleepTime, " seconds. Received 429 HTTP status")
						time.Sleep(sleepTime)
					}

					item.Reject(true)
					log.Warningln("Couldn't crawl (requeueing):", u.String(), err)
				case http_crawler.RedirectError:
					setErr := ttlHashSet.Set(u.String(), AlreadyCrawled)
					if setErr != nil {
						log.Errorln("Couldn't mark item as already crawled:", u.String(), setErr)
					}

					fallthrough
				default:
					item.Reject(false)
					log.Warningln("Couldn't crawl (rejecting):", u.String(), err)
				}

				continue
			}

			item.HTMLBody = body

			if item.IsHTML() {
				extract <- item
			} else {
				if err = item.Ack(false); err != nil {
					log.Errorln("Ack failed (CrawlURL): ", item.URL())
				}

				err = ttlHashSet.Set(item.URL(), AlreadyCrawled)
				if err != nil {
					log.Errorln("Couldn't mark item as already crawled:", item.URL(), err)
				}
			}

			util.StatsDTiming("crawl_url", start, time.Now())
		}
	}

	for i := 1; i <= crawlerThreads; i++ {
		go crawlLoop(ttlHashSet, crawlChannel, extractChannel, crawler, maxCrawlRetries)
	}

	return extractChannel
}

func WriteItemToDisk(basePath string, crawlChannel <-chan *CrawlerMessageItem) <-chan *CrawlerMessageItem {
	extractChannel := make(chan *CrawlerMessageItem, 2)

	writeLoop := func(
		crawl <-chan *CrawlerMessageItem,
		extract chan<- *CrawlerMessageItem,
	) {
		for item := range crawl {
			start := time.Now()
			relativeFilePath, err := item.RelativeFilePath()

			if err != nil {
				item.Reject(false)
				log.Errorln("Couldn't write to disk (rejecting):", err)
				continue
			}

			filePath := filepath.Join(basePath, relativeFilePath)
			basePath := filepath.Dir(filePath)
			err = os.MkdirAll(basePath, 0755)

			if err != nil {
				item.Reject(false)
				log.Errorln("Couldn't write to disk (rejecting):", filePath, err)
				continue
			}

			err = ioutil.WriteFile(filePath, item.HTMLBody, 0644)

			if err != nil {
				item.Reject(false)
				log.Errorln("Couldn't write to disk (rejecting):", filePath, err)
				continue
			}

			log.Infoln("Wrote URL body to disk for:", item.URL())
			extract <- item

			util.StatsDTiming("write_to_disk", start, time.Now())
		}
	}

	go writeLoop(crawlChannel, extractChannel)

	return extractChannel
}

func ExtractURLs(extractChannel <-chan *CrawlerMessageItem) (<-chan string, <-chan *CrawlerMessageItem) {
	publishChannel := make(chan string, 100)
	acknowledgeChannel := make(chan *CrawlerMessageItem, 1)

	extractLoop := func(
		extract <-chan *CrawlerMessageItem,
		publish chan<- string,
		acknowledge chan<- *CrawlerMessageItem,
	) {
		for item := range extract {
			start := time.Now()
			urls, err := item.ExtractURLs()
			if err != nil {
				item.Reject(false)
				log.Errorln("ExtractURLs (rejecting):", string(item.Body), err)

				continue
			}

			log.Infoln("Extracted URLs:", len(urls))

			for _, u := range urls {
				publish <- u.String()
			}

			acknowledge <- item

			util.StatsDTiming("extract_urls", start, time.Now())
		}
	}

	go extractLoop(extractChannel, publishChannel, acknowledgeChannel)

	return publishChannel, acknowledgeChannel
}

func PublishURLs(ttlHashSet *ttl_hash_set.TTLHashSet, queueManager *queue.QueueManager, publish <-chan string) {
	for url := range publish {
		start := time.Now()
		crawlCount, err := ttlHashSet.Get(url)

		if err != nil {
			log.Errorln("Couldn't check existence of URL:", url, err)
			continue
		}

		if crawlCount == AlreadyCrawled {
			log.Infoln("URL extracted from page already crawled:", url)
		} else if crawlCount == NotRecentlyCrawled {
			err = queueManager.Publish("#", "text/plain", url)
			if err != nil {
				log.Fatalln("Delivery failed:", url, err)
			}
		}

		util.StatsDGauge("publish_urls", int64(len(publish)))
		util.StatsDTiming("publish_urls", start, time.Now())
	}
}

func AcknowledgeItem(inbound <-chan *CrawlerMessageItem, ttlHashSet *ttl_hash_set.TTLHashSet) {
	for item := range inbound {
		start := time.Now()
		url := item.URL()

		err := ttlHashSet.Set(url, AlreadyCrawled)
		if err != nil {
			item.Reject(false)
			log.Errorln("Acknowledge failed (rejecting):", url, err)
			continue
		}

		if err = item.Ack(false); err != nil {
			log.Errorln("Ack failed (AcknowledgeItem): ", item.URL())
		}
		log.Infoln("Acknowledged:", url)

		util.StatsDTiming("acknowledge_item", start, time.Now())
	}
}
<|endoftext|>


<commit_before>package main

import (
	"log"

	"github.com/alphagov/govuk_crawler_worker/http_crawler"
	"github.com/alphagov/govuk_crawler_worker/queue"
	"github.com/alphagov/govuk_crawler_worker/ttl_hash_set"
	"github.com/streadway/amqp"
)

func AcknowledgeItem(inbound <-chan *CrawlerMessageItem, ttlHashSet *ttl_hash_set.TTLHashSet) {
	for item := range inbound {
		url := item.URL()

		_, err := ttlHashSet.Add(url)
		if err != nil {
			item.Reject(false)
			log.Println("Acknowledge failed (rejecting):", url, err)
			continue
		}

		item.Ack(false)
		log.Println("Acknowledged:", url)
	}
}

func CrawlURL(crawlChannel <-chan *CrawlerMessageItem, crawler *http_crawler.Crawler) <-chan *CrawlerMessageItem {
	extract := make(chan *CrawlerMessageItem, 2)

	for i := 0; i < 2; i++ {
		go func() {
			for item := range crawlChannel {
				url := item.URL()
				log.Println("Crawling URL:", url)

				body, err := crawler.Crawl(url)
				if err != nil {
					item.Reject(false)
					log.Println("Couldn't crawl (rejecting):", url, err)
					continue
				}

				item.HTMLBody = body

				if item.IsHTML() {
					extract <- item
				} else {
					item.Ack(false)
				}
			}
		}()
	}

	return extract
}

func ExtractURLs(extract <-chan *CrawlerMessageItem) (<-chan string, <-chan *CrawlerMessageItem) {
	publishChannel := make(chan string, 100)
	acknowledgeChannel := make(chan *CrawlerMessageItem, 1)

	go func() {
		for item := range extract {
			urls, err := item.ExtractURLs()
			if err != nil {
				item.Reject(false)
				log.Println("ExtractURLs (rejecting):", string(item.Body), err)
			}

			log.Println("Extracted URLs:", len(urls))

			for _, url := range urls {
				publishChannel <- url
			}

			acknowledgeChannel <- item
		}
	}()

	return publishChannel, acknowledgeChannel
}

func PublishURLs(ttlHashSet *ttl_hash_set.TTLHashSet, queueManager *queue.QueueManager, publish <-chan string) {
	for url := range publish {
		exists, err := ttlHashSet.Exists(url)

		if err != nil {
			log.Println("Couldn't check existence of URL:", url, err)

			if err.Error() == "use of closed network connection" {
				log.Fatalln("No connection to Redis:", err)
			}
		}

		if !exists {
			err = queueManager.Publish("#", "text/plain", url)
			if err != nil {
				log.Fatalln("Delivery failed:", url, err)
			}
		}
	}
}

func ReadFromQueue(inbound <-chan amqp.Delivery, ttlHashSet *ttl_hash_set.TTLHashSet, blacklistPaths []string) chan *CrawlerMessageItem {
	outbound := make(chan *CrawlerMessageItem, 2)

	go func() {
		for item := range inbound {
			message := NewCrawlerMessageItem(item, "", blacklistPaths)

			exists, err := ttlHashSet.Exists(message.URL())
			if err != nil {
				if err.Error() == "use of closed network connection" {
					log.Fatalln("No connection to Redis:", err)
				} else {
					item.Reject(true)
					log.Println("Couldn't check existence of (rejecting):", message.URL(), err)
					continue
				}
			}

			if !exists {
				outbound <- message
			} else {
				log.Println("URL already crawled:", message.URL())
				item.Ack(false)
			}
		}
	}()

	return outbound
}
<commit_msg>Create multiple go routines explicitly; using anonymous funcs<commit_after>package main

import (
	"log"

	"github.com/alphagov/govuk_crawler_worker/http_crawler"
	"github.com/alphagov/govuk_crawler_worker/queue"
	"github.com/alphagov/govuk_crawler_worker/ttl_hash_set"
	"github.com/streadway/amqp"
)

func AcknowledgeItem(inbound <-chan *CrawlerMessageItem, ttlHashSet *ttl_hash_set.TTLHashSet) {
	for item := range inbound {
		url := item.URL()

		_, err := ttlHashSet.Add(url)
		if err != nil {
			item.Reject(false)
			log.Println("Acknowledge failed (rejecting):", url, err)
			continue
		}

		item.Ack(false)
		log.Println("Acknowledged:", url)
	}
}

func CrawlURL(crawlChannel <-chan *CrawlerMessageItem, crawler *http_crawler.Crawler) <-chan *CrawlerMessageItem {
	extractChannel := make(chan *CrawlerMessageItem, 2)

	crawlLoop := func(
		crawl <-chan *CrawlerMessageItem,
		extract chan<- *CrawlerMessageItem,
		crawler *http_crawler.Crawler,
	) {
		for item := range crawl {
			url := item.URL()
			log.Println("Crawling URL:", url)

			body, err := crawler.Crawl(url)
			if err != nil {
				item.Reject(false)
				log.Println("Couldn't crawl (rejecting):", url, err)
				continue
			}

			item.HTMLBody = body

			if item.IsHTML() {
				extract <- item
			} else {
				item.Ack(false)
			}
		}
	}

	go crawlLoop(crawlChannel, extractChannel, crawler)
	go crawlLoop(crawlChannel, extractChannel, crawler)

	return extractChannel
}

func ExtractURLs(extractChannel <-chan *CrawlerMessageItem) (<-chan string, <-chan *CrawlerMessageItem) {
	publishChannel := make(chan string, 100)
	acknowledgeChannel := make(chan *CrawlerMessageItem, 1)

	extractLoop := func(
		extract <-chan *CrawlerMessageItem,
		publish chan<- string,
		acknowledge chan<- *CrawlerMessageItem,
	) {
		for item := range extract {
			urls, err := item.ExtractURLs()
			if err != nil {
				item.Reject(false)
				log.Println("ExtractURLs (rejecting):", string(item.Body), err)
			}

			log.Println("Extracted URLs:", len(urls))

			for _, url := range urls {
				publish <- url
			}

			acknowledge <- item
		}
	}

	go extractLoop(extractChannel, publishChannel, acknowledgeChannel)

	return publishChannel, acknowledgeChannel
}

func PublishURLs(ttlHashSet *ttl_hash_set.TTLHashSet, queueManager *queue.QueueManager, publish <-chan string) {
	for url := range publish {
		exists, err := ttlHashSet.Exists(url)

		if err != nil {
			log.Println("Couldn't check existence of URL:", url, err)

			if err.Error() == "use of closed network connection" {
				log.Fatalln("No connection to Redis:", err)
			}
		}

		if !exists {
			err = queueManager.Publish("#", "text/plain", url)
			if err != nil {
				log.Fatalln("Delivery failed:", url, err)
			}
		}
	}
}

func ReadFromQueue(inboundChannel <-chan amqp.Delivery, ttlHashSet *ttl_hash_set.TTLHashSet, blacklistPaths []string) chan *CrawlerMessageItem {
	outboundChannel := make(chan *CrawlerMessageItem, 2)

	readLoop := func(
		inbound <-chan amqp.Delivery,
		outbound chan<- *CrawlerMessageItem,
		ttlHashSet *ttl_hash_set.TTLHashSet,
		blacklistPaths []string,
	) {
		for item := range inbound {
			message := NewCrawlerMessageItem(item, "", blacklistPaths)

			exists, err := ttlHashSet.Exists(message.URL())
			if err != nil {
				if err.Error() == "use of closed network connection" {
					log.Fatalln("No connection to Redis:", err)
				} else {
					item.Reject(true)
					log.Println("Couldn't check existence of (rejecting):", message.URL(), err)
					continue
				}
			}

			if !exists {
				outbound <- message
			} else {
				log.Println("URL already crawled:", message.URL())
				item.Ack(false)
			}
		}
	}

	go readLoop(inboundChannel, outboundChannel, ttlHashSet, blacklistPaths)

	return outboundChannel
}
<|endoftext|>