<commit_before>package gxutil

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"

	stump "github.com/whyrusleeping/stump"
)

type ErrAlreadyInstalled struct {
	pkg string
}

func IsErrAlreadyInstalled(err error) bool {
	_, ok := err.(ErrAlreadyInstalled)
	return ok
}

func (eai ErrAlreadyInstalled) Error() string {
	return fmt.Sprintf("package %s already installed", eai.pkg)
}

func (pm *PM) GetPackageTo(hash, out string) (*Package, error) {
	var pkg Package
	_, err := os.Stat(out)
	if err == nil {
		err := FindPackageInDir(&pkg, out)
		if err == nil {
			return &pkg, nil
		}

		stump.VLog("Target directory already exists but isn't a valid package, cleaning up...")
		if oErr := os.RemoveAll(out); oErr != nil {
			stump.Error("cannot purge existing target directory:", oErr)
			return nil, oErr
		}
	}

	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}

	if err := pm.tryFetch(hash, out); err != nil {
		return nil, err
	}

	err = FindPackageInDir(&pkg, out)
	if err != nil {
		return nil, err
	}

	return &pkg, nil
}

func (pm *PM) CacheAndLinkPackage(ref, cacheloc, out string) error {
	if err := pm.tryFetch(ref, cacheloc); err != nil {
		return err
	}

	finfo, err := os.Lstat(out)
	switch {
	case err == nil:
		if finfo.Mode()&os.ModeSymlink != 0 {
			target, err := os.Readlink(out)
			if err != nil {
				return err
			}

			// expecting 'ref' to be '/ipfs/QmFoo/pkg'
			if target != cacheloc {
				panic("not handling dep changes yet")
			}

			// Link already exists
			return nil
		} else {
			// TODO: should we force these to be links?
			// we want to support people just cloning packages into place here, so how should we handle it here?
			panic("not yet handling non-linked packages...")
		}
	case os.IsNotExist(err):
		// ok
	default:
		return err
	}

	if err := os.MkdirAll(filepath.Dir(out), 0755); err != nil {
		return err
	}

	return os.Symlink(cacheloc, out)
}

func (pm *PM) tryFetch(hash, target string) error {
	temp := target + ".part"

	// check if a fetch was previously started and failed, cleanup if found
	_, err := os.Stat(temp)
	if err == nil {
		stump.VLog("Found previously failed fetch, cleaning up...")
		if err := os.RemoveAll(temp); err != nil {
			stump.Error("cleaning up previous aborted transfer: %s", err)
		}
	}

	begin := time.Now()
	stump.VLog("  - fetching %s via ipfs api", hash)
	defer func() {
		stump.VLog("  - fetch finished in %s", time.Since(begin))
	}()
	tries := 3
	for i := 0; i < tries; i++ {
		if err := pm.Shell().Get(hash, temp); err != nil {
			stump.Error("from shell.Get(): %v", err)

			rmerr := os.RemoveAll(temp)
			if rmerr != nil {
				stump.Error("cleaning up temp download directory: %s", rmerr)
			}

			if i == tries-1 {
				return err
			}
			stump.Log("retrying fetch %s after a second...", hash)
			time.Sleep(time.Second)
		} else {
			if err := chmodR(temp, 0444); err != nil {
				return err
			}
			return os.Rename(temp, target)
		}
	}
	panic("unreachable")
}

func chmodR(dir string, perm os.FileMode) error {
	return filepath.Walk(dir, func(p string, info os.FileInfo, err error) error {
		if p == dir {
			return nil
		}

		if err == nil {
			if info.Mode()&os.ModeSymlink != 0 {
				return nil
			}

			perm := perm
			if info.IsDir() {
				perm |= 0111
			}

			return os.Chmod(p, perm)
		}
		return nil
	})
}

func FindPackageInDir(pkg interface{}, dir string) error {
	if err := LoadPackageFile(pkg, filepath.Join(dir, PkgFileName)); err == nil {
		return nil
	}

	name, err := PackageNameInDir(dir)
	if err != nil {
		return err
	}
	return LoadPackageFile(pkg, filepath.Join(dir, name, PkgFileName))
}

func PackageNameInDir(dir string) (string, error) {
	fs, err := ioutil.ReadDir(dir)
	if err != nil {
		return "", err
	}

	if len(fs) == 0 {
		return "", fmt.Errorf("no package found in hashdir: %s", dir)
	}

	if len(fs) > 1 {
		return "", fmt.Errorf("found multiple packages in hashdir: %s", dir)
	}

	return fs[0].Name(), nil
}
<commit_msg>check if ref is already fetched<commit_after>package gxutil

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"

	stump "github.com/whyrusleeping/stump"
)

type ErrAlreadyInstalled struct {
	pkg string
}

func IsErrAlreadyInstalled(err error) bool {
	_, ok := err.(ErrAlreadyInstalled)
	return ok
}

func (eai ErrAlreadyInstalled) Error() string {
	return fmt.Sprintf("package %s already installed", eai.pkg)
}

func (pm *PM) GetPackageTo(hash, out string) (*Package, error) {
	var pkg Package
	_, err := os.Stat(out)
	if err == nil {
		err := FindPackageInDir(&pkg, out)
		if err == nil {
			return &pkg, nil
		}

		stump.VLog("Target directory already exists but isn't a valid package, cleaning up...")
		if oErr := os.RemoveAll(out); oErr != nil {
			stump.Error("cannot purge existing target directory:", oErr)
			return nil, oErr
		}
	}

	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}

	if err := pm.tryFetch(hash, out); err != nil {
		return nil, err
	}

	err = FindPackageInDir(&pkg, out)
	if err != nil {
		return nil, err
	}

	return &pkg, nil
}

func (pm *PM) CacheAndLinkPackage(ref, cacheloc, out string) error {
	if err := pm.tryFetch(ref, cacheloc); err != nil {
		return err
	}

	finfo, err := os.Lstat(out)
	switch {
	case err == nil:
		if finfo.Mode()&os.ModeSymlink != 0 {
			target, err := os.Readlink(out)
			if err != nil {
				return err
			}

			// expecting 'ref' to be '/ipfs/QmFoo/pkg'
			if target != cacheloc {
				panic("not handling dep changes yet")
			}

			// Link already exists
			return nil
		} else {
			// TODO: should we force these to be links?
			// we want to support people just cloning packages into place here, so how should we handle it here?
			panic("not yet handling non-linked packages...")
		}
	case os.IsNotExist(err):
		// ok
	default:
		return err
	}

	if err := os.MkdirAll(filepath.Dir(out), 0755); err != nil {
		return err
	}

	return os.Symlink(cacheloc, out)
}

func (pm *PM) tryFetch(hash, target string) error {
	temp := target + ".part"

	// check if already downloaded
	_, err := os.Stat(target)
	if err == nil {
		stump.VLog("already fetched %s", target)
		return nil
	}

	// check if a fetch was previously started and failed, cleanup if found
	_, err = os.Stat(temp)
	if err == nil {
		stump.VLog("Found previously failed fetch, cleaning up...")
		if err := os.RemoveAll(temp); err != nil {
			stump.Error("cleaning up previous aborted transfer: %s", err)
		}
	}

	begin := time.Now()
	stump.VLog("  - fetching %s via ipfs api", hash)
	defer func() {
		stump.VLog("  - fetch finished in %s", time.Since(begin))
	}()
	tries := 3
	for i := 0; i < tries; i++ {
		if err := pm.Shell().Get(hash, temp); err != nil {
			stump.Error("from shell.Get(): %v", err)

			rmerr := os.RemoveAll(temp)
			if rmerr != nil {
				stump.Error("cleaning up temp download directory: %s", rmerr)
			}

			if i == tries-1 {
				return err
			}
			stump.Log("retrying fetch %s after a second...", hash)
			time.Sleep(time.Second)
		} else {
			if err := chmodR(temp, 0444); err != nil {
				return err
			}
			return os.Rename(temp, target)
		}
	}
	panic("unreachable")
}

func chmodR(dir string, perm os.FileMode) error {
	return filepath.Walk(dir, func(p string, info os.FileInfo, err error) error {
		if p == dir {
			return nil
		}

		if err == nil {
			if info.Mode()&os.ModeSymlink != 0 {
				return nil
			}

			perm := perm
			if info.IsDir() {
				perm |= 0111
			}

			return os.Chmod(p, perm)
		}
		return nil
	})
}

func FindPackageInDir(pkg interface{}, dir string) error {
	if err := LoadPackageFile(pkg, filepath.Join(dir, PkgFileName)); err == nil {
		return nil
	}

	name, err := PackageNameInDir(dir)
	if err != nil {
		return err
	}
	return LoadPackageFile(pkg, filepath.Join(dir, name, PkgFileName))
}

func PackageNameInDir(dir string) (string, error) {
	fs, err := ioutil.ReadDir(dir)
	if err != nil {
		return "", err
	}

	if len(fs) == 0 {
		return "", fmt.Errorf("no package found in hashdir: %s", dir)
	}

	if len(fs) > 1 {
		return "", fmt.Errorf("found multiple packages in hashdir: %s", dir)
	}

	return fs[0].Name(), nil
}
<|endoftext|>


<commit_before>package cover

import (
	"fmt"
	"log"
	"sort"
)

type Meta struct {
	Size uint
	Name string
}

type Node struct {
	Right, Up, Left, Down *Node
	Col                   *Node
	Meta                  *Meta
}

// Initializes a node with neighbours pointing to itself.
func NewNode() *Node {
	n := &Node{}
	n.Left = n
	n.Right = n
	n.Up = n
	n.Down = n
	return n
}

// Initializes a column node as a normal node + meta.
func NewColNode(s string) *Node {
	n := NewNode()
	n.Meta = &Meta{Name: s}
	return n
}

// Appends a node to a row by putting it before the current node.
func (r *Node) RowAppend(n *Node) {
	n.Right = r
	n.Left = r.Left
	// inserts the node at the en of the row
	r.Left.Right = n
	r.Left = n
}

// Appends a node to a col by putting it before the current node.
// Note that the current node has to own meta in order to update the size.
func (c *Node) ColAppend(n *Node) {
	n.Col = c
	n.Down = c
	n.Up = c.Up
	// inserts the node at bottom of the col
	c.Up.Down = n
	c.Up = n
	c.Meta.Size++
}

func (n *Node) String() string {
	return fmt.Sprintf("&{Right:%p Up:%p Left:%p Down:%p Col:%p Meta:%+v}", n.Right, n.Up, n.Left, n.Down, n.Col, n.Meta)
}

func (c *Node) Cover() {
	log.Println("Cover col", c.Meta.Name)
	c.Right.Left = c.Left
	c.Left.Right = c.Right
	for i := c.Down; i != c; i = i.Down {
		for j := i.Right; j != i; j = j.Right {
			j.Down.Up = j.Up
			j.Up.Down = j.Down
			j.Col.Meta.Size--
		}
	}
}

func (c *Node) Uncover() {
	log.Println("Uncover col", c.Meta.Name)
	for i := c.Up; i != c; i = i.Up {
		for j := i.Left; j != i; j = j.Left {
			j.Col.Meta.Size++
			j.Down.Up = j
			j.Up.Down = j
		}
	}
	c.Right.Left = c
	c.Left.Right = c
}

type SparseMatrix struct {
	*Node
}

/*
Given a binary matrix like:
  A  B  C  D  E  F  G
[[0, 0, 1, 0, 1, 1, 0] (3: CEF)
 [1, 0, 0, 1, 0, 0, 1]
 [0, 1, 1, 0, 0, 1, 0]
 [1, 0, 0, 1, 0, 0, 0] (1: AD)
 [0, 1, 0, 0, 0, 0, 1] (2: BG)
 [0, 0, 0, 1, 1, 0, 1]]
return a sparse matrix made of horizontally and vertically
double linked nodes for 1 values.
*/
func NewSparseMatrix(matrix [][]int, headers []string) *SparseMatrix {
	rowCount := len(matrix)
	colCount := len(headers)
	root := &Node{Meta: &Meta{Name: "root"}}
	root.Left = root
	root.Right = root
	// create the columns
	for _, h := range headers {
		head := NewColNode(h)
		root.RowAppend(head)
	}
	for i := 0; i < rowCount; i++ {
		var prev, head *Node
		head = root.Right
		for j := 0; j < colCount; j++ {
			if matrix[i][j] > 0 {
				node := NewNode()
				head.ColAppend(node)
				if prev != nil {
					prev.RowAppend(node)
				} else {
					prev = node
				}
			}
			head = head.Right
		}
	}
	return &SparseMatrix{root}
}

func (m *SparseMatrix) SmallestCol() *Node {
	var r *Node
	min := ^uint(0)
	// we want the underlying node rather than the matrix for comparison
	root := m.Root()
	for col := root.Right; col != root; col = col.Right {
		if col.Meta.Size < min {
			r = col
			min = col.Meta.Size
		}
	}
	return r
}

// Get the root element of the matrix.
func (m *SparseMatrix) Root() *Node {
	return m.Left.Right
}
func (m *SparseMatrix) Col(name string) *Node {
	root := m.Root()
	for col := root.Right; col != root; col = col.Right {
		if col.Meta.Name == name {
			return col
		}
	}
	panic(fmt.Sprintf("Column \"%v\" not found", name))
}

func (m *SparseMatrix) Search(O *Solution, k int, g Guesser) {
	log.Println(k)
	root := m.Root()
	if root.Right == root {
		fmt.Println(O)
		return
	}
	c, bt := g.ChooseCol(k)
	c.Cover()
	for r := c.Down; r != c; r = r.Down {
		O.Set(k, r)
		for j := r.Right; j != r; j = j.Right {
			j.Col.Cover()
		}
		m.Search(O, k+1, g)
		if !bt {
			return
		}
		r = O.Get(k)
		c = r.Col
		for j := r.Left; j != r; j = j.Left {
			j.Col.Uncover()
		}
	}
	c.Uncover()
}

type Solver struct {
	matrix *SparseMatrix
}

func NewSolver(m [][]int, h []string) *Solver {
	s := Solver{matrix: NewSparseMatrix(m, h)}
	return &s
}
func (s *Solver) Solve(partial map[int][]string) *Solution {
	keys := make([]int, 0, len(partial))
	for key, _ := range partial {
		keys = append(keys, key)
	}
	sort.Sort(sort.Reverse(sort.IntSlice(keys)))
	log.Println("Initial config is", partial)
	O := new(Solution)
	k := 0
	m := s.matrix
	for _, digit := range keys {
		for _, c := range partial[digit] {
			n := m.Col(c)
			n.Cover()
			for j := 0; j < digit; j++ {
				n = n.Down
			}
			O.Set(k, n)
			for o := n.Right; o != n; o = o.Right {
				o.Col.Cover()
			}
			k++
		}
	}
	fmt.Printf("Initial solution is\n%v", O)
	s.matrix.Search(O, k, s)
	return O
}

type Guesser interface {
	ChooseCol(int) (*Node, bool)
}

// Given a specific level, returns a node for the next step and a boolean
// telling wether this step should backtracked or not.
func (s *Solver) ChooseCol(k int) (*Node, bool) {
	m := s.matrix
	log.Println("guess is", m.SmallestCol().Meta.Name, "(", m.SmallestCol().Meta.Size, "), bt", true)
	return m.SmallestCol(), true
}

type Solution struct {
	items []*Node
}

func NewSolution() *Solution {
	s := Solution{items: make([]*Node, 0)}
	return &s
}
func (s *Solution) Set(i int, n *Node) {
	if i < len(s.items) {
		s.items[i] = n
	} else {
		s.items = append(s.items, n)
	}
}
func (s *Solution) Get(i int) *Node {
	return s.items[i]
}

func (s *Solution) Len() int {
	return len(s.items)
}

func (s *Solution) String() string {
	o := ""
	for _, n := range s.items {
		if n != nil {
			o += n.Col.Meta.Name
			for m := n.Right; n != m; m = m.Right {
				o += " " + m.Col.Meta.Name
			}
			o += "\n"
		}
	}
	return o
}
<commit_msg>Simplify Solution struct: it just aliases a Node array.<commit_after>package cover

import (
	"fmt"
	"log"
	"sort"
)

type Meta struct {
	Size uint
	Name string
}

type Node struct {
	Right, Up, Left, Down *Node
	Col                   *Node
	Meta                  *Meta
}

// Initializes a node with neighbours pointing to itself.
func NewNode() *Node {
	n := &Node{}
	n.Left = n
	n.Right = n
	n.Up = n
	n.Down = n
	return n
}

// Initializes a column node as a normal node + meta.
func NewColNode(s string) *Node {
	n := NewNode()
	n.Meta = &Meta{Name: s}
	return n
}

// Appends a node to a row by putting it before the current node.
func (r *Node) RowAppend(n *Node) {
	n.Right = r
	n.Left = r.Left
	// inserts the node at the en of the row
	r.Left.Right = n
	r.Left = n
}

// Appends a node to a col by putting it before the current node.
// Note that the current node has to own meta in order to update the size.
func (c *Node) ColAppend(n *Node) {
	n.Col = c
	n.Down = c
	n.Up = c.Up
	// inserts the node at bottom of the col
	c.Up.Down = n
	c.Up = n
	c.Meta.Size++
}

func (n *Node) String() string {
	return fmt.Sprintf("&{Right:%p Up:%p Left:%p Down:%p Col:%p Meta:%+v}", n.Right, n.Up, n.Left, n.Down, n.Col, n.Meta)
}

func (c *Node) Cover() {
	log.Println("Cover col", c.Meta.Name)
	c.Right.Left = c.Left
	c.Left.Right = c.Right
	for i := c.Down; i != c; i = i.Down {
		for j := i.Right; j != i; j = j.Right {
			j.Down.Up = j.Up
			j.Up.Down = j.Down
			j.Col.Meta.Size--
		}
	}
}

func (c *Node) Uncover() {
	log.Println("Uncover col", c.Meta.Name)
	for i := c.Up; i != c; i = i.Up {
		for j := i.Left; j != i; j = j.Left {
			j.Col.Meta.Size++
			j.Down.Up = j
			j.Up.Down = j
		}
	}
	c.Right.Left = c
	c.Left.Right = c
}

type SparseMatrix struct {
	*Node
}

/*
Given a binary matrix like:
  A  B  C  D  E  F  G
[[0, 0, 1, 0, 1, 1, 0] (3: CEF)
 [1, 0, 0, 1, 0, 0, 1]
 [0, 1, 1, 0, 0, 1, 0]
 [1, 0, 0, 1, 0, 0, 0] (1: AD)
 [0, 1, 0, 0, 0, 0, 1] (2: BG)
 [0, 0, 0, 1, 1, 0, 1]]
return a sparse matrix made of horizontally and vertically
double linked nodes for 1 values.
*/
func NewSparseMatrix(matrix [][]int, headers []string) *SparseMatrix {
	rowCount := len(matrix)
	colCount := len(headers)
	root := &Node{Meta: &Meta{Name: "root"}}
	root.Left = root
	root.Right = root
	// create the columns
	for _, h := range headers {
		head := NewColNode(h)
		root.RowAppend(head)
	}
	for i := 0; i < rowCount; i++ {
		var prev, head *Node
		head = root.Right
		for j := 0; j < colCount; j++ {
			if matrix[i][j] > 0 {
				node := NewNode()
				head.ColAppend(node)
				if prev != nil {
					prev.RowAppend(node)
				} else {
					prev = node
				}
			}
			head = head.Right
		}
	}
	return &SparseMatrix{root}
}

func (m *SparseMatrix) SmallestCol() *Node {
	var r *Node
	min := ^uint(0)
	// we want the underlying node rather than the matrix for comparison
	root := m.Root()
	for col := root.Right; col != root; col = col.Right {
		if col.Meta.Size < min {
			r = col
			min = col.Meta.Size
		}
	}
	return r
}

// Get the root element of the matrix.
func (m *SparseMatrix) Root() *Node {
	return m.Left.Right
}
func (m *SparseMatrix) Col(name string) *Node {
	root := m.Root()
	for col := root.Right; col != root; col = col.Right {
		if col.Meta.Name == name {
			return col
		}
	}
	panic(fmt.Sprintf("Column \"%v\" not found", name))
}

func (m *SparseMatrix) Search(O *Solution, k int, g Guesser) {
	log.Println(k)
	root := m.Root()
	if root.Right == root {
		fmt.Println(O)
		return
	}
	c, bt := g.ChooseCol(k)
	c.Cover()
	for r := c.Down; r != c; r = r.Down {
		O.Set(k, r)
		for j := r.Right; j != r; j = j.Right {
			j.Col.Cover()
		}
		m.Search(O, k+1, g)
		if !bt {
			return
		}
		r = O.Get(k)
		c = r.Col
		for j := r.Left; j != r; j = j.Left {
			j.Col.Uncover()
		}
	}
	c.Uncover()
}

type Solver struct {
	matrix *SparseMatrix
}

func NewSolver(m [][]int, h []string) *Solver {
	s := Solver{matrix: NewSparseMatrix(m, h)}
	return &s
}
func (s *Solver) Solve(partial map[int][]string) *Solution {
	keys := make([]int, 0, len(partial))
	for key, _ := range partial {
		keys = append(keys, key)
	}
	sort.Sort(sort.Reverse(sort.IntSlice(keys)))
	log.Println("Initial config is", partial)
	O := new(Solution)
	k := 0
	m := s.matrix
	for _, digit := range keys {
		for _, c := range partial[digit] {
			n := m.Col(c)
			n.Cover()
			for j := 0; j < digit; j++ {
				n = n.Down
			}
			O.Set(k, n)
			for o := n.Right; o != n; o = o.Right {
				o.Col.Cover()
			}
			k++
		}
	}
	fmt.Printf("Initial solution is\n%v", O)
	s.matrix.Search(O, k, s)
	return O
}

type Guesser interface {
	ChooseCol(int) (*Node, bool)
}

// Given a specific level, returns a node for the next step and a boolean
// telling wether this step should backtracked or not.
func (s *Solver) ChooseCol(k int) (*Node, bool) {
	m := s.matrix
	log.Println("guess is", m.SmallestCol().Meta.Name, "(", m.SmallestCol().Meta.Size, "), bt", true)
	return m.SmallestCol(), true
}

type Solution []*Node

func (s *Solution) Set(i int, n *Node) {
	if i < len(*s) {
		(*s)[i] = n
	} else {
		(*s) = append((*s), n)
	}
}
func (s *Solution) Get(i int) *Node {
	return (*s)[i]
}
func (s *Solution) Len() int {
	return len(*s)
}

func (s *Solution) String() string {
	o := ""
	for _, n := range *s {
		if n != nil {
			o += n.Col.Meta.Name
			for m := n.Right; n != m; m = m.Right {
				o += " " + m.Col.Meta.Name
			}
			o += "\n"
		}
	}
	return o
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"time"
)

func doHTTP(url string, input io.Reader) (io.ReadCloser, error) {
	method := "GET"
	if input != nil {
		method = "POST"
	}
	rq, err := http.NewRequest(method, url, input)
	if err != nil {
		return nil, err
	}
	rq.Header.Set("User-Agent", "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)")
	if input != nil {
		rq.Header.Set("Content-Type", "application/json")
	}
	rsp, err := http.DefaultClient.Do(rq)
	if err != nil {
		return nil, err
	}
	if rsp.StatusCode != http.StatusOK {
		rsp.Body.Close()
		return nil, fmt.Errorf("got %s fetching %s", rsp.Status, url)
	}
	return rsp.Body, nil
}

func doJson(url string, input interface{}, output interface{}) error {
	var post io.Reader
	if input != nil {
		body := &bytes.Buffer{}
		err := json.NewEncoder(body).Encode(input)
		if err != nil {
			return err
		}
		post = body
	}
	result, err := doHTTP(url, post)
	if err != nil {
		return err
	}
	defer result.Close()
	return json.NewDecoder(result).Decode(output)
}

type SearchPaging struct {
	Range      int `json:"range"`
	StartIndex int `json:"startIndex"`
}

type SearchSorts struct {
	Direction string `json:"direction"`
	Type      string `json:"type"`
}

type SearchFilters struct {
	EnableFilter    bool          `json:"activeFiltre"`
	Functions       []int         `json:"fonctions"`
	Places          []int         `json:"lieux"`
	Keywords        string        `json:"motsCles"`
	Experience      []int         `json:"niveauxExperience"`
	Paging          SearchPaging  `json:"pagination"`
	MinSalary       int           `json:"salaireMinimum"`
	MaxSalary       int           `json:"salaireMaximum"`
	Sectors         []int         `json:"secteursActivite"`
	Sorts           []SearchSorts `json:"sorts"`
	ClientType      string        `json:"typeClient"`
	ContractTypes   []int         `json:"typesContrat"`
	ConventionTypes []int         `json:"typesConvention"`
}

func searchOffers(start, count int) ([]string, error) {
	filter := &SearchFilters{
		EnableFilter: true,
		Functions:    []int{},
		Places:       []int{ /*705*/ },
		Experience:   []int{},
		Paging: SearchPaging{
			Range:      count,
			StartIndex: start,
		},
		MinSalary: 60,
		MaxSalary: 120,
		Sectors:   []int{},
		Sorts: []SearchSorts{
			{
				Direction: "DESCENDING",
				Type:      "DATE",
			},
		},
		ClientType:      "CADRE",
		ContractTypes:   []int{},
		ConventionTypes: []int{},
	}
	results := &struct {
		Results []struct {
			URI string `json:"@uriOffre"`
		} `json:"resultats"`
	}{}
	url := "https://cadres.apec.fr/cms/webservices/rechercheOffre/ids"
	err := doJson(url, filter, results)
	if err != nil {
		return nil, err
	}
	ids := []string{}
	for _, uri := range results.Results {
		parts := strings.Split(uri.URI, "numeroOffre=")
		if len(parts) != 2 {
			return nil, fmt.Errorf("invalid offer identifier: %s", uri.URI)
		}
		ids = append(ids, parts[1])
	}
	return ids, nil
}

func getOffer(id string) ([]byte, error) {
	u := "https://cadres.apec.fr/cms/webservices/offre/public?numeroOffre=" + id
	output, err := doHTTP(u, nil)
	if err != nil {
		return nil, err
	}
	defer output.Close()
	return ioutil.ReadAll(output)
}

var (
	crawlCmd      = app.Command("crawl", "crawl APEC offers")
	crawlStoreDir = crawlCmd.Arg("store", "data store directory").Required().String()
)

func enumerateOffers(callback func([]string) error) error {
	start := 0
	count := 250
	baseDelay := 5 * time.Second
	maxDelay := 5 * time.Minute
	delay := baseDelay
	for ; ; time.Sleep(delay) {
		time.Sleep(delay)
		fmt.Printf("fetching from %d to %d\n", start, start+count)
		ids, err := searchOffers(start, count)
		if err != nil {
			fmt.Printf("fetching failed with: %s\n", err)
			delay *= 2
			if delay > maxDelay {
				return err
			}
			continue
		}
		delay = baseDelay
		start += count
		err = callback(ids)
		if err != nil {
			return err
		}
		if len(ids) < count {
			break
		}
	}
	return nil
}

func crawlOffers() error {
	store, err := CreateStore(*crawlStoreDir)
	if err != nil {
		return err
	}
	added, deleted := 0, 0
	seen := map[string]bool{}
	err = enumerateOffers(func(ids []string) error {
		for _, id := range ids {
			seen[id] = true
			if store.Has(id) {
				continue
			}
			fmt.Printf("fetching %s\n", id)
			data, err := getOffer(id)
			if err != nil {
				return err
			}
			time.Sleep(time.Second)
			written, err := store.Write(id, data)
			if err != nil {
				return err
			}
			if !written {
				fmt.Printf("racing %s\n", id)
				continue
			}
			added += 1
		}
		return nil
	})
	if err != nil {
		return err
	}
	for _, id := range store.List() {
		if !seen[id] {
			fmt.Printf("deleting %s\n", id)
			store.Delete(id)
			deleted += 1
		}
	}
	fmt.Printf("%d added, %d deleted, %d total\n", added, deleted, store.Size())
	return nil
}
<commit_msg>crawl: add --min-salary option<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"time"
)

func doHTTP(url string, input io.Reader) (io.ReadCloser, error) {
	method := "GET"
	if input != nil {
		method = "POST"
	}
	rq, err := http.NewRequest(method, url, input)
	if err != nil {
		return nil, err
	}
	rq.Header.Set("User-Agent", "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)")
	if input != nil {
		rq.Header.Set("Content-Type", "application/json")
	}
	rsp, err := http.DefaultClient.Do(rq)
	if err != nil {
		return nil, err
	}
	if rsp.StatusCode != http.StatusOK {
		rsp.Body.Close()
		return nil, fmt.Errorf("got %s fetching %s", rsp.Status, url)
	}
	return rsp.Body, nil
}

func doJson(url string, input interface{}, output interface{}) error {
	var post io.Reader
	if input != nil {
		body := &bytes.Buffer{}
		err := json.NewEncoder(body).Encode(input)
		if err != nil {
			return err
		}
		post = body
	}
	result, err := doHTTP(url, post)
	if err != nil {
		return err
	}
	defer result.Close()
	return json.NewDecoder(result).Decode(output)
}

type SearchPaging struct {
	Range      int `json:"range"`
	StartIndex int `json:"startIndex"`
}

type SearchSorts struct {
	Direction string `json:"direction"`
	Type      string `json:"type"`
}

type SearchFilters struct {
	EnableFilter    bool          `json:"activeFiltre"`
	Functions       []int         `json:"fonctions"`
	Places          []int         `json:"lieux"`
	Keywords        string        `json:"motsCles"`
	Experience      []int         `json:"niveauxExperience"`
	Paging          SearchPaging  `json:"pagination"`
	MinSalary       int           `json:"salaireMinimum"`
	MaxSalary       int           `json:"salaireMaximum"`
	Sectors         []int         `json:"secteursActivite"`
	Sorts           []SearchSorts `json:"sorts"`
	ClientType      string        `json:"typeClient"`
	ContractTypes   []int         `json:"typesContrat"`
	ConventionTypes []int         `json:"typesConvention"`
}

func searchOffers(start, count, minSalary int) ([]string, error) {
	filter := &SearchFilters{
		EnableFilter: true,
		Functions:    []int{},
		Places:       []int{ /*705*/ },
		Experience:   []int{},
		Paging: SearchPaging{
			Range:      count,
			StartIndex: start,
		},
		MinSalary: minSalary,
		MaxSalary: 1000,
		Sectors:   []int{},
		Sorts: []SearchSorts{
			{
				Direction: "DESCENDING",
				Type:      "DATE",
			},
		},
		ClientType:      "CADRE",
		ContractTypes:   []int{},
		ConventionTypes: []int{},
	}
	results := &struct {
		Results []struct {
			URI string `json:"@uriOffre"`
		} `json:"resultats"`
	}{}
	url := "https://cadres.apec.fr/cms/webservices/rechercheOffre/ids"
	err := doJson(url, filter, results)
	if err != nil {
		return nil, err
	}
	ids := []string{}
	for _, uri := range results.Results {
		parts := strings.Split(uri.URI, "numeroOffre=")
		if len(parts) != 2 {
			return nil, fmt.Errorf("invalid offer identifier: %s", uri.URI)
		}
		ids = append(ids, parts[1])
	}
	return ids, nil
}

func getOffer(id string) ([]byte, error) {
	u := "https://cadres.apec.fr/cms/webservices/offre/public?numeroOffre=" + id
	output, err := doHTTP(u, nil)
	if err != nil {
		return nil, err
	}
	defer output.Close()
	return ioutil.ReadAll(output)
}

func enumerateOffers(minSalary int, callback func([]string) error) error {
	start := 0
	count := 250
	baseDelay := 5 * time.Second
	maxDelay := 5 * time.Minute
	delay := baseDelay
	for ; ; time.Sleep(delay) {
		time.Sleep(delay)
		fmt.Printf("fetching from %d to %d\n", start, start+count)
		ids, err := searchOffers(start, count, minSalary)
		if err != nil {
			fmt.Printf("fetching failed with: %s\n", err)
			delay *= 2
			if delay > maxDelay {
				return err
			}
			continue
		}
		delay = baseDelay
		start += count
		err = callback(ids)
		if err != nil {
			return err
		}
		if len(ids) < count {
			break
		}
	}
	return nil
}

var (
	crawlCmd       = app.Command("crawl", "crawl APEC offers")
	crawlStoreDir  = crawlCmd.Arg("store", "data store directory").Required().String()
	crawlMinSalary = crawlCmd.Arg("min-salary", "minimum salary in kEUR").Default("50").Int()
)

func crawlOffers() error {
	store, err := CreateStore(*crawlStoreDir)
	if err != nil {
		return err
	}
	added, deleted := 0, 0
	seen := map[string]bool{}
	err = enumerateOffers(*crawlMinSalary, func(ids []string) error {
		for _, id := range ids {
			seen[id] = true
			if store.Has(id) {
				continue
			}
			fmt.Printf("fetching %s\n", id)
			data, err := getOffer(id)
			if err != nil {
				return err
			}
			time.Sleep(time.Second)
			written, err := store.Write(id, data)
			if err != nil {
				return err
			}
			if !written {
				fmt.Printf("racing %s\n", id)
				continue
			}
			added += 1
		}
		return nil
	})
	if err != nil {
		return err
	}
	for _, id := range store.List() {
		if !seen[id] {
			fmt.Printf("deleting %s\n", id)
			store.Delete(id)
			deleted += 1
		}
	}
	fmt.Printf("%d added, %d deleted, %d total\n", added, deleted, store.Size())
	return nil
}
<|endoftext|>


<commit_before>package api

import (
	"fmt"
	"github.com/Aptomi/aptomi/pkg/engine/resolve"
	"github.com/Aptomi/aptomi/pkg/event"
	"github.com/Aptomi/aptomi/pkg/lang"
	"github.com/Aptomi/aptomi/pkg/runtime"
	"github.com/Aptomi/aptomi/pkg/visualization"
	"github.com/julienschmidt/httprouter"
	"github.com/sirupsen/logrus"
	"net/http"
	"strconv"
	"strings"
)

type graphWrapper struct {
	Data interface{}
}

func (g *graphWrapper) GetKind() string {
	return "graph"
}

func (api *coreAPI) handlePolicyDiagram(writer http.ResponseWriter, request *http.Request, params httprouter.Params) {
	mode := params.ByName("mode")
	gen := params.ByName("gen")

	if len(gen) == 0 {
		gen = strconv.Itoa(int(runtime.LastGen))
	}

	policy, _, err := api.store.GetPolicy(runtime.ParseGeneration(gen))
	if err != nil {
		panic(fmt.Sprintf("error while getting requested policy: %s", err))
	}

	var graph *visualization.Graph
	switch strings.ToLower(mode) {
	case "policy":
		// show just policy
		graphBuilder := visualization.NewGraphBuilder(policy, nil, nil)
		graph = graphBuilder.Policy(visualization.PolicyCfgDefault)
	case "desired":
		// show instances in desired state
		// todo: add request id to the event log scope
		resolver := resolve.NewPolicyResolver(policy, api.externalData, event.NewLog(logrus.WarnLevel, "api-policy-diagram"))
		state := resolver.ResolveAllDependencies()
		graphBuilder := visualization.NewGraphBuilder(policy, state, api.externalData)
		graph = graphBuilder.DependencyResolution(visualization.DependencyResolutionCfgDefault)
	case "actual":
		// show instances in actual state
		state, _ := api.store.GetActualState()
		{
			// since we are not storing dependency keys, calculate them on the fly for actual state
			resolver := resolve.NewPolicyResolver(policy, api.externalData, event.NewLog(logrus.WarnLevel, "api-policy-diagram"))
			desiredState := resolver.ResolveAllDependencies()
			state.SetDependencyInstanceMap(desiredState.GetDependencyInstanceMap())
		}

		graphBuilder := visualization.NewGraphBuilder(policy, state, api.externalData)
		graph = graphBuilder.DependencyResolution(visualization.DependencyResolutionCfgDefault)
	default:
		panic("uknown mode: " + mode)
	}

	api.contentType.WriteOne(writer, request, &graphWrapper{Data: graph.GetData()})
}

func (api *coreAPI) handlePolicyDiagramCompare(writer http.ResponseWriter, request *http.Request, params httprouter.Params) {
	mode := params.ByName("mode")
	gen := params.ByName("gen")
	if len(gen) == 0 {
		gen = strconv.Itoa(int(runtime.LastGen))
	}

	genBase := params.ByName("genBase")
	if len(genBase) == 0 {
		genBase = strconv.Itoa(int(runtime.LastGen))
	}

	policy, _, err := api.store.GetPolicy(runtime.ParseGeneration(gen))
	if err != nil {
		panic(fmt.Sprintf("error while getting requested policy: %s", err))
	}
	policyBase, _, err := api.store.GetPolicy(runtime.ParseGeneration(genBase))
	if err != nil {
		panic(fmt.Sprintf("error while getting requested policy: %s", err))
	}

	var graph *visualization.Graph
	switch strings.ToLower(mode) {
	case "policy":
		// show just policy diff
		graph = visualization.NewGraphBuilder(policy, nil, nil).Policy(visualization.PolicyCfgDefault)
		graphBase := visualization.NewGraphBuilder(policyBase, nil, nil).Policy(visualization.PolicyCfgDefault)
		graph.CalcDelta(graphBase)
	case "desired":
		// show instances in desired state (diff)
		// todo: add request id to the event log scope
		resolver := resolve.NewPolicyResolver(policy, api.externalData, event.NewLog(logrus.WarnLevel, "api-policy-diagram"))
		state := resolver.ResolveAllDependencies()
		graphBuilder := visualization.NewGraphBuilder(policy, state, api.externalData)
		graph = graphBuilder.DependencyResolution(visualization.DependencyResolutionCfgDefault)

		// todo: add request id to the event log scope
		resolverBase := resolve.NewPolicyResolver(policyBase, api.externalData, event.NewLog(logrus.WarnLevel, "api-policy-diagram"))
		stateBase := resolverBase.ResolveAllDependencies()
		graphBuilderBase := visualization.NewGraphBuilder(policyBase, stateBase, api.externalData)
		graphBase := graphBuilderBase.DependencyResolution(visualization.DependencyResolutionCfgDefault)

		graph.CalcDelta(graphBase)
	case "actual":
		// show instances in actual state (diff)
		state, _ := api.store.GetActualState()
		{
			// since we are not storing dependency keys, calculate them on the fly for actual state
			resolver := resolve.NewPolicyResolver(policy, api.externalData, event.NewLog(logrus.WarnLevel, "api-policy-diagram"))
			desiredState := resolver.ResolveAllDependencies()
			state.SetDependencyInstanceMap(desiredState.GetDependencyInstanceMap())
		}

		// show instances in desired state (diff)
		graphBuilder := visualization.NewGraphBuilder(policy, state, api.externalData)
		graph = graphBuilder.DependencyResolution(visualization.DependencyResolutionCfgDefault)

		graphBuilderBase := visualization.NewGraphBuilder(policyBase, state, api.externalData)
		graphBase := graphBuilderBase.DependencyResolution(visualization.DependencyResolutionCfgDefault)

		graph.CalcDelta(graphBase)
	default:
		panic("uknown mode: " + mode)
	}

	api.contentType.WriteOne(writer, request, &graphWrapper{Data: graph.GetData()})
}

func (api *coreAPI) handleObjectDiagram(writer http.ResponseWriter, request *http.Request, params httprouter.Params) {
	ns := params.ByName("ns")
	kind := params.ByName("kind")
	name := params.ByName("name")

	policy, _, err := api.store.GetPolicy(runtime.LastGen)
	if err != nil {
		panic(fmt.Sprintf("error while getting policy: %s", err))
	}

	obj, err := policy.GetObject(kind, name, ns)
	if err != nil {
		panic(fmt.Sprintf("error while getting object from policy: %s", err))
	}

	var resolution *resolve.PolicyResolution
	if kind == lang.DependencyObject.Kind {
		resolver := resolve.NewPolicyResolver(policy, api.externalData, event.NewLog(logrus.WarnLevel, "api-object-diagram"))
		resolution = resolver.ResolveAllDependencies()
	}

	var graph *visualization.Graph
	graphBuilder := visualization.NewGraphBuilder(policy, resolution, api.externalData)
	graph = graphBuilder.Object(obj)

	api.contentType.WriteOne(writer, request, &graphWrapper{Data: graph.GetData()})
}
<commit_msg>made all calls to calculate desired state more explicit<commit_after>package api

import (
	"fmt"
	"github.com/Aptomi/aptomi/pkg/engine/resolve"
	"github.com/Aptomi/aptomi/pkg/event"
	"github.com/Aptomi/aptomi/pkg/lang"
	"github.com/Aptomi/aptomi/pkg/runtime"
	"github.com/Aptomi/aptomi/pkg/visualization"
	"github.com/julienschmidt/httprouter"
	"github.com/sirupsen/logrus"
	"net/http"
	"strconv"
	"strings"
)

type graphWrapper struct {
	Data interface{}
}

func (g *graphWrapper) GetKind() string {
	return "graph"
}

func (api *coreAPI) handlePolicyDiagram(writer http.ResponseWriter, request *http.Request, params httprouter.Params) {
	mode := params.ByName("mode")
	gen := params.ByName("gen")

	if len(gen) == 0 {
		gen = strconv.Itoa(int(runtime.LastGen))
	}

	policy, _, err := api.store.GetPolicy(runtime.ParseGeneration(gen))
	if err != nil {
		panic(fmt.Sprintf("error while getting requested policy: %s", err))
	}

	var graph *visualization.Graph
	switch strings.ToLower(mode) {
	case "policy":
		// show just policy
		graphBuilder := visualization.NewGraphBuilder(policy, nil, nil)
		graph = graphBuilder.Policy(visualization.PolicyCfgDefault)
	case "desired":
		// show instances in desired state
		desiredState := resolve.NewPolicyResolver(policy, api.externalData, event.NewLog(logrus.WarnLevel, "api-policy-diagram")).ResolveAllDependencies()
		graphBuilder := visualization.NewGraphBuilder(policy, desiredState, api.externalData)
		graph = graphBuilder.DependencyResolution(visualization.DependencyResolutionCfgDefault)
	case "actual":
		// show instances in actual state
		state, _ := api.store.GetActualState()
		{
			// since we are not storing dependency keys, calculate them on the fly for actual state
			desiredState := resolve.NewPolicyResolver(policy, api.externalData, event.NewLog(logrus.WarnLevel, "api-policy-diagram")).ResolveAllDependencies()
			state.SetDependencyInstanceMap(desiredState.GetDependencyInstanceMap())
		}

		graphBuilder := visualization.NewGraphBuilder(policy, state, api.externalData)
		graph = graphBuilder.DependencyResolution(visualization.DependencyResolutionCfgDefault)
	default:
		panic("unknown mode: " + mode)
	}

	api.contentType.WriteOne(writer, request, &graphWrapper{Data: graph.GetData()})
}

func (api *coreAPI) handlePolicyDiagramCompare(writer http.ResponseWriter, request *http.Request, params httprouter.Params) {
	mode := params.ByName("mode")
	gen := params.ByName("gen")
	if len(gen) == 0 {
		gen = strconv.Itoa(int(runtime.LastGen))
	}

	genBase := params.ByName("genBase")
	if len(genBase) == 0 {
		genBase = strconv.Itoa(int(runtime.LastGen))
	}

	policy, _, err := api.store.GetPolicy(runtime.ParseGeneration(gen))
	if err != nil {
		panic(fmt.Sprintf("error while getting requested policy: %s", err))
	}
	policyBase, _, err := api.store.GetPolicy(runtime.ParseGeneration(genBase))
	if err != nil {
		panic(fmt.Sprintf("error while getting requested policy: %s", err))
	}

	var graph *visualization.Graph
	switch strings.ToLower(mode) {
	case "policy":
		// policy & policy base
		graph = visualization.NewGraphBuilder(policy, nil, nil).Policy(visualization.PolicyCfgDefault)
		graphBase := visualization.NewGraphBuilder(policyBase, nil, nil).Policy(visualization.PolicyCfgDefault)

		// diff
		graph.CalcDelta(graphBase)
	case "desired":
		// desired state (next)
		desiredState := resolve.NewPolicyResolver(policy, api.externalData, event.NewLog(logrus.WarnLevel, "api-policy-diagram")).ResolveAllDependencies()
		graphBuilder := visualization.NewGraphBuilder(policy, desiredState, api.externalData)
		graph = graphBuilder.DependencyResolution(visualization.DependencyResolutionCfgDefault)

		// desired state (prev)
		desiredStateBase := resolve.NewPolicyResolver(policyBase, api.externalData, event.NewLog(logrus.WarnLevel, "api-policy-diagram")).ResolveAllDependencies()
		graphBuilderBase := visualization.NewGraphBuilder(policyBase, desiredStateBase, api.externalData)
		graphBase := graphBuilderBase.DependencyResolution(visualization.DependencyResolutionCfgDefault)

		// diff
		graph.CalcDelta(graphBase)
	case "actual":
		// actual state
		actualState, _ := api.store.GetActualState()
		{
			// since we are not storing dependency keys, calculate them on the fly for actual state
			desiredState := resolve.NewPolicyResolver(policy, api.externalData, event.NewLog(logrus.WarnLevel, "api-policy-diagram")).ResolveAllDependencies()
			actualState.SetDependencyInstanceMap(desiredState.GetDependencyInstanceMap())
		}

		// desired state
		graphBuilder := visualization.NewGraphBuilder(policy, actualState, api.externalData)
		graph = graphBuilder.DependencyResolution(visualization.DependencyResolutionCfgDefault)

		graphBuilderBase := visualization.NewGraphBuilder(policyBase, actualState, api.externalData)
		graphBase := graphBuilderBase.DependencyResolution(visualization.DependencyResolutionCfgDefault)

		// diff
		graph.CalcDelta(graphBase)
	default:
		panic("unknown mode: " + mode)
	}

	api.contentType.WriteOne(writer, request, &graphWrapper{Data: graph.GetData()})
}

func (api *coreAPI) handleObjectDiagram(writer http.ResponseWriter, request *http.Request, params httprouter.Params) {
	ns := params.ByName("ns")
	kind := params.ByName("kind")
	name := params.ByName("name")

	policy, _, err := api.store.GetPolicy(runtime.LastGen)
	if err != nil {
		panic(fmt.Sprintf("error while getting policy: %s", err))
	}

	obj, err := policy.GetObject(kind, name, ns)
	if err != nil {
		panic(fmt.Sprintf("error while getting object from policy: %s", err))
	}

	var desiredState *resolve.PolicyResolution
	if kind == lang.DependencyObject.Kind {
		desiredState = resolve.NewPolicyResolver(policy, api.externalData, event.NewLog(logrus.WarnLevel, "api-object-diagram")).ResolveAllDependencies()
	}

	var graph *visualization.Graph
	graphBuilder := visualization.NewGraphBuilder(policy, desiredState, api.externalData)
	graph = graphBuilder.Object(obj)

	api.contentType.WriteOne(writer, request, &graphWrapper{Data: graph.GetData()})
}
<|endoftext|>


<commit_before>// Brief: Bible Passage handling
// Primary responsibility: Parsing of Bible Passages from HTML

package app

import (
	"fmt"
	"log"
	"strings"

	"golang.org/x/net/html"

	"github.com/julwrites/BotMultiplexer/pkg/def"
	"github.com/julwrites/ScriptureBot/pkg/utils"
)

func GetReference(doc *html.Node) string {
	refNode, err := utils.FindByClass(doc, "bcv")
	if err != nil {
		log.Printf("Error parsing for reference: %v", err)
		return ""
	}

	return refNode.FirstChild.Data
}

func ParseNodesForPassage(node *html.Node) string {
	var text string
	var parts []string

	for child := node.FirstChild; child != nil; child = child.NextSibling {
		parts = append(parts, text)

		switch tag := child.Data; tag {
		case "span":
			childText := strings.Trim(ParseNodesForPassage(child), " ")
			if len(childText) > 0 {
				parts = append(parts, childText)
			}
		case "sup":
			isFootnote := func(node *html.Node) bool {
				for _, attr := range node.Attr {
					if attr.Key == "class" && attr.Val == "footnote" {
						return true
					}
				}
				return false
			}
			if isFootnote(child) {
				break
			}
			childText := strings.Trim(ParseNodesForPassage(child), " ")
			if len(childText) > 0 {
				parts = append(parts, fmt.Sprintf("^%s^", childText))
			}
			break
		case "p":
			return ParseNodesForPassage(child)
		default:
			parts = append(parts, child.Data)
		}
	}

	text = strings.Join(parts, "")

	if node.Data == "h1" || node.Data == "h2" || node.Data == "h3" {
		text = fmt.Sprintf("*%s*", text)
	}
	return text
}

func GetPassage(doc *html.Node, env *def.SessionData) string {
	passageNode, startErr := utils.FindByClass(doc, fmt.Sprintf("version-%s result-text-style-normal text-html", utils.DeserializeUserConfig(env.User.Config).Version))
	if startErr != nil {
		log.Printf("Error parsing for passage: %v", startErr)
		return ""
	}

	filtNodes := utils.FilterChildren(passageNode, func(child *html.Node) bool {
		switch tag := child.Data; tag {
		case "h1":
			fallthrough
		case "h2":
			fallthrough
		case "h3":
			fallthrough
		case "h4":
			fallthrough
		case "p":
			return true
		case "div":
			for _, attr := range child.Attr {
				if attr.Key == "class" && strings.Contains(attr.Val, "poetry") {
					return true
				}
			}
		}
		return false
	})

	textBlocks := utils.MapNodeList(filtNodes, ParseNodesForPassage)

	var passage strings.Builder

	for _, block := range textBlocks {
		passage.WriteString(block)
		passage.WriteString("\n")
	}

	return passage.String()
}

func GetBiblePassage(env def.SessionData) def.SessionData {
	if len(env.Msg.Message) > 0 {

		doc := utils.QueryBiblePassage(env.Msg.Message, utils.DeserializeUserConfig(env.User.Config).Version)

		ref := GetReference(doc)
		log.Printf("Reference retrieved: %s", ref)

		if len(ref) > 0 {
			log.Printf("Getting passage")
			env.Res.Message = GetPassage(doc, &env)
			log.Printf("Passage retrieved %s", env.Res.Message)
		}
	}

	return env
}
<commit_msg>Fixing parsing for break<commit_after>// Brief: Bible Passage handling
// Primary responsibility: Parsing of Bible Passages from HTML

package app

import (
	"fmt"
	"log"
	"strings"

	"golang.org/x/net/html"

	"github.com/julwrites/BotMultiplexer/pkg/def"
	"github.com/julwrites/ScriptureBot/pkg/utils"
)

func GetReference(doc *html.Node) string {
	refNode, err := utils.FindByClass(doc, "bcv")
	if err != nil {
		log.Printf("Error parsing for reference: %v", err)
		return ""
	}

	return refNode.FirstChild.Data
}

func ParseNodesForPassage(node *html.Node) string {
	var text string
	var parts []string

	for child := node.FirstChild; child != nil; child = child.NextSibling {
		parts = append(parts, text)

		switch tag := child.Data; tag {
		case "span":
			childText := strings.Trim(ParseNodesForPassage(child), " ")
			if len(childText) > 0 {
				parts = append(parts, childText)
			}
		case "sup":
			isFootnote := func(node *html.Node) bool {
				for _, attr := range node.Attr {
					if attr.Key == "class" && attr.Val == "footnote" {
						return true
					}
				}
				return false
			}
			if isFootnote(child) {
				break
			}
			childText := strings.Trim(ParseNodesForPassage(child), " ")
			if len(childText) > 0 {
				parts = append(parts, fmt.Sprintf("^%s^", childText))
			}
			break
		case "p":
			parts = append(parts, ParseNodesForPassage(child))
			break
		case "br":
			parts = append(parts, " ")
			break
		default:
			parts = append(parts, child.Data)
		}
	}

	text = strings.Join(parts, "")

	if node.Data == "h1" || node.Data == "h2" || node.Data == "h3" || node.Data == "h4" {
		text = fmt.Sprintf("*%s*", text)
	}
	return text
}

func GetPassage(doc *html.Node, env *def.SessionData) string {
	passageNode, startErr := utils.FindByClass(doc, fmt.Sprintf("version-%s result-text-style-normal text-html", utils.DeserializeUserConfig(env.User.Config).Version))
	if startErr != nil {
		log.Printf("Error parsing for passage: %v", startErr)
		return ""
	}

	filtNodes := utils.FilterChildren(passageNode, func(child *html.Node) bool {
		switch tag := child.Data; tag {
		case "h1":
			fallthrough
		case "h2":
			fallthrough
		case "h3":
			fallthrough
		case "h4":
			fallthrough
		case "p":
			return true
		case "div":
			for _, attr := range child.Attr {
				if attr.Key == "class" && strings.Contains(attr.Val, "poetry") {
					return true
				}
			}
		}
		return false
	})

	textBlocks := utils.MapNodeList(filtNodes, ParseNodesForPassage)

	var passage strings.Builder

	for _, block := range textBlocks {
		passage.WriteString(block)
		passage.WriteString("\n")
	}

	return passage.String()
}

func GetBiblePassage(env def.SessionData) def.SessionData {
	if len(env.Msg.Message) > 0 {

		doc := utils.QueryBiblePassage(env.Msg.Message, utils.DeserializeUserConfig(env.User.Config).Version)

		ref := GetReference(doc)
		log.Printf("Reference retrieved: %s", ref)

		if len(ref) > 0 {
			log.Printf("Getting passage")
			env.Res.Message = GetPassage(doc, &env)
			log.Printf("Passage retrieved %s", env.Res.Message)
		}
	}

	return env
}
<|endoftext|>


<commit_before>/*
 * Copyright 2018 The original author or authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package core

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"
)

const (
	istioNamespace  = "istio-system"
	istioCrds       = "https://storage.googleapis.com/riff-releases/istio/istio-1.0.0-riff-crds.yaml"
	istioRelease    = "https://storage.googleapis.com/riff-releases/istio/istio-1.0.0-riff-main.yaml"
	servingRelease  = "https://storage.googleapis.com/knative-releases/serving/previous/v20180809-6b01d8e/release-no-mon.yaml"
	eventingRelease = "https://storage.googleapis.com/knative-releases/eventing/previous/v20180809-34ab480/release.yaml"
	stubBusRelease  = "https://storage.googleapis.com/knative-releases/eventing/previous/v20180809-34ab480/release-clusterbus-stub.yaml"
)

type SystemInstallOptions struct {
	NodePort bool
	Force    bool
}

type SystemUninstallOptions struct {
	Istio bool
	Force bool
}

var (
	knativeNamespaces = []string{"knative-eventing", "knative-serving", "knative-build"}
	allNameSpaces     = append(knativeNamespaces, istioNamespace)
)

func (kc *kubectlClient) SystemInstall(options SystemInstallOptions) (bool, error) {

	err := ensureNotTerminating(kc, allNameSpaces, "Please try again later.")
	if err != nil {
		return false, err
	}

	istioStatus, err := getNamespaceStatus(kc, istioNamespace)
	if istioStatus == "'NotFound'" {
		fmt.Print("Installing Istio components\n")
		err = applyResources(kc, istioCrds)
		if err != nil {
			return false, err
		}
		time.Sleep(5 * time.Second) // wait for them to get created
		istioYaml, err := loadRelease(istioRelease)
		if err != nil {
			return false, err
		}
		if options.NodePort {
			istioYaml = bytes.Replace(istioYaml, []byte("LoadBalancer"), []byte("NodePort"), -1)
		}
		fmt.Printf("Applying resources defined in: %s\n", istioRelease)
		istioLog, err := kc.kubeCtl.ExecStdin([]string{"apply", "-f", "-"}, &istioYaml)
		if err != nil {
			fmt.Printf("%s\n", istioLog)
			if strings.Contains(istioLog, "forbidden") {
				fmt.Print("It looks like you don't have cluster-admin permissions.\n\n")
			}
			return false, err
		}

		fmt.Print("Istio for riff installed\n\n")
	} else {
		if !options.Force {
			answer, err := confirm("Istio is already installed, do you want to install the Knative components for riff?")
			if err != nil {
				return false, err
			}
			if !answer {
				return false, nil
			}
		}
	}

	err = waitForIstioComponents(kc)
	if err != nil {
		return false, err
	}

	fmt.Print("Installing Knative components\n")

	servingYaml, err := loadRelease(servingRelease)
	if err != nil {
		return false, err
	}
	if options.NodePort {
		servingYaml = bytes.Replace(servingYaml, []byte("LoadBalancer"), []byte("NodePort"), -1)
	}
	fmt.Printf("Applying resources defined in: %s\n", servingRelease)
	servingLog, err := kc.kubeCtl.ExecStdin([]string{"apply", "-f", "-"}, &servingYaml)
	if err != nil {
		fmt.Printf("%s\n", servingLog)
		return false, err
	}

	applyResources(kc, eventingRelease)

	applyResources(kc, stubBusRelease)

	fmt.Print("Knative for riff installed\n\n")
	return true, nil
}

func (kc *kubectlClient) SystemUninstall(options SystemUninstallOptions) (bool, error) {

	err := ensureNotTerminating(kc, allNameSpaces, "This would indicate that the system was already uninstalled.")
	if err != nil {
		return false, err
	}
	knativeNsCount, err := checkNamespacesExists(kc, knativeNamespaces)
	istioNsCount, err := checkNamespacesExists(kc, []string{istioNamespace})
	if err != nil {
		return false, err
	}
	if knativeNsCount == 0 {
		fmt.Print("No Knative components for riff found\n")
	} else {
		if !options.Force {
			answer, err := confirm("Are you sure you want to uninstall the riff system?")
			if err != nil {
				return false, err
			}
			if !answer {
				return false, nil
			}
		}
		fmt.Print("Removing Knative for riff components\n")
		err = deleteCrds(kc, "knative.dev")
		if err != nil {
			return false, err
		}
		err = deleteClusterResources(kc, "clusterrolebinding", "knative-")
		if err != nil {
			return false, err
		}
		err = deleteClusterResources(kc, "clusterrolebinding", "build-controller-")
		if err != nil {
			return false, err
		}
		err = deleteClusterResources(kc, "clusterrolebinding", "eventing-controller-")
		if err != nil {
			return false, err
		}
		err = deleteClusterResources(kc, "clusterrolebinding", "clusterbus-controller-")
		if err != nil {
			return false, err
		}
		err = deleteClusterResources(kc, "clusterrole", "knative-")
		if err != nil {
			return false, err
		}
		err = deleteNamespaces(kc, knativeNamespaces)
		if err != nil {
			return false, err
		}
	}
	if istioNsCount == 0 {
		fmt.Print("No Istio components found\n")
	} else {
		if !options.Istio {
			if options.Force {
				return true, nil
			}
			answer, err := confirm("Do you also want to uninstall Istio components?")
			if err != nil {
				return false, err
			}
			if !answer {
				return false, nil
			}
		}
		fmt.Print("Removing Istio components\n")
		err = deleteCrds(kc, "istio.io")
		if err != nil {
			return false, err
		}
		err = deleteClusterResources(kc, "clusterrolebinding", "istio-")
		if err != nil {
			return false, err
		}
		err = deleteClusterResources(kc, "clusterrole", "istio-")
		if err != nil {
			return false, err
		}
		err = deleteNamespaces(kc, []string{istioNamespace})
		if err != nil {
			return false, err
		}
	}
	return true, nil
}

func resolveReleaseURLs(filename string) (url.URL, error) {
	u, err := url.Parse(filename)
	if err != nil {
		return url.URL{}, err
	}
	if u.Scheme == "http" || u.Scheme == "https" {
		return *u, nil
	}
	return *u, fmt.Errorf("filename must be file, http or https, got %s", u.Scheme)
}

func loadRelease(release string) ([]byte, error) {
	releaseUrl, err := resolveReleaseURLs(release)
	if err != nil {
		return nil, err
	}
	resp, err := http.Get(releaseUrl.String())
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	return body, nil
}

func waitForIstioComponents(kc *kubectlClient) error {
	fmt.Print("Waiting for the Istio components to start ")
	for i := 0; i < 36; i++ {
		fmt.Print(".")
		pods := kc.kubeClient.CoreV1().Pods(istioNamespace)
		podList, err := pods.List(metav1.ListOptions{})
		if err != nil {
			return err
		}
		waitLonger := false
		for _, pod := range podList.Items {
			if !strings.HasPrefix(pod.Name, "istio-") {
				continue
			}
			if pod.Status.Phase != "Running" && pod.Status.Phase != "Succeeded" {
				waitLonger = true
				break
			} else {
				if pod.Status.Phase == "Running" {
					containers := pod.Status.ContainerStatuses
					for _, cont := range containers {
						if !cont.Ready {
							waitLonger = true
							break
						}
					}
				}
			}
		}
		if !waitLonger {
			fmt.Print(" all components are 'Running'\n\n")
			return nil
		}
		time.Sleep(10 * time.Second) // wait for them to start
	}
	return errors.New("the Istio components did not start in time")
}

func applyResources(kc *kubectlClient, release string) error {
	releaseUrl, err := resolveReleaseURLs(release)
	if err != nil {
		return err
	}
	fmt.Printf("Applying resources defined in: %s\n", releaseUrl.String())
	releaseLog, err := kc.kubeCtl.Exec([]string{"apply", "-f", releaseUrl.String()})
	if err != nil {
		fmt.Printf("%s", releaseLog)
	}
	return nil
}

func deleteNamespaces(kc *kubectlClient, namespaces []string) error {
	for _, namespace := range namespaces {
		fmt.Printf("Deleting resources defined in: %s\n", namespace)
		deleteLog, err := kc.kubeCtl.Exec([]string{"delete", "namespace", namespace})
		if err != nil {
			fmt.Printf("%s", deleteLog)
		}
	}
	return nil
}

func deleteClusterResources(kc *kubectlClient, resourceType string, prefix string) error {
	fmt.Printf("Deleting %ss prefixed with %s\n", resourceType, prefix)
	resourceList, err := kc.kubeCtl.Exec([]string{"get", resourceType, "-ocustom-columns=name:metadata.name"})
	if err != nil {
		return err
	}
	resource := strings.Split(string(resourceList), "\n")
	var resourcesToDelete []string
	for _, resource := range resource {
		if strings.HasPrefix(resource, prefix) {
			resourcesToDelete = append(resourcesToDelete, resource)
		}
	}
	if len(resourcesToDelete) > 0 {
		resourceLog, err := kc.kubeCtl.Exec(append([]string{"delete", resourceType}, resourcesToDelete...))
		if err != nil {
			fmt.Printf("%s", resourceLog)
			return err
		}
	}
	return nil
}

func deleteCrds(kc *kubectlClient, suffix string) error {
	fmt.Printf("Deleting CRDs for %s\n", suffix)
	crdList, err := kc.kubeCtl.Exec([]string{"get", "customresourcedefinitions", "-ocustom-columns=name:metadata.name"})
	if err != nil {
		return err
	}
	crds := strings.Split(string(crdList), "\n")
	var crdsToDelete []string
	for _, crd := range crds {
		if strings.HasSuffix(crd, suffix) {
			crdsToDelete = append(crdsToDelete, crd)
		}
	}
	if len(crdsToDelete) > 0 {
		crdLog, err := kc.kubeCtl.Exec(append([]string{"delete", "customresourcedefinition"}, crdsToDelete...))
		if err != nil {
			fmt.Printf("%s", crdLog)
			return err
		}
	}
	return nil
}

func checkNamespacesExists(kc *kubectlClient, names []string) (int, error) {
	count := 0
	for _, name := range names {
		status, err := getNamespaceStatus(kc, name)
		if err != nil {
			return count, err
		}
		if status != "'NotFound'" {
			count = +1
		}
	}
	return count, nil
}

func ensureNotTerminating(kc *kubectlClient, names []string, message string) error {
	for _, name := range names {
		status, err := getNamespaceStatus(kc, name)
		if err != nil {
			return err
		}
		if status == "'Terminating'" {
			return errors.New(fmt.Sprintf("The %s namespace is currently 'Terminating'. %s", name, message))
		}
	}
	return nil
}

func getNamespaceStatus(kc *kubectlClient, name string) (string, error) {
	nsLog, err := kc.kubeCtl.Exec([]string{"get", "namespace", name, "-o", "jsonpath='{.status.phase}'"})
	if err != nil {
		if strings.Contains(nsLog, "NotFound") {
			return "'NotFound'", nil
		}
		return "", err
	}
	return nsLog, nil
}

func confirm(s string) (bool, error) {
	reader := bufio.NewReader(os.Stdin)
	fmt.Printf("%s [y/N]: ", s)
	res, err := reader.ReadString('\n')
	if err != nil {
		return false, err
	}
	if len(res) < 2 {
		return false, nil
	}
	answer := strings.ToLower(strings.TrimSpace(res))[0] == 'y'
	return answer, nil
}
<commit_msg>Provide detailed steps for fixing the permissions problem<commit_after>/*
 * Copyright 2018 The original author or authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package core

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"
)

const (
	istioNamespace  = "istio-system"
	istioCrds       = "https://storage.googleapis.com/riff-releases/istio/istio-1.0.0-riff-crds.yaml"
	istioRelease    = "https://storage.googleapis.com/riff-releases/istio/istio-1.0.0-riff-main.yaml"
	servingRelease  = "https://storage.googleapis.com/knative-releases/serving/previous/v20180809-6b01d8e/release-no-mon.yaml"
	eventingRelease = "https://storage.googleapis.com/knative-releases/eventing/previous/v20180809-34ab480/release.yaml"
	stubBusRelease  = "https://storage.googleapis.com/knative-releases/eventing/previous/v20180809-34ab480/release-clusterbus-stub.yaml"
)

type SystemInstallOptions struct {
	NodePort bool
	Force    bool
}

type SystemUninstallOptions struct {
	Istio bool
	Force bool
}

var (
	knativeNamespaces = []string{"knative-eventing", "knative-serving", "knative-build"}
	allNameSpaces     = append(knativeNamespaces, istioNamespace)
)

func (kc *kubectlClient) SystemInstall(options SystemInstallOptions) (bool, error) {

	err := ensureNotTerminating(kc, allNameSpaces, "Please try again later.")
	if err != nil {
		return false, err
	}

	istioStatus, err := getNamespaceStatus(kc, istioNamespace)
	if istioStatus == "'NotFound'" {
		fmt.Print("Installing Istio components\n")
		err = applyResources(kc, istioCrds)
		if err != nil {
			return false, err
		}
		time.Sleep(5 * time.Second) // wait for them to get created
		istioYaml, err := loadRelease(istioRelease)
		if err != nil {
			return false, err
		}
		if options.NodePort {
			istioYaml = bytes.Replace(istioYaml, []byte("LoadBalancer"), []byte("NodePort"), -1)
		}
		fmt.Printf("Applying resources defined in: %s\n", istioRelease)
		istioLog, err := kc.kubeCtl.ExecStdin([]string{"apply", "-f", "-"}, &istioYaml)
		if err != nil {
			fmt.Printf("%s\n", istioLog)
			if strings.Contains(istioLog, "forbidden") {
				fmt.Print(`It looks like you don't have cluster-admin permissions.

To fix this you need to:
 1. Delete he current failed installation using:
      riff system uninstall --istio --force
 2. Give the user account used for installation cluster-admin permissions, you can use the following command:
      kubectl create clusterrolebinding cluster-admin-binding \
        --clusterrole=cluster-admin \
        --user=<install-user>
 3. Re-install riff

`)
			}
			return false, err
		}

		fmt.Print("Istio for riff installed\n\n")
	} else {
		if !options.Force {
			answer, err := confirm("Istio is already installed, do you want to install the Knative components for riff?")
			if err != nil {
				return false, err
			}
			if !answer {
				return false, nil
			}
		}
	}

	err = waitForIstioComponents(kc)
	if err != nil {
		return false, err
	}

	fmt.Print("Installing Knative components\n")

	servingYaml, err := loadRelease(servingRelease)
	if err != nil {
		return false, err
	}
	if options.NodePort {
		servingYaml = bytes.Replace(servingYaml, []byte("LoadBalancer"), []byte("NodePort"), -1)
	}
	fmt.Printf("Applying resources defined in: %s\n", servingRelease)
	servingLog, err := kc.kubeCtl.ExecStdin([]string{"apply", "-f", "-"}, &servingYaml)
	if err != nil {
		fmt.Printf("%s\n", servingLog)
		return false, err
	}

	applyResources(kc, eventingRelease)

	applyResources(kc, stubBusRelease)

	fmt.Print("Knative for riff installed\n\n")
	return true, nil
}

func (kc *kubectlClient) SystemUninstall(options SystemUninstallOptions) (bool, error) {

	err := ensureNotTerminating(kc, allNameSpaces, "This would indicate that the system was already uninstalled.")
	if err != nil {
		return false, err
	}
	knativeNsCount, err := checkNamespacesExists(kc, knativeNamespaces)
	istioNsCount, err := checkNamespacesExists(kc, []string{istioNamespace})
	if err != nil {
		return false, err
	}
	if knativeNsCount == 0 {
		fmt.Print("No Knative components for riff found\n")
	} else {
		if !options.Force {
			answer, err := confirm("Are you sure you want to uninstall the riff system?")
			if err != nil {
				return false, err
			}
			if !answer {
				return false, nil
			}
		}
		fmt.Print("Removing Knative for riff components\n")
		err = deleteCrds(kc, "knative.dev")
		if err != nil {
			return false, err
		}
		err = deleteClusterResources(kc, "clusterrolebinding", "knative-")
		if err != nil {
			return false, err
		}
		err = deleteClusterResources(kc, "clusterrolebinding", "build-controller-")
		if err != nil {
			return false, err
		}
		err = deleteClusterResources(kc, "clusterrolebinding", "eventing-controller-")
		if err != nil {
			return false, err
		}
		err = deleteClusterResources(kc, "clusterrolebinding", "clusterbus-controller-")
		if err != nil {
			return false, err
		}
		err = deleteClusterResources(kc, "clusterrole", "knative-")
		if err != nil {
			return false, err
		}
		err = deleteNamespaces(kc, knativeNamespaces)
		if err != nil {
			return false, err
		}
	}
	if istioNsCount == 0 {
		fmt.Print("No Istio components found\n")
	} else {
		if !options.Istio {
			if options.Force {
				return true, nil
			}
			answer, err := confirm("Do you also want to uninstall Istio components?")
			if err != nil {
				return false, err
			}
			if !answer {
				return false, nil
			}
		}
		fmt.Print("Removing Istio components\n")
		err = deleteCrds(kc, "istio.io")
		if err != nil {
			return false, err
		}
		err = deleteClusterResources(kc, "clusterrolebinding", "istio-")
		if err != nil {
			return false, err
		}
		err = deleteClusterResources(kc, "clusterrole", "istio-")
		if err != nil {
			return false, err
		}
		err = deleteNamespaces(kc, []string{istioNamespace})
		if err != nil {
			return false, err
		}
	}
	return true, nil
}

func resolveReleaseURLs(filename string) (url.URL, error) {
	u, err := url.Parse(filename)
	if err != nil {
		return url.URL{}, err
	}
	if u.Scheme == "http" || u.Scheme == "https" {
		return *u, nil
	}
	return *u, fmt.Errorf("filename must be file, http or https, got %s", u.Scheme)
}

func loadRelease(release string) ([]byte, error) {
	releaseUrl, err := resolveReleaseURLs(release)
	if err != nil {
		return nil, err
	}
	resp, err := http.Get(releaseUrl.String())
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	return body, nil
}

func waitForIstioComponents(kc *kubectlClient) error {
	fmt.Print("Waiting for the Istio components to start ")
	for i := 0; i < 36; i++ {
		fmt.Print(".")
		pods := kc.kubeClient.CoreV1().Pods(istioNamespace)
		podList, err := pods.List(metav1.ListOptions{})
		if err != nil {
			return err
		}
		waitLonger := false
		for _, pod := range podList.Items {
			if !strings.HasPrefix(pod.Name, "istio-") {
				continue
			}
			if pod.Status.Phase != "Running" && pod.Status.Phase != "Succeeded" {
				waitLonger = true
				break
			} else {
				if pod.Status.Phase == "Running" {
					containers := pod.Status.ContainerStatuses
					for _, cont := range containers {
						if !cont.Ready {
							waitLonger = true
							break
						}
					}
				}
			}
		}
		if !waitLonger {
			fmt.Print(" all components are 'Running'\n\n")
			return nil
		}
		time.Sleep(10 * time.Second) // wait for them to start
	}
	return errors.New("the Istio components did not start in time")
}

func applyResources(kc *kubectlClient, release string) error {
	releaseUrl, err := resolveReleaseURLs(release)
	if err != nil {
		return err
	}
	fmt.Printf("Applying resources defined in: %s\n", releaseUrl.String())
	releaseLog, err := kc.kubeCtl.Exec([]string{"apply", "-f", releaseUrl.String()})
	if err != nil {
		fmt.Printf("%s", releaseLog)
	}
	return nil
}

func deleteNamespaces(kc *kubectlClient, namespaces []string) error {
	for _, namespace := range namespaces {
		fmt.Printf("Deleting resources defined in: %s\n", namespace)
		deleteLog, err := kc.kubeCtl.Exec([]string{"delete", "namespace", namespace})
		if err != nil {
			fmt.Printf("%s", deleteLog)
		}
	}
	return nil
}

func deleteClusterResources(kc *kubectlClient, resourceType string, prefix string) error {
	fmt.Printf("Deleting %ss prefixed with %s\n", resourceType, prefix)
	resourceList, err := kc.kubeCtl.Exec([]string{"get", resourceType, "-ocustom-columns=name:metadata.name"})
	if err != nil {
		return err
	}
	resource := strings.Split(string(resourceList), "\n")
	var resourcesToDelete []string
	for _, resource := range resource {
		if strings.HasPrefix(resource, prefix) {
			resourcesToDelete = append(resourcesToDelete, resource)
		}
	}
	if len(resourcesToDelete) > 0 {
		resourceLog, err := kc.kubeCtl.Exec(append([]string{"delete", resourceType}, resourcesToDelete...))
		if err != nil {
			fmt.Printf("%s", resourceLog)
			return err
		}
	}
	return nil
}

func deleteCrds(kc *kubectlClient, suffix string) error {
	fmt.Printf("Deleting CRDs for %s\n", suffix)
	crdList, err := kc.kubeCtl.Exec([]string{"get", "customresourcedefinitions", "-ocustom-columns=name:metadata.name"})
	if err != nil {
		return err
	}
	crds := strings.Split(string(crdList), "\n")
	var crdsToDelete []string
	for _, crd := range crds {
		if strings.HasSuffix(crd, suffix) {
			crdsToDelete = append(crdsToDelete, crd)
		}
	}
	if len(crdsToDelete) > 0 {
		crdLog, err := kc.kubeCtl.Exec(append([]string{"delete", "customresourcedefinition"}, crdsToDelete...))
		if err != nil {
			fmt.Printf("%s", crdLog)
			return err
		}
	}
	return nil
}

func checkNamespacesExists(kc *kubectlClient, names []string) (int, error) {
	count := 0
	for _, name := range names {
		status, err := getNamespaceStatus(kc, name)
		if err != nil {
			return count, err
		}
		if status != "'NotFound'" {
			count = +1
		}
	}
	return count, nil
}

func ensureNotTerminating(kc *kubectlClient, names []string, message string) error {
	for _, name := range names {
		status, err := getNamespaceStatus(kc, name)
		if err != nil {
			return err
		}
		if status == "'Terminating'" {
			return errors.New(fmt.Sprintf("The %s namespace is currently 'Terminating'. %s", name, message))
		}
	}
	return nil
}

func getNamespaceStatus(kc *kubectlClient, name string) (string, error) {
	nsLog, err := kc.kubeCtl.Exec([]string{"get", "namespace", name, "-o", "jsonpath='{.status.phase}'"})
	if err != nil {
		if strings.Contains(nsLog, "NotFound") {
			return "'NotFound'", nil
		}
		return "", err
	}
	return nsLog, nil
}

func confirm(s string) (bool, error) {
	reader := bufio.NewReader(os.Stdin)
	fmt.Printf("%s [y/N]: ", s)
	res, err := reader.ReadString('\n')
	if err != nil {
		return false, err
	}
	if len(res) < 2 {
		return false, nil
	}
	answer := strings.ToLower(strings.TrimSpace(res))[0] == 'y'
	return answer, nil
}
<|endoftext|>


<commit_before>package jobs

import (
	"context"
	"errors"
	"fmt"
	"math/rand"
	"runtime"
	"runtime/debug"
	"sync/atomic"
	"time"

	"github.com/cozy/cozy-stack/pkg/logger"
	"github.com/cozy/cozy-stack/pkg/metrics"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/sirupsen/logrus"
)

var (
	defaultConcurrency  = runtime.NumCPU()
	defaultMaxExecCount = 3
	defaultRetryDelay   = 60 * time.Millisecond
	defaultTimeout      = 10 * time.Second
)

type (
	// WorkerInitFunc is called at the start of the worker system, only once. It
	// is not called before every job process. It can be useful to initialize a
	// global variable used by the worker.
	WorkerInitFunc func() error

	// WorkerStartFunc is optionally called at the beginning of the each job
	// process and can produce a context value.
	WorkerStartFunc func(ctx *WorkerContext) (*WorkerContext, error)

	// WorkerFunc represent the work function that a worker should implement.
	WorkerFunc func(ctx *WorkerContext) error

	// WorkerCommit is an optional method that is always called once after the
	// execution of the WorkerFunc.
	WorkerCommit func(ctx *WorkerContext, errjob error) error

	// WorkerConfig is the configuration parameter of a worker defined by the job
	// system. It contains parameters of the worker along with the worker main
	// function that perform the work against a job's message.
	WorkerConfig struct {
		WorkerInit   WorkerInitFunc
		WorkerStart  WorkerStartFunc
		WorkerFunc   WorkerFunc
		WorkerCommit WorkerCommit
		WorkerType   string
		Concurrency  int
		MaxExecCount int
		Timeout      time.Duration
		RetryDelay   time.Duration
	}

	// Worker is a unit of work that will consume from a queue and execute the do
	// method for each jobs it pulls.
	Worker struct {
		Type    string
		Conf    *WorkerConfig
		jobs    chan *Job
		running uint32
		closed  chan struct{}
	}

	// WorkerContext is a context.Context passed to the worker for each job
	// execution and contains specific values from the job.
	WorkerContext struct {
		context.Context
		job     *Job
		log     *logrus.Entry
		id      string
		cookie  interface{}
		noRetry bool
	}
)

var slots chan struct{}

func setNbSlots(nb int) {
	slots = make(chan struct{}, nb)
	for i := 0; i < nb; i++ {
		slots <- struct{}{}
	}
}

// Clone clones the worker config
func (w *WorkerConfig) Clone() *WorkerConfig {
	cloned := *w
	return &cloned
}

// NewWorkerContext returns a context.Context usable by a worker.
func NewWorkerContext(workerID string, job *Job) *WorkerContext {
	ctx := context.Background()
	id := fmt.Sprintf("%s/%s", workerID, job.ID())
	log := logger.WithDomain(job.Domain).WithField("job_id", job.ID()).WithField("worker_id", workerID)
	return &WorkerContext{
		Context: ctx,
		job:     job,
		log:     log,
		id:      id,
	}
}

// WithTimeout returns a clone of the context with a different deadline.
func (c *WorkerContext) WithTimeout(timeout time.Duration) (*WorkerContext, context.CancelFunc) {
	ctx, cancel := context.WithTimeout(c.Context, timeout)
	newCtx := c.clone()
	newCtx.Context = ctx
	return newCtx, cancel
}

// WithCookie returns a clone of the context with a new cookie value.
func (c *WorkerContext) WithCookie(cookie interface{}) *WorkerContext {
	newCtx := c.clone()
	newCtx.cookie = cookie
	return newCtx
}

// SetNoRetry set the no-retry flag to prevent a retry on the next execution.
func (c *WorkerContext) SetNoRetry() {
	c.noRetry = true
}

// NoRetry returns the no-retry flag.
func (c *WorkerContext) NoRetry() bool {
	return c.noRetry
}

func (c *WorkerContext) clone() *WorkerContext {
	return &WorkerContext{
		Context: c.Context,
		job:     c.job,
		log:     c.log,
		id:      c.id,
		cookie:  c.cookie,
	}
}

// ID returns a unique identifier for the worker context.
func (c *WorkerContext) ID() string {
	return c.id
}

// Logger return the logger associated with the worker context.
func (c *WorkerContext) Logger() *logrus.Entry {
	return c.log
}

// UnmarshalMessage unmarshals the message contained in the worker context.
func (c *WorkerContext) UnmarshalMessage(v interface{}) error {
	return c.job.Message.Unmarshal(v)
}

// UnmarshalEvent unmarshals the event contained in the worker context.
func (c *WorkerContext) UnmarshalEvent(v interface{}) error {
	if c.job == nil || c.job.Event == nil {
		return errors.New("jobs: does not have an event associated")
	}
	return c.job.Event.Unmarshal(v)
}

// Domain returns the domain associated with the worker context.
func (c *WorkerContext) Domain() string {
	return c.job.Domain
}

// TriggerID returns the possible trigger identifier responsible for launching
// the job.
func (c *WorkerContext) TriggerID() (string, bool) {
	triggerID := c.job.TriggerID
	return triggerID, triggerID != ""
}

// Cookie returns the cookie associated with the worker context.
func (c *WorkerContext) Cookie() interface{} {
	return c.cookie
}

// NewWorker creates a new instance of Worker with the given configuration.
func NewWorker(conf *WorkerConfig) *Worker {
	return &Worker{
		Type: conf.WorkerType,
		Conf: conf,
	}
}

// Manual returns if the job was started manually
func (c *WorkerContext) Manual() bool {
	return c.job.Manual
}

// Start is used to start the worker consumption of messages from its queue.
func (w *Worker) Start(jobs chan *Job) error {
	if !atomic.CompareAndSwapUint32(&w.running, 0, 1) {
		return ErrClosed
	}
	w.jobs = jobs
	w.closed = make(chan struct{})
	if w.Conf.WorkerInit != nil {
		if err := w.Conf.WorkerInit(); err != nil {
			return fmt.Errorf("Could not start worker %s: %s", w.Type, err)
		}
	}
	for i := 0; i < w.Conf.Concurrency; i++ {
		name := fmt.Sprintf("%s/%d", w.Type, i)
		joblog.Debugf("Start worker %s", name)
		go w.work(name, w.closed)
	}
	return nil
}

// Shutdown is used to close the worker, waiting for all tasks to end
func (w *Worker) Shutdown(ctx context.Context) error {
	if !atomic.CompareAndSwapUint32(&w.running, 1, 0) {
		return ErrClosed
	}
	close(w.jobs)
	for i := 0; i < w.Conf.Concurrency; i++ {
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-w.closed:
		}
	}
	return nil
}

func (w *Worker) work(workerID string, closed chan<- struct{}) {
	for job := range w.jobs {
		domain := job.Domain
		if domain == "" {
			joblog.Errorf("[job] %s: missing domain from job request", workerID)
			continue
		}
		parentCtx := NewWorkerContext(workerID, job)
		if err := job.AckConsumed(); err != nil {
			parentCtx.Logger().Errorf("[job] error acking consume job: %s",
				err.Error())
			continue
		}
		t := &task{
			w:    w,
			ctx:  parentCtx,
			job:  job,
			conf: w.defaultedConf(job.Options),
		}
		var runResultLabel string
		var errAck error
		errRun := t.run()
		if errRun == ErrAbort {
			errRun = nil
		}
		if errRun != nil {
			parentCtx.Logger().Errorf("[job] error while performing job: %s",
				errRun.Error())
			runResultLabel = metrics.WorkerExecResultErrored
			errAck = job.Nack(errRun)
		} else {
			runResultLabel = metrics.WorkerExecResultSuccess
			errAck = job.Ack()
		}
		metrics.WorkerExecCounter.WithLabelValues(w.Type, runResultLabel).Inc()
		if errAck != nil {
			parentCtx.Logger().Errorf("[job] error while acking job done: %s",
				errAck.Error())
		}

		// Delete the trigger associated with the job (if any) when we receive a
		// ErrBadTrigger.
		//
		// XXX: better retro-action between broker and scheduler to avoid going
		// through the global job-system.
		if job.TriggerID != "" && globalJobSystem != nil {
			if _, ok := errRun.(ErrBadTrigger); ok {
				globalJobSystem.DeleteTrigger(job.Domain, job.TriggerID)
			}
		}
	}
	joblog.Debugf("[job] %s: worker shut down", workerID)
	closed <- struct{}{}
}

func (w *Worker) defaultedConf(opts *JobOptions) *WorkerConfig {
	c := w.Conf.Clone()
	if c.Concurrency == 0 {
		c.Concurrency = defaultConcurrency
	}
	if c.MaxExecCount == 0 {
		c.MaxExecCount = defaultMaxExecCount
	}
	if c.RetryDelay == 0 {
		c.RetryDelay = defaultRetryDelay
	}
	if c.Timeout == 0 {
		c.Timeout = defaultTimeout
	}
	if opts == nil {
		return c
	}
	if opts.MaxExecCount != 0 && opts.MaxExecCount < c.MaxExecCount {
		c.MaxExecCount = opts.MaxExecCount
	}
	if opts.Timeout > 0 && opts.Timeout < c.Timeout {
		c.Timeout = opts.Timeout
	}
	return c
}

type task struct {
	w    *Worker
	ctx  *WorkerContext
	conf *WorkerConfig
	job  *Job

	startTime time.Time
	execCount int
}

func (t *task) run() (err error) {
	t.startTime = time.Now()
	t.execCount = 0
	if t.conf.WorkerStart != nil {
		t.ctx, err = t.conf.WorkerStart(t.ctx)
		if err != nil {
			return err
		}
	}
	defer func() {
		if t.conf.WorkerCommit != nil {
			if errc := t.conf.WorkerCommit(t.ctx, err); errc != nil {
				t.ctx.Logger().Warnf("[job] error while commiting job: %s",
					errc.Error())
			}
		}
	}()
	for {
		retry, delay, timeout := t.nextDelay(err)
		if !retry {
			return err
		}
		if err != nil {
			t.ctx.Logger().Warnf("[job] error while performing job: %s (retry in %s)",
				err.Error(), delay)
		}

		if delay > 0 {
			time.Sleep(delay)
		}

		t.ctx.Logger().Debugf("[job] executing job (%d) (timeout set to %s)",
			t.execCount, timeout)

		var execResultLabel string
		timer := prometheus.NewTimer(prometheus.ObserverFunc(func(v float64) {
			metrics.WorkerExecDurations.WithLabelValues(t.w.Type, execResultLabel).Observe(v)
		}))

		ctx, cancel := t.ctx.WithTimeout(timeout)
		err = t.exec(ctx)
		if err == nil {
			execResultLabel = metrics.WorkerExecResultSuccess
			timer.ObserveDuration()
			cancel()
			break
		}
		execResultLabel = metrics.WorkerExecResultErrored
		timer.ObserveDuration()

		// Even though ctx should have expired already, it is good practice to call
		// its cancelation function in any case. Failure to do so may keep the
		// context and its parent alive longer than necessary.
		cancel()
		t.execCount++

		if ctx.NoRetry() {
			break
		}
	}
	metrics.WorkerExecRetries.WithLabelValues(t.w.Type).Observe(float64(t.execCount))
	return nil
}

func (t *task) exec(ctx *WorkerContext) (err error) {
	var slot struct{}
	if slots != nil {
		slot = <-slots
	}
	defer func() {
		if slots != nil {
			slots <- slot
		}
		if r := recover(); r != nil {
			var ok bool
			err, ok = r.(error)
			if !ok {
				err = fmt.Errorf("%v", r)
			}
			ctx.Logger().Errorf("[panic] %s: %s", r, debug.Stack())
		}
	}()
	return t.conf.WorkerFunc(ctx)
}

func (t *task) nextDelay(prevError error) (bool, time.Duration, time.Duration) {
	// for certain kinds of errors, we do not have a retry since these error
	// cannot be recovered from
	{
		if _, ok := prevError.(ErrBadTrigger); ok {
			return false, 0, 0
		}
		switch prevError {
		case ErrAbort, ErrMessageUnmarshal, ErrMessageNil:
			return false, 0, 0
		}
	}

	c := t.conf

	if t.execCount >= c.MaxExecCount {
		return false, 0, 0
	}

	// the worker timeout should take into account the maximum execution time
	// allowed to the task
	timeout := c.Timeout

	var nextDelay time.Duration
	if t.execCount == 0 {
		// on first execution, execute immediately
		nextDelay = 0
	} else {
		nextDelay = c.RetryDelay << uint(t.execCount-1)

		// fuzzDelay number between delay * (1 +/- 0.1)
		fuzzDelay := int(0.1 * float64(nextDelay))
		nextDelay = nextDelay + time.Duration((rand.Intn(2*fuzzDelay) - fuzzDelay))
	}

	return true, nextDelay, timeout
}
<commit_msg>Jobs: return error at end of work, even for non-retried<commit_after>package jobs

import (
	"context"
	"errors"
	"fmt"
	"math/rand"
	"runtime"
	"runtime/debug"
	"sync/atomic"
	"time"

	"github.com/cozy/cozy-stack/pkg/logger"
	"github.com/cozy/cozy-stack/pkg/metrics"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/sirupsen/logrus"
)

var (
	defaultConcurrency  = runtime.NumCPU()
	defaultMaxExecCount = 3
	defaultRetryDelay   = 60 * time.Millisecond
	defaultTimeout      = 10 * time.Second
)

type (
	// WorkerInitFunc is called at the start of the worker system, only once. It
	// is not called before every job process. It can be useful to initialize a
	// global variable used by the worker.
	WorkerInitFunc func() error

	// WorkerStartFunc is optionally called at the beginning of the each job
	// process and can produce a context value.
	WorkerStartFunc func(ctx *WorkerContext) (*WorkerContext, error)

	// WorkerFunc represent the work function that a worker should implement.
	WorkerFunc func(ctx *WorkerContext) error

	// WorkerCommit is an optional method that is always called once after the
	// execution of the WorkerFunc.
	WorkerCommit func(ctx *WorkerContext, errjob error) error

	// WorkerConfig is the configuration parameter of a worker defined by the job
	// system. It contains parameters of the worker along with the worker main
	// function that perform the work against a job's message.
	WorkerConfig struct {
		WorkerInit   WorkerInitFunc
		WorkerStart  WorkerStartFunc
		WorkerFunc   WorkerFunc
		WorkerCommit WorkerCommit
		WorkerType   string
		Concurrency  int
		MaxExecCount int
		Timeout      time.Duration
		RetryDelay   time.Duration
	}

	// Worker is a unit of work that will consume from a queue and execute the do
	// method for each jobs it pulls.
	Worker struct {
		Type    string
		Conf    *WorkerConfig
		jobs    chan *Job
		running uint32
		closed  chan struct{}
	}

	// WorkerContext is a context.Context passed to the worker for each job
	// execution and contains specific values from the job.
	WorkerContext struct {
		context.Context
		job     *Job
		log     *logrus.Entry
		id      string
		cookie  interface{}
		noRetry bool
	}
)

var slots chan struct{}

func setNbSlots(nb int) {
	slots = make(chan struct{}, nb)
	for i := 0; i < nb; i++ {
		slots <- struct{}{}
	}
}

// Clone clones the worker config
func (w *WorkerConfig) Clone() *WorkerConfig {
	cloned := *w
	return &cloned
}

// NewWorkerContext returns a context.Context usable by a worker.
func NewWorkerContext(workerID string, job *Job) *WorkerContext {
	ctx := context.Background()
	id := fmt.Sprintf("%s/%s", workerID, job.ID())
	log := logger.WithDomain(job.Domain).WithField("job_id", job.ID()).WithField("worker_id", workerID)
	return &WorkerContext{
		Context: ctx,
		job:     job,
		log:     log,
		id:      id,
	}
}

// WithTimeout returns a clone of the context with a different deadline.
func (c *WorkerContext) WithTimeout(timeout time.Duration) (*WorkerContext, context.CancelFunc) {
	ctx, cancel := context.WithTimeout(c.Context, timeout)
	newCtx := c.clone()
	newCtx.Context = ctx
	return newCtx, cancel
}

// WithCookie returns a clone of the context with a new cookie value.
func (c *WorkerContext) WithCookie(cookie interface{}) *WorkerContext {
	newCtx := c.clone()
	newCtx.cookie = cookie
	return newCtx
}

// SetNoRetry set the no-retry flag to prevent a retry on the next execution.
func (c *WorkerContext) SetNoRetry() {
	c.noRetry = true
}

// NoRetry returns the no-retry flag.
func (c *WorkerContext) NoRetry() bool {
	return c.noRetry
}

func (c *WorkerContext) clone() *WorkerContext {
	return &WorkerContext{
		Context: c.Context,
		job:     c.job,
		log:     c.log,
		id:      c.id,
		cookie:  c.cookie,
	}
}

// ID returns a unique identifier for the worker context.
func (c *WorkerContext) ID() string {
	return c.id
}

// Logger return the logger associated with the worker context.
func (c *WorkerContext) Logger() *logrus.Entry {
	return c.log
}

// UnmarshalMessage unmarshals the message contained in the worker context.
func (c *WorkerContext) UnmarshalMessage(v interface{}) error {
	return c.job.Message.Unmarshal(v)
}

// UnmarshalEvent unmarshals the event contained in the worker context.
func (c *WorkerContext) UnmarshalEvent(v interface{}) error {
	if c.job == nil || c.job.Event == nil {
		return errors.New("jobs: does not have an event associated")
	}
	return c.job.Event.Unmarshal(v)
}

// Domain returns the domain associated with the worker context.
func (c *WorkerContext) Domain() string {
	return c.job.Domain
}

// TriggerID returns the possible trigger identifier responsible for launching
// the job.
func (c *WorkerContext) TriggerID() (string, bool) {
	triggerID := c.job.TriggerID
	return triggerID, triggerID != ""
}

// Cookie returns the cookie associated with the worker context.
func (c *WorkerContext) Cookie() interface{} {
	return c.cookie
}

// NewWorker creates a new instance of Worker with the given configuration.
func NewWorker(conf *WorkerConfig) *Worker {
	return &Worker{
		Type: conf.WorkerType,
		Conf: conf,
	}
}

// Manual returns if the job was started manually
func (c *WorkerContext) Manual() bool {
	return c.job.Manual
}

// Start is used to start the worker consumption of messages from its queue.
func (w *Worker) Start(jobs chan *Job) error {
	if !atomic.CompareAndSwapUint32(&w.running, 0, 1) {
		return ErrClosed
	}
	w.jobs = jobs
	w.closed = make(chan struct{})
	if w.Conf.WorkerInit != nil {
		if err := w.Conf.WorkerInit(); err != nil {
			return fmt.Errorf("Could not start worker %s: %s", w.Type, err)
		}
	}
	for i := 0; i < w.Conf.Concurrency; i++ {
		name := fmt.Sprintf("%s/%d", w.Type, i)
		joblog.Debugf("Start worker %s", name)
		go w.work(name, w.closed)
	}
	return nil
}

// Shutdown is used to close the worker, waiting for all tasks to end
func (w *Worker) Shutdown(ctx context.Context) error {
	if !atomic.CompareAndSwapUint32(&w.running, 1, 0) {
		return ErrClosed
	}
	close(w.jobs)
	for i := 0; i < w.Conf.Concurrency; i++ {
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-w.closed:
		}
	}
	return nil
}

func (w *Worker) work(workerID string, closed chan<- struct{}) {
	for job := range w.jobs {
		domain := job.Domain
		if domain == "" {
			joblog.Errorf("[job] %s: missing domain from job request", workerID)
			continue
		}
		parentCtx := NewWorkerContext(workerID, job)
		if err := job.AckConsumed(); err != nil {
			parentCtx.Logger().Errorf("[job] error acking consume job: %s",
				err.Error())
			continue
		}
		t := &task{
			w:    w,
			ctx:  parentCtx,
			job:  job,
			conf: w.defaultedConf(job.Options),
		}
		var runResultLabel string
		var errAck error
		errRun := t.run()
		if errRun == ErrAbort {
			errRun = nil
		}
		if errRun != nil {
			parentCtx.Logger().Errorf("[job] error while performing job: %s",
				errRun.Error())
			runResultLabel = metrics.WorkerExecResultErrored
			errAck = job.Nack(errRun)
		} else {
			runResultLabel = metrics.WorkerExecResultSuccess
			errAck = job.Ack()
		}
		metrics.WorkerExecCounter.WithLabelValues(w.Type, runResultLabel).Inc()
		if errAck != nil {
			parentCtx.Logger().Errorf("[job] error while acking job done: %s",
				errAck.Error())
		}

		// Delete the trigger associated with the job (if any) when we receive a
		// ErrBadTrigger.
		//
		// XXX: better retro-action between broker and scheduler to avoid going
		// through the global job-system.
		if job.TriggerID != "" && globalJobSystem != nil {
			if _, ok := errRun.(ErrBadTrigger); ok {
				globalJobSystem.DeleteTrigger(job.Domain, job.TriggerID)
			}
		}
	}
	joblog.Debugf("[job] %s: worker shut down", workerID)
	closed <- struct{}{}
}

func (w *Worker) defaultedConf(opts *JobOptions) *WorkerConfig {
	c := w.Conf.Clone()
	if c.Concurrency == 0 {
		c.Concurrency = defaultConcurrency
	}
	if c.MaxExecCount == 0 {
		c.MaxExecCount = defaultMaxExecCount
	}
	if c.RetryDelay == 0 {
		c.RetryDelay = defaultRetryDelay
	}
	if c.Timeout == 0 {
		c.Timeout = defaultTimeout
	}
	if opts == nil {
		return c
	}
	if opts.MaxExecCount != 0 && opts.MaxExecCount < c.MaxExecCount {
		c.MaxExecCount = opts.MaxExecCount
	}
	if opts.Timeout > 0 && opts.Timeout < c.Timeout {
		c.Timeout = opts.Timeout
	}
	return c
}

type task struct {
	w    *Worker
	ctx  *WorkerContext
	conf *WorkerConfig
	job  *Job

	startTime time.Time
	execCount int
}

func (t *task) run() (err error) {
	t.startTime = time.Now()
	t.execCount = 0
	if t.conf.WorkerStart != nil {
		t.ctx, err = t.conf.WorkerStart(t.ctx)
		if err != nil {
			return err
		}
	}
	defer func() {
		if t.conf.WorkerCommit != nil {
			if errc := t.conf.WorkerCommit(t.ctx, err); errc != nil {
				t.ctx.Logger().Warnf("[job] error while commiting job: %s",
					errc.Error())
			}
		}
	}()
	for {
		retry, delay, timeout := t.nextDelay(err)
		if !retry {
			break
		}
		if err != nil {
			t.ctx.Logger().Warnf("[job] error while performing job: %s (retry in %s)",
				err.Error(), delay)
		}

		if delay > 0 {
			time.Sleep(delay)
		}

		t.ctx.Logger().Debugf("[job] executing job (%d) (timeout set to %s)",
			t.execCount, timeout)

		var execResultLabel string
		timer := prometheus.NewTimer(prometheus.ObserverFunc(func(v float64) {
			metrics.WorkerExecDurations.WithLabelValues(t.w.Type, execResultLabel).Observe(v)
		}))

		ctx, cancel := t.ctx.WithTimeout(timeout)
		err = t.exec(ctx)
		if err == nil {
			execResultLabel = metrics.WorkerExecResultSuccess
			timer.ObserveDuration()
			cancel()
			break
		}
		execResultLabel = metrics.WorkerExecResultErrored
		timer.ObserveDuration()

		// Even though ctx should have expired already, it is good practice to call
		// its cancelation function in any case. Failure to do so may keep the
		// context and its parent alive longer than necessary.
		cancel()
		t.execCount++

		if ctx.NoRetry() {
			break
		}
	}

	metrics.WorkerExecRetries.WithLabelValues(t.w.Type).Observe(float64(t.execCount))
	return
}

func (t *task) exec(ctx *WorkerContext) (err error) {
	var slot struct{}
	if slots != nil {
		slot = <-slots
	}
	defer func() {
		if slots != nil {
			slots <- slot
		}
		if r := recover(); r != nil {
			var ok bool
			err, ok = r.(error)
			if !ok {
				err = fmt.Errorf("%v", r)
			}
			ctx.Logger().Errorf("[panic] %s: %s", r, debug.Stack())
		}
	}()
	return t.conf.WorkerFunc(ctx)
}

func (t *task) nextDelay(prevError error) (bool, time.Duration, time.Duration) {
	// for certain kinds of errors, we do not have a retry since these error
	// cannot be recovered from
	{
		if _, ok := prevError.(ErrBadTrigger); ok {
			return false, 0, 0
		}
		switch prevError {
		case ErrAbort, ErrMessageUnmarshal, ErrMessageNil:
			return false, 0, 0
		}
	}

	c := t.conf

	if t.execCount >= c.MaxExecCount {
		return false, 0, 0
	}

	// the worker timeout should take into account the maximum execution time
	// allowed to the task
	timeout := c.Timeout

	var nextDelay time.Duration
	if t.execCount == 0 {
		// on first execution, execute immediately
		nextDelay = 0
	} else {
		nextDelay = c.RetryDelay << uint(t.execCount-1)

		// fuzzDelay number between delay * (1 +/- 0.1)
		fuzzDelay := int(0.1 * float64(nextDelay))
		nextDelay = nextDelay + time.Duration((rand.Intn(2*fuzzDelay) - fuzzDelay))
	}

	return true, nextDelay, timeout
}
<|endoftext|>


<commit_before>package pwh

import (
	"bytes"
	"crypto/hmac"
	"crypto/sha512"
	"math/rand"
	"sync"
)

const pwTable = "*?0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"

type PWHasher struct {
	lock       sync.RWMutex
	publicSalt []byte
	complexity int
}

func New(publicSalt string, complexity int) (pwh *PWHasher) {
	pwh = &PWHasher{}
	pwh.Config(publicSalt, complexity)
	return
}

func (pwh *PWHasher) Config(publicSalt string, complexity int) {
	if complexity < 1 {
		complexity = 1
	}
	publicSaltHasher := sha512.New()
	publicSaltHasher.Write([]byte(publicSalt))

	pwh.lock.Lock()
	defer pwh.lock.Unlock()

	pwh.complexity = complexity
	pwh.publicSalt = publicSaltHasher.Sum(nil)
}

func (pwh *PWHasher) Hash(word, salt string) string {
	pwh.lock.RLock()
	defer pwh.lock.RUnlock()

	return string(pwh.hash(rand.Int()%pwh.complexity, word, salt))
}

func (pwh *PWHasher) Match(word, salt, hash string) bool {
	pwh.lock.RLock()
	defer pwh.lock.RUnlock()

	for i := 0; i < pwh.complexity; i++ {
		if bytes.Equal([]byte(hash), pwh.hash(i, word, salt)) {
			return true
		}
	}
	return false
}

func (pwh *PWHasher) MatchX(word, salt, hash string, routines int) bool {
	if routines < 2 {
		return pwh.Match(word, salt, hash)
	}

	pwh.lock.RLock()
	defer pwh.lock.RUnlock()

	groups := (pwh.complexity + routines - 1) / routines
	matchc := make(chan bool, routines)
	matched := 0
	for i := 0; i < routines; i++ {
		go func(i int) {
			for s, e := i*groups, (i+1)*groups; s < e; s++ {
				if matched == routines {
					return
				}
				if bytes.Equal([]byte(hash), pwh.hash(s, word, salt)) {
					matchc <- true
					return
				}
			}
			matchc <- false
		}(i)
	}
	for {
		if <-matchc {
			matched = routines // Use for stoping all hasher goroutines
			return true
		} else if matched++; matched == routines {
			return false
		}
	}
}

func (pwh *PWHasher) hash(r int, word, salt string) []byte {
	codeTable := make([]byte, 64)
	for i, p := 0, rand.New(rand.NewSource(int64(r))).Perm(64); i < 64; i++ {
		codeTable[i] = pwTable[p[i]]
	}
	hmac := hmac.New(sha512.New384, pwh.publicSalt)
	hmac.Write([]byte(word))
	hmac.Write([]byte(salt))
	hashBytes := hmac.Sum(nil)
	hash := make([]byte, 64)
	for i, j := 0, 0; i < 48; i += 3 {
		j = i * 4 / 3
		hash[j] = codeTable[hashBytes[i]>>2]
		hash[j+1] = codeTable[(hashBytes[i]&0x3)<<4|hashBytes[i+1]>>4]
		hash[j+2] = codeTable[(hashBytes[i+1]&0xf)<<2|hashBytes[i+2]>>6]
		hash[j+3] = codeTable[hashBytes[i+2]&0x3f]
	}
	return hash
}

var globalPWHasher *PWHasher

func Config(publicSalt string, complexity int) {
	if complexity < 1 {
		complexity = 1
	}
	publicSaltHasher := sha512.New()
	publicSaltHasher.Write([]byte(publicSalt))
	globalPWHasher.complexity = complexity
	globalPWHasher.publicSalt = publicSaltHasher.Sum(nil)
}

func Hash(word, salt string) string {
	return globalPWHasher.Hash(word, salt)
}

func Match(word, salt, hash string) bool {
	return globalPWHasher.Match(word, salt, hash)
}

func MatchX(word, salt, hash string, routines int) bool {
	return globalPWHasher.MatchX(word, salt, hash, routines)
}

func init() {
	globalPWHasher = New("go", 1024)
}
<commit_msg>change hash logic<commit_after>package pwh

import (
	"bytes"
	"crypto/hmac"
	"crypto/sha512"
	"math/rand"
	"sync"
)

const pwTable = "*?0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"

var globalPWHasher *PWHasher

type PWHasher struct {
	lock       sync.RWMutex
	publicSalt []byte
	complexity int
}

func New(publicSalt string, complexity int) (pwh *PWHasher) {
	pwh = &PWHasher{}
	pwh.Config(publicSalt, complexity)
	return
}

func (pwh *PWHasher) Config(publicSalt string, complexity int) {
	if complexity < 1 {
		complexity = 1
	}
	publicSaltHasher := sha512.New()
	publicSaltHasher.Write([]byte(publicSalt))

	pwh.lock.Lock()
	defer pwh.lock.Unlock()

	pwh.complexity = complexity
	pwh.publicSalt = publicSaltHasher.Sum(nil)
}

func (pwh *PWHasher) Hash(word, salt string) string {
	pwh.lock.RLock()
	defer pwh.lock.RUnlock()

	return string(pwh.hash(rand.Int()%pwh.complexity, word, salt))
}

func (pwh *PWHasher) Match(word, salt, hash string) bool {
	pwh.lock.RLock()
	defer pwh.lock.RUnlock()

	for i := 0; i < pwh.complexity; i++ {
		if bytes.Equal([]byte(hash), pwh.hash(i, word, salt)) {
			return true
		}
	}
	return false
}

func (pwh *PWHasher) MatchX(word, salt, hash string, routines int) bool {
	if routines < 2 {
		return pwh.Match(word, salt, hash)
	}

	pwh.lock.RLock()
	defer pwh.lock.RUnlock()

	groups := (pwh.complexity + routines - 1) / routines
	matchc := make(chan bool, routines)
	matched := 0
	for i := 0; i < routines; i++ {
		go func(i int) {
			for s, e := i*groups, (i+1)*groups; s < e; s++ {
				if matched == routines {
					return
				}
				if bytes.Equal([]byte(hash), pwh.hash(s, word, salt)) {
					matchc <- true
					return
				}
			}
			matchc <- false
		}(i)
	}
	for {
		if <-matchc {
			matched = routines // Use for stoping all hasher goroutines
			return true
		} else if matched++; matched == routines {
			return false
		}
	}
}

func (pwh *PWHasher) hash(r int, word, salt string) []byte {
	codeTable := make([]byte, 64)
	for i, p := 0, rand.New(rand.NewSource(int64(r))).Perm(64); i < 64; i++ {
		codeTable[i] = pwTable[p[i]]
	}
	hasher := hmac.New(sha512.New, pwh.publicSalt)
	hasher.Write([]byte(salt))
	hasher = hmac.New(sha512.New384, hasher.Sum(nil))
	hasher.Write([]byte(word))
	hashBytes := hasher.Sum(nil)
	hash := make([]byte, 64)
	for i, j := 0, 0; i < 48; i += 3 {
		j = i * 4 / 3
		hash[j] = codeTable[hashBytes[i]>>2]
		hash[j+1] = codeTable[(hashBytes[i]&0x3)<<4|hashBytes[i+1]>>4]
		hash[j+2] = codeTable[(hashBytes[i+1]&0xf)<<2|hashBytes[i+2]>>6]
		hash[j+3] = codeTable[hashBytes[i+2]&0x3f]
	}
	return hash
}

func Config(publicSalt string, complexity int) {
	if complexity < 1 {
		complexity = 1
	}
	publicSaltHasher := sha512.New()
	publicSaltHasher.Write([]byte(publicSalt))
	globalPWHasher.complexity = complexity
	globalPWHasher.publicSalt = publicSaltHasher.Sum(nil)
}

func Hash(word, salt string) string {
	return globalPWHasher.Hash(word, salt)
}

func Match(word, salt, hash string) bool {
	return globalPWHasher.Match(word, salt, hash)
}

func MatchX(word, salt, hash string, routines int) bool {
	return globalPWHasher.MatchX(word, salt, hash, routines)
}

func init() {
	globalPWHasher = New("go", 1024)
}
<|endoftext|>