<commit_before>package fzf

import (
	"strconv"
	"strings"
	"unicode/utf8"

	"github.com/junegunn/fzf/src/tui"
)

type ansiOffset struct {
	offset [2]int32
	color  ansiState
}

type ansiState struct {
	fg   tui.Color
	bg   tui.Color
	attr tui.Attr
	lbg  tui.Color
}

func (s *ansiState) colored() bool {
	return s.fg != -1 || s.bg != -1 || s.attr > 0 || s.lbg >= 0
}

func (s *ansiState) equals(t *ansiState) bool {
	if t == nil {
		return !s.colored()
	}
	return s.fg == t.fg && s.bg == t.bg && s.attr == t.attr && s.lbg == t.lbg
}

func (s *ansiState) ToString() string {
	if !s.colored() {
		return ""
	}

	ret := ""
	if s.attr&tui.Bold > 0 {
		ret += "1;"
	}
	if s.attr&tui.Dim > 0 {
		ret += "2;"
	}
	if s.attr&tui.Italic > 0 {
		ret += "3;"
	}
	if s.attr&tui.Underline > 0 {
		ret += "4;"
	}
	if s.attr&tui.Blink > 0 {
		ret += "5;"
	}
	if s.attr&tui.Reverse > 0 {
		ret += "7;"
	}
	ret += toAnsiString(s.fg, 30) + toAnsiString(s.bg, 40)

	return "\x1b[" + strings.TrimSuffix(ret, ";") + "m"
}

func toAnsiString(color tui.Color, offset int) string {
	col := int(color)
	ret := ""
	if col == -1 {
		ret += strconv.Itoa(offset + 9)
	} else if col < 8 {
		ret += strconv.Itoa(offset + col)
	} else if col < 16 {
		ret += strconv.Itoa(offset - 30 + 90 + col - 8)
	} else if col < 256 {
		ret += strconv.Itoa(offset+8) + ";5;" + strconv.Itoa(col)
	} else if col >= (1 << 24) {
		r := strconv.Itoa((col >> 16) & 0xff)
		g := strconv.Itoa((col >> 8) & 0xff)
		b := strconv.Itoa(col & 0xff)
		ret += strconv.Itoa(offset+8) + ";2;" + r + ";" + g + ";" + b
	}
	return ret + ";"
}

func isPrint(c uint8) bool {
	return '\x20' <= c && c <= '\x7e'
}

func matchOperatingSystemCommand(s string) int {
	// `\x1b][0-9];[[:print:]]+(?:\x1b\\\\|\x07)`
	//                        ^ match starting here
	//
	i := 5 // prefix matched in nextAnsiEscapeSequence()
	for ; i < len(s) && isPrint(s[i]); i++ {
	}
	if i < len(s) {
		if s[i] == '\x07' {
			return i + 1
		}
		if s[i] == '\x1b' && i < len(s)-1 && s[i+1] == '\\' {
			return i + 2
		}
	}
	return -1
}

func matchControlSequence(s string) int {
	// `\x1b[\\[()][0-9;]*[a-zA-Z@]`
	//                   ^ match starting here
	//
	i := 2 // prefix matched in nextAnsiEscapeSequence()
	for ; i < len(s) && (isNumeric(s[i]) || s[i] == ';'); i++ {
	}
	if i < len(s) {
		c := s[i]
		if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || c == '@' {
			return i + 1
		}
	}
	return -1
}

func isCtrlSeqStart(c uint8) bool {
	return c == '\\' || c == '[' || c == '(' || c == ')'
}

// nextAnsiEscapeSequence returns the ANSI escape sequence and is equivalent to
// calling FindStringIndex() on the below regex (which was originally used):
//
// "(?:\x1b[\\[()][0-9;]*[a-zA-Z@]|\x1b][0-9];[[:print:]]+(?:\x1b\\\\|\x07)|\x1b.|[\x0e\x0f]|.\x08)"
//
func nextAnsiEscapeSequence(s string) (int, int) {
	// fast check for ANSI escape sequences
	i := 0
	for ; i < len(s); i++ {
		switch s[i] {
		case '\x0e', '\x0f', '\x1b', '\x08':
			// We ignore the fact that '\x08' cannot be the first char
			// in the string and be an escape sequence for the sake of
			// speed and simplicity.
			goto Loop
		}
	}
	return -1, -1

Loop:
	for ; i < len(s); i++ {
		switch s[i] {
		case '\x08':
			// backtrack to match: `.\x08`
			if i > 0 && s[i-1] != '\n' {
				if s[i-1] < utf8.RuneSelf {
					return i - 1, i + 1
				}
				_, n := utf8.DecodeLastRuneInString(s[:i])
				return i - n, i + 1
			}
		case '\x1b':
			// match: `\x1b[\\[()][0-9;]*[a-zA-Z@]`
			if i+2 < len(s) && isCtrlSeqStart(s[i+1]) {
				if j := matchControlSequence(s[i:]); j != -1 {
					return i, i + j
				}
			}

			// match: `\x1b][0-9];[[:print:]]+(?:\x1b\\\\|\x07)`
			if i+5 < len(s) && s[i+1] == ']' && isNumeric(s[i+2]) &&
				s[i+3] == ';' && isPrint(s[i+4]) {

				if j := matchOperatingSystemCommand(s[i:]); j != -1 {
					return i, i + j
				}
			}

			// match: `\x1b.`
			if i+1 < len(s) && s[i+1] != '\n' {
				if s[i+1] < utf8.RuneSelf {
					return i, i + 2
				}
				_, n := utf8.DecodeRuneInString(s[i+1:])
				return i, i + n + 1
			}
		case '\x0e', '\x0f':
			// match: `[\x0e\x0f]`
			return i, i + 1
		}
	}
	return -1, -1
}

func extractColor(str string, state *ansiState, proc func(string, *ansiState) bool) (string, *[]ansiOffset, *ansiState) {
	// We append to a stack allocated variable that we'll
	// later copy and return, to save on allocations.
	offsets := make([]ansiOffset, 0, 32)

	if state != nil {
		offsets = append(offsets, ansiOffset{[2]int32{0, 0}, *state})
	}

	var (
		pstate    *ansiState // lazily allocated
		output    strings.Builder
		prevIdx   int
		runeCount int
	)
	for idx := 0; idx < len(str); {
		// Make sure that we found an ANSI code
		start, end := nextAnsiEscapeSequence(str[idx:])
		if start == -1 {
			break
		}
		start += idx
		idx += end

		// Check if we should continue
		prev := str[prevIdx:start]
		if proc != nil && !proc(prev, state) {
			return "", nil, nil
		}
		prevIdx = idx

		if len(prev) != 0 {
			runeCount += utf8.RuneCountInString(prev)
			// Grow the buffer size to the maximum possible length (string length
			// containing ansi codes) to avoid repetitive allocation
			if output.Cap() == 0 {
				output.Grow(len(str))
			}
			output.WriteString(prev)
		}

		newState := interpretCode(str[start:idx], state)
		if !newState.equals(state) {
			if state != nil {
				// Update last offset
				(&offsets[len(offsets)-1]).offset[1] = int32(runeCount)
			}

			if newState.colored() {
				// Append new offset
				if pstate == nil {
					pstate = &ansiState{}
				}
				*pstate = newState
				state = pstate
				offsets = append(offsets, ansiOffset{
					[2]int32{int32(runeCount), int32(runeCount)},
					newState,
				})
			} else {
				// Discard state
				state = nil
			}
		}
	}

	var rest string
	var trimmed string
	if prevIdx == 0 {
		// No ANSI code found
		rest = str
		trimmed = str
	} else {
		rest = str[prevIdx:]
		output.WriteString(rest)
		trimmed = output.String()
	}
	if proc != nil {
		proc(rest, state)
	}
	if len(offsets) > 0 {
		if len(rest) > 0 && state != nil {
			// Update last offset
			runeCount += utf8.RuneCountInString(rest)
			(&offsets[len(offsets)-1]).offset[1] = int32(runeCount)
		}
		// Return a copy of the offsets slice
		a := make([]ansiOffset, len(offsets))
		copy(a, offsets)
		return trimmed, &a, state
	}
	return trimmed, nil, state
}

func parseAnsiCode(s string) (int, string) {
	var remaining string
	if i := strings.IndexByte(s, ';'); i >= 0 {
		remaining = s[i+1:]
		s = s[:i]
	}

	if len(s) > 0 {
		// Inlined version of strconv.Atoi() that only handles positive
		// integers and does not allocate on error.
		code := 0
		for _, ch := range []byte(s) {
			ch -= '0'
			if ch > 9 {
				return -1, remaining
			}
			code = code*10 + int(ch)
		}
		return code, remaining
	}

	return -1, remaining
}

func interpretCode(ansiCode string, prevState *ansiState) ansiState {
	var state ansiState
	if prevState == nil {
		state = ansiState{-1, -1, 0, -1}
	} else {
		state = ansiState{prevState.fg, prevState.bg, prevState.attr, prevState.lbg}
	}
	if ansiCode[0] != '\x1b' || ansiCode[1] != '[' || ansiCode[len(ansiCode)-1] != 'm' {
		if prevState != nil && strings.HasSuffix(ansiCode, "0K") {
			state.lbg = prevState.bg
		}
		return state
	}

	if len(ansiCode) <= 3 {
		state.fg = -1
		state.bg = -1
		state.attr = 0
		return state
	}
	ansiCode = ansiCode[2 : len(ansiCode)-1]

	state256 := 0
	ptr := &state.fg

	for len(ansiCode) != 0 {
		var num int
		if num, ansiCode = parseAnsiCode(ansiCode); num != -1 {
			switch state256 {
			case 0:
				switch num {
				case 38:
					ptr = &state.fg
					state256++
				case 48:
					ptr = &state.bg
					state256++
				case 39:
					state.fg = -1
				case 49:
					state.bg = -1
				case 1:
					state.attr = state.attr | tui.Bold
				case 2:
					state.attr = state.attr | tui.Dim
				case 3:
					state.attr = state.attr | tui.Italic
				case 4:
					state.attr = state.attr | tui.Underline
				case 5:
					state.attr = state.attr | tui.Blink
				case 7:
					state.attr = state.attr | tui.Reverse
				case 23: // tput rmso
					state.attr = state.attr &^ tui.Italic
				case 24: // tput rmul
					state.attr = state.attr &^ tui.Underline
				case 0:
					state.fg = -1
					state.bg = -1
					state.attr = 0
					state256 = 0
				default:
					if num >= 30 && num <= 37 {
						state.fg = tui.Color(num - 30)
					} else if num >= 40 && num <= 47 {
						state.bg = tui.Color(num - 40)
					} else if num >= 90 && num <= 97 {
						state.fg = tui.Color(num - 90 + 8)
					} else if num >= 100 && num <= 107 {
						state.bg = tui.Color(num - 100 + 8)
					}
				}
			case 1:
				switch num {
				case 2:
					state256 = 10 // MAGIC
				case 5:
					state256++
				default:
					state256 = 0
				}
			case 2:
				*ptr = tui.Color(num)
				state256 = 0
			case 10:
				*ptr = tui.Color(1<<24) | tui.Color(num<<16)
				state256++
			case 11:
				*ptr = *ptr | tui.Color(num<<8)
				state256++
			case 12:
				*ptr = *ptr | tui.Color(num)
				state256 = 0
			}
		}
	}

	if state256 > 0 {
		*ptr = -1
	}
	return state
}
<commit_msg>Ignore more ANSI escape sequences<commit_after>package fzf

import (
	"strconv"
	"strings"
	"unicode/utf8"

	"github.com/junegunn/fzf/src/tui"
)

type ansiOffset struct {
	offset [2]int32
	color  ansiState
}

type ansiState struct {
	fg   tui.Color
	bg   tui.Color
	attr tui.Attr
	lbg  tui.Color
}

func (s *ansiState) colored() bool {
	return s.fg != -1 || s.bg != -1 || s.attr > 0 || s.lbg >= 0
}

func (s *ansiState) equals(t *ansiState) bool {
	if t == nil {
		return !s.colored()
	}
	return s.fg == t.fg && s.bg == t.bg && s.attr == t.attr && s.lbg == t.lbg
}

func (s *ansiState) ToString() string {
	if !s.colored() {
		return ""
	}

	ret := ""
	if s.attr&tui.Bold > 0 {
		ret += "1;"
	}
	if s.attr&tui.Dim > 0 {
		ret += "2;"
	}
	if s.attr&tui.Italic > 0 {
		ret += "3;"
	}
	if s.attr&tui.Underline > 0 {
		ret += "4;"
	}
	if s.attr&tui.Blink > 0 {
		ret += "5;"
	}
	if s.attr&tui.Reverse > 0 {
		ret += "7;"
	}
	ret += toAnsiString(s.fg, 30) + toAnsiString(s.bg, 40)

	return "\x1b[" + strings.TrimSuffix(ret, ";") + "m"
}

func toAnsiString(color tui.Color, offset int) string {
	col := int(color)
	ret := ""
	if col == -1 {
		ret += strconv.Itoa(offset + 9)
	} else if col < 8 {
		ret += strconv.Itoa(offset + col)
	} else if col < 16 {
		ret += strconv.Itoa(offset - 30 + 90 + col - 8)
	} else if col < 256 {
		ret += strconv.Itoa(offset+8) + ";5;" + strconv.Itoa(col)
	} else if col >= (1 << 24) {
		r := strconv.Itoa((col >> 16) & 0xff)
		g := strconv.Itoa((col >> 8) & 0xff)
		b := strconv.Itoa(col & 0xff)
		ret += strconv.Itoa(offset+8) + ";2;" + r + ";" + g + ";" + b
	}
	return ret + ";"
}

func isPrint(c uint8) bool {
	return '\x20' <= c && c <= '\x7e'
}

func matchOperatingSystemCommand(s string) int {
	// `\x1b][0-9];[[:print:]]+(?:\x1b\\\\|\x07)`
	//                        ^ match starting here
	//
	i := 5 // prefix matched in nextAnsiEscapeSequence()
	for ; i < len(s) && isPrint(s[i]); i++ {
	}
	if i < len(s) {
		if s[i] == '\x07' {
			return i + 1
		}
		if s[i] == '\x1b' && i < len(s)-1 && s[i+1] == '\\' {
			return i + 2
		}
	}
	return -1
}

func matchControlSequence(s string) int {
	// `\x1b[\\[()][0-9;?]*[a-zA-Z@]`
	//                    ^ match starting here
	//
	i := 2 // prefix matched in nextAnsiEscapeSequence()
	for ; i < len(s) && (isNumeric(s[i]) || s[i] == ';' || s[i] == '?'); i++ {
	}
	if i < len(s) {
		c := s[i]
		if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || c == '@' {
			return i + 1
		}
	}
	return -1
}

func isCtrlSeqStart(c uint8) bool {
	return c == '\\' || c == '[' || c == '(' || c == ')'
}

// nextAnsiEscapeSequence returns the ANSI escape sequence and is equivalent to
// calling FindStringIndex() on the below regex (which was originally used):
//
// "(?:\x1b[\\[()][0-9;?]*[a-zA-Z@]|\x1b][0-9];[[:print:]]+(?:\x1b\\\\|\x07)|\x1b.|[\x0e\x0f]|.\x08)"
//
func nextAnsiEscapeSequence(s string) (int, int) {
	// fast check for ANSI escape sequences
	i := 0
	for ; i < len(s); i++ {
		switch s[i] {
		case '\x0e', '\x0f', '\x1b', '\x08':
			// We ignore the fact that '\x08' cannot be the first char
			// in the string and be an escape sequence for the sake of
			// speed and simplicity.
			goto Loop
		}
	}
	return -1, -1

Loop:
	for ; i < len(s); i++ {
		switch s[i] {
		case '\x08':
			// backtrack to match: `.\x08`
			if i > 0 && s[i-1] != '\n' {
				if s[i-1] < utf8.RuneSelf {
					return i - 1, i + 1
				}
				_, n := utf8.DecodeLastRuneInString(s[:i])
				return i - n, i + 1
			}
		case '\x1b':
			// match: `\x1b[\\[()][0-9;?]*[a-zA-Z@]`
			if i+2 < len(s) && isCtrlSeqStart(s[i+1]) {
				if j := matchControlSequence(s[i:]); j != -1 {
					return i, i + j
				}
			}

			// match: `\x1b][0-9];[[:print:]]+(?:\x1b\\\\|\x07)`
			if i+5 < len(s) && s[i+1] == ']' && isNumeric(s[i+2]) &&
				s[i+3] == ';' && isPrint(s[i+4]) {

				if j := matchOperatingSystemCommand(s[i:]); j != -1 {
					return i, i + j
				}
			}

			// match: `\x1b.`
			if i+1 < len(s) && s[i+1] != '\n' {
				if s[i+1] < utf8.RuneSelf {
					return i, i + 2
				}
				_, n := utf8.DecodeRuneInString(s[i+1:])
				return i, i + n + 1
			}
		case '\x0e', '\x0f':
			// match: `[\x0e\x0f]`
			return i, i + 1
		}
	}
	return -1, -1
}

func extractColor(str string, state *ansiState, proc func(string, *ansiState) bool) (string, *[]ansiOffset, *ansiState) {
	// We append to a stack allocated variable that we'll
	// later copy and return, to save on allocations.
	offsets := make([]ansiOffset, 0, 32)

	if state != nil {
		offsets = append(offsets, ansiOffset{[2]int32{0, 0}, *state})
	}

	var (
		pstate    *ansiState // lazily allocated
		output    strings.Builder
		prevIdx   int
		runeCount int
	)
	for idx := 0; idx < len(str); {
		// Make sure that we found an ANSI code
		start, end := nextAnsiEscapeSequence(str[idx:])
		if start == -1 {
			break
		}
		start += idx
		idx += end

		// Check if we should continue
		prev := str[prevIdx:start]
		if proc != nil && !proc(prev, state) {
			return "", nil, nil
		}
		prevIdx = idx

		if len(prev) != 0 {
			runeCount += utf8.RuneCountInString(prev)
			// Grow the buffer size to the maximum possible length (string length
			// containing ansi codes) to avoid repetitive allocation
			if output.Cap() == 0 {
				output.Grow(len(str))
			}
			output.WriteString(prev)
		}

		newState := interpretCode(str[start:idx], state)
		if !newState.equals(state) {
			if state != nil {
				// Update last offset
				(&offsets[len(offsets)-1]).offset[1] = int32(runeCount)
			}

			if newState.colored() {
				// Append new offset
				if pstate == nil {
					pstate = &ansiState{}
				}
				*pstate = newState
				state = pstate
				offsets = append(offsets, ansiOffset{
					[2]int32{int32(runeCount), int32(runeCount)},
					newState,
				})
			} else {
				// Discard state
				state = nil
			}
		}
	}

	var rest string
	var trimmed string
	if prevIdx == 0 {
		// No ANSI code found
		rest = str
		trimmed = str
	} else {
		rest = str[prevIdx:]
		output.WriteString(rest)
		trimmed = output.String()
	}
	if proc != nil {
		proc(rest, state)
	}
	if len(offsets) > 0 {
		if len(rest) > 0 && state != nil {
			// Update last offset
			runeCount += utf8.RuneCountInString(rest)
			(&offsets[len(offsets)-1]).offset[1] = int32(runeCount)
		}
		// Return a copy of the offsets slice
		a := make([]ansiOffset, len(offsets))
		copy(a, offsets)
		return trimmed, &a, state
	}
	return trimmed, nil, state
}

func parseAnsiCode(s string) (int, string) {
	var remaining string
	if i := strings.IndexByte(s, ';'); i >= 0 {
		remaining = s[i+1:]
		s = s[:i]
	}

	if len(s) > 0 {
		// Inlined version of strconv.Atoi() that only handles positive
		// integers and does not allocate on error.
		code := 0
		for _, ch := range []byte(s) {
			ch -= '0'
			if ch > 9 {
				return -1, remaining
			}
			code = code*10 + int(ch)
		}
		return code, remaining
	}

	return -1, remaining
}

func interpretCode(ansiCode string, prevState *ansiState) ansiState {
	var state ansiState
	if prevState == nil {
		state = ansiState{-1, -1, 0, -1}
	} else {
		state = ansiState{prevState.fg, prevState.bg, prevState.attr, prevState.lbg}
	}
	if ansiCode[0] != '\x1b' || ansiCode[1] != '[' || ansiCode[len(ansiCode)-1] != 'm' {
		if prevState != nil && strings.HasSuffix(ansiCode, "0K") {
			state.lbg = prevState.bg
		}
		return state
	}

	if len(ansiCode) <= 3 {
		state.fg = -1
		state.bg = -1
		state.attr = 0
		return state
	}
	ansiCode = ansiCode[2 : len(ansiCode)-1]

	state256 := 0
	ptr := &state.fg

	for len(ansiCode) != 0 {
		var num int
		if num, ansiCode = parseAnsiCode(ansiCode); num != -1 {
			switch state256 {
			case 0:
				switch num {
				case 38:
					ptr = &state.fg
					state256++
				case 48:
					ptr = &state.bg
					state256++
				case 39:
					state.fg = -1
				case 49:
					state.bg = -1
				case 1:
					state.attr = state.attr | tui.Bold
				case 2:
					state.attr = state.attr | tui.Dim
				case 3:
					state.attr = state.attr | tui.Italic
				case 4:
					state.attr = state.attr | tui.Underline
				case 5:
					state.attr = state.attr | tui.Blink
				case 7:
					state.attr = state.attr | tui.Reverse
				case 23: // tput rmso
					state.attr = state.attr &^ tui.Italic
				case 24: // tput rmul
					state.attr = state.attr &^ tui.Underline
				case 0:
					state.fg = -1
					state.bg = -1
					state.attr = 0
					state256 = 0
				default:
					if num >= 30 && num <= 37 {
						state.fg = tui.Color(num - 30)
					} else if num >= 40 && num <= 47 {
						state.bg = tui.Color(num - 40)
					} else if num >= 90 && num <= 97 {
						state.fg = tui.Color(num - 90 + 8)
					} else if num >= 100 && num <= 107 {
						state.bg = tui.Color(num - 100 + 8)
					}
				}
			case 1:
				switch num {
				case 2:
					state256 = 10 // MAGIC
				case 5:
					state256++
				default:
					state256 = 0
				}
			case 2:
				*ptr = tui.Color(num)
				state256 = 0
			case 10:
				*ptr = tui.Color(1<<24) | tui.Color(num<<16)
				state256++
			case 11:
				*ptr = *ptr | tui.Color(num<<8)
				state256++
			case 12:
				*ptr = *ptr | tui.Color(num)
				state256 = 0
			}
		}
	}

	if state256 > 0 {
		*ptr = -1
	}
	return state
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"github.com/ProfessorBeekums/PbStockResearcher/config"
	"github.com/ProfessorBeekums/PbStockResearcher/log"
	"github.com/ProfessorBeekums/PbStockResearcher/persist"
	"github.com/ProfessorBeekums/PbStockResearcher/scraper"
	"github.com/ProfessorBeekums/PbStockResearcher/tmpStore"
)

var year int
var quarter int

func init() {
	flag.IntVar(&year, "year", 0, "The year to scrape")
	flag.IntVar(&quarter, "quarter", 0, "The quarter to scrape")
}

func main() {
	log.Println("Starting program")

	flag.Parse()

	c := config.NewConfig("/home/beekums/Projects/stockResearch/config")

	log.Println("Loaded config: ", c)

	ts := tmpStore.NewTempStore(c.TmpDir)

	mysql := persist.NewMysqlDb(c.MysqlUser, c.MysqlPass, c.MysqlDb)

	scraper := scraper.NewEdgarFullIndexScraper(year, quarter, ts, mysql, mysql)

	scraper.ScrapeEdgarQuarterlyIndex()

	log.Println("Ending program")
}
<commit_msg>cleanup naming so intellij is not confused<commit_after>package main

import (
	"flag"
	"github.com/ProfessorBeekums/PbStockResearcher/config"
	"github.com/ProfessorBeekums/PbStockResearcher/log"
	"github.com/ProfessorBeekums/PbStockResearcher/persist"
	"github.com/ProfessorBeekums/PbStockResearcher/scraper"
	"github.com/ProfessorBeekums/PbStockResearcher/tmpStore"
)

var year int
var quarter int

func init() {
	flag.IntVar(&year, "year", 0, "The year to scrape")
	flag.IntVar(&quarter, "quarter", 0, "The quarter to scrape")
}

func main() {
	log.Println("Starting program")

	flag.Parse()

	c := config.NewConfig("/home/beekums/Projects/stockResearch/config")

	log.Println("Loaded config: ", c)

	ts := tmpStore.NewTempStore(c.TmpDir)

	mysql := persist.NewMysqlDb(c.MysqlUser, c.MysqlPass, c.MysqlDb)

	pbScraper := scraper.NewEdgarFullIndexScraper(year, quarter, ts, mysql, mysql)

	pbScraper.ScrapeEdgarQuarterlyIndex()

	log.Println("Ending program")
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os"
	"time"

	"github.com/valyala/fasthttp"
	"net/url"
)

var bytesSlash = []byte("/") // heap optimization

// HTTPClient is a reusable HTTP Client.
type HTTPClient struct {
	client     fasthttp.Client
	Host       []byte
	HostString string
	uri        []byte
	debug      int
}

// HTTPClientDoOptions wraps options uses when calling `Do`.
type HTTPClientDoOptions struct {
	Debug                int
	PrettyPrintResponses bool
}

// NewHTTPClient creates a new HTTPClient.
func NewHTTPClient(host string, debug int) *HTTPClient {
	return &HTTPClient{
		client: fasthttp.Client{
			Name: "query_benchmarker",
		},
		Host:       []byte(host),
		HostString: host,
		uri:        []byte{}, // heap optimization
		debug:      debug,
	}
}

// Do performs the action specified by the given Query. It uses fasthttp, and
// tries to minimize heap allocations.
func (w *HTTPClient) Do(q *Query, opts *HTTPClientDoOptions) (lag float64, err error) {
	// populate uri from the reusable byte slice:
	w.uri = w.uri[:0]
	w.uri = append(w.uri, w.Host...)
	w.uri = append(w.uri, bytesSlash...)
	w.uri = append(w.uri, q.Path...)

	// populate a request with data from the Query:
	req := fasthttp.AcquireRequest()
	defer fasthttp.ReleaseRequest(req)

	req.Header.SetMethodBytes(q.Method)
	req.Header.SetRequestURIBytes(w.uri)
	req.SetBody(q.Body)
	if opts.Debug > 0 {
		values, _ := url.ParseQuery(string(q.Path))
		fmt.Printf("debug: query - %s\n", values)
	}
	// Perform the request while tracking latency:
	resp := fasthttp.AcquireResponse()
	defer fasthttp.ReleaseResponse(resp)
	start := time.Now()
	err = w.client.Do(req, resp)
	lag = float64(time.Since(start).Nanoseconds()) / 1e6 // milliseconds

	// Check that the status code was 200 OK:
	if err == nil {
		sc := resp.StatusCode()
		if sc != fasthttp.StatusOK {
			err = fmt.Errorf("Invalid write response (status %d): %s", sc, resp.Body())
			return
		}
	}

	if opts != nil {
		// Print debug messages, if applicable:
		switch opts.Debug {
		case 1:
			fmt.Fprintf(os.Stderr, "debug: %s in %7.2fms\n", q.HumanLabel, lag)
		case 2:
			fmt.Fprintf(os.Stderr, "debug: %s in %7.2fms -- %s\n", q.HumanLabel, lag, q.HumanDescription)
		case 3:
			fmt.Fprintf(os.Stderr, "debug: %s in %7.2fms -- %s\n", q.HumanLabel, lag, q.HumanDescription)
			fmt.Fprintf(os.Stderr, "debug:   request: %s\n", string(q.String()))
		case 4:
			fmt.Fprintf(os.Stderr, "debug: %s in %7.2fms -- %s\n", q.HumanLabel, lag, q.HumanDescription)
			fmt.Fprintf(os.Stderr, "debug:   request: %s\n", string(q.String()))
			fmt.Fprintf(os.Stderr, "debug:   response: %s\n", string(resp.Body()))
		default:
		}

		// Pretty print JSON responses, if applicable:
		if opts.PrettyPrintResponses {
			// InfluxQL responses are in JSON and can be pretty-printed here.
			// Flux responses are just simple CSV.

			prefix := fmt.Sprintf("ID %d: ", q.ID)
			if json.Valid(resp.Body()) {
				var pretty bytes.Buffer
				err = json.Indent(&pretty, resp.Body(), prefix, "  ")
				if err != nil {
					return
				}

				_, err = fmt.Fprintf(os.Stderr, "%s%s\n", prefix, pretty)
				if err != nil {
					return
				}
			} else {
				_, err = fmt.Fprintf(os.Stderr, "%s%s\n", prefix, resp.Body())
				if err != nil {
					return
				}
			}
		}
	}

	return lag, err
}
<commit_msg>Fixing not threadsafe uri handling<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os"
	"time"

	"github.com/valyala/fasthttp"
	"net/url"
)

var bytesSlash = []byte("/") // heap optimization

// HTTPClient is a reusable HTTP Client.
type HTTPClient struct {
	client     fasthttp.Client
	Host       []byte
	HostString string
	debug      int
}

// HTTPClientDoOptions wraps options uses when calling `Do`.
type HTTPClientDoOptions struct {
	Debug                int
	PrettyPrintResponses bool
}

// NewHTTPClient creates a new HTTPClient.
func NewHTTPClient(host string, debug int) *HTTPClient {
	return &HTTPClient{
		client: fasthttp.Client{
			Name: "query_benchmarker",
		},
		Host:       []byte(host),
		HostString: host,
		debug:      debug,
	}
}

// Do performs the action specified by the given Query. It uses fasthttp, and
// tries to minimize heap allocations.
func (w *HTTPClient) Do(q *Query, opts *HTTPClientDoOptions) (lag float64, err error) {
	// populate uri from the reusable byte slice:
	uri := make([]byte, 0, 100)
	uri = append(uri, w.Host...)
	uri = append(uri, bytesSlash...)
	uri = append(uri, q.Path...)

	// populate a request with data from the Query:
	req := fasthttp.AcquireRequest()
	defer fasthttp.ReleaseRequest(req)

	req.Header.SetMethodBytes(q.Method)
	req.Header.SetRequestURIBytes(uri)
	req.SetBody(q.Body)
	if opts.Debug > 0 {
		values, _ := url.ParseQuery(string(q.Path))
		fmt.Printf("debug: query - %s\n", values)
	}
	// Perform the request while tracking latency:
	resp := fasthttp.AcquireResponse()
	defer fasthttp.ReleaseResponse(resp)
	start := time.Now()
	err = w.client.Do(req, resp)
	lag = float64(time.Since(start).Nanoseconds()) / 1e6 // milliseconds

	// Check that the status code was 200 OK:
	if err == nil {
		sc := resp.StatusCode()
		if sc != fasthttp.StatusOK {
			err = fmt.Errorf("Invalid write response (status %d): %s", sc, resp.Body())
			return
		}
	}

	if opts != nil {
		// Print debug messages, if applicable:
		switch opts.Debug {
		case 1:
			fmt.Fprintf(os.Stderr, "debug: %s in %7.2fms\n", q.HumanLabel, lag)
		case 2:
			fmt.Fprintf(os.Stderr, "debug: %s in %7.2fms -- %s\n", q.HumanLabel, lag, q.HumanDescription)
		case 3:
			fmt.Fprintf(os.Stderr, "debug: %s in %7.2fms -- %s\n", q.HumanLabel, lag, q.HumanDescription)
			fmt.Fprintf(os.Stderr, "debug:   request: %s\n", string(q.String()))
		case 4:
			fmt.Fprintf(os.Stderr, "debug: %s in %7.2fms -- %s\n", q.HumanLabel, lag, q.HumanDescription)
			fmt.Fprintf(os.Stderr, "debug:   request: %s\n", string(q.String()))
			fmt.Fprintf(os.Stderr, "debug:   response: %s\n", string(resp.Body()))
		default:
		}

		// Pretty print JSON responses, if applicable:
		if opts.PrettyPrintResponses {
			// InfluxQL responses are in JSON and can be pretty-printed here.
			// Flux responses are just simple CSV.

			prefix := fmt.Sprintf("ID %d: ", q.ID)
			if json.Valid(resp.Body()) {
				var pretty bytes.Buffer
				err = json.Indent(&pretty, resp.Body(), prefix, "  ")
				if err != nil {
					return
				}

				_, err = fmt.Fprintf(os.Stderr, "%s%s\n", prefix, pretty)
				if err != nil {
					return
				}
			} else {
				_, err = fmt.Fprintf(os.Stderr, "%s%s\n", prefix, resp.Body())
				if err != nil {
					return
				}
			}
		}
	}

	return lag, err
}
<|endoftext|>


<commit_before>// Copyright 2015 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package docker

import (
	"bytes"
	"crypto/sha256"
	"encoding/base64"
	"fmt"
	"io"
	"regexp"
	"strings"

	"github.com/control-center/serviced/commons"
	dockerclient "github.com/fsouza/go-dockerclient"
	"github.com/zenoss/glog"
)

const (
	DefaultSocket   = "unix:///var/run/docker.sock"
	DefaultRegistry = "https://index.docker.io/v1/"
	Latest          = "latest"
	MaxLayers       = 127 - 2
)

// IsImageNotFound parses an err to determine whether the image is not found
func IsImageNotFound(err error) bool {
	if err != nil {
		if err == dockerclient.ErrNoSuchImage {
			return true
		}
		var checks = []*regexp.Regexp{
			regexp.MustCompile("Tag .* not found in repository .*"),
			regexp.MustCompile("Error: image .* not found"),
		}
		for _, check := range checks {
			if ok := check.MatchString(err.Error()); ok {
				return true
			}
		}
	}
	return false
}

// Docker is the docker client for the dfs
type Docker interface {
	FindImage(image string) (*dockerclient.Image, error)
	SaveImages(images []string, writer io.Writer) error
	LoadImage(reader io.Reader) error
	PushImage(image string) error
	PullImage(image string) error
	TagImage(oldImage, newImage string) error
	RemoveImage(image string) error
	FindContainer(ctr string) (*dockerclient.Container, error)
	CommitContainer(ctr, image string) (*dockerclient.Image, error)
	GetImageHash(image string) (string, error)
}

type DockerClient struct {
	dc *dockerclient.Client
}

func NewDockerClient() (*DockerClient, error) {
	dc, err := dockerclient.NewClient(DefaultSocket)
	if err != nil {
		return nil, err
	}
	return &DockerClient{dc}, nil
}

func (d *DockerClient) FindImage(image string) (*dockerclient.Image, error) {
	return d.dc.InspectImage(image)
}

func (d *DockerClient) SaveImages(images []string, writer io.Writer) error {
	opts := dockerclient.ExportImagesOptions{
		Names:        images,
		OutputStream: writer,
	}
	return d.dc.ExportImages(opts)
}

func (d *DockerClient) LoadImage(reader io.Reader) error {
	opts := dockerclient.LoadImageOptions{
		InputStream: reader,
	}
	return d.dc.LoadImage(opts)
}

func (d *DockerClient) PushImage(image string) error {
	imageID, err := commons.ParseImageID(image)
	if err != nil {
		return err
	}
	opts := dockerclient.PushImageOptions{
		Name:     imageID.BaseName(),
		Tag:      imageID.Tag,
		Registry: imageID.Registry(),
	}
	creds := d.fetchCreds(imageID.Registry())
	return d.dc.PushImage(opts, creds)
}

func (d *DockerClient) PullImage(image string) error {
	imageID, err := commons.ParseImageID(image)
	if err != nil {
		return err
	}
	opts := dockerclient.PullImageOptions{
		Repository: imageID.BaseName(),
		Registry:   imageID.Registry(),
		Tag:        imageID.Tag,
	}
	creds := d.fetchCreds(imageID.Registry())
	return d.dc.PullImage(opts, creds)
}

func (d *DockerClient) TagImage(oldImage, newImage string) error {
	newImageID, err := commons.ParseImageID(newImage)
	if err != nil {
		return err
	}
	opts := dockerclient.TagImageOptions{
		Repo:  newImageID.BaseName(),
		Tag:   newImageID.Tag,
		Force: true,
	}
	return d.dc.TagImage(oldImage, opts)
}

func (d *DockerClient) RemoveImage(image string) error {
	return d.dc.RemoveImage(image)
}

func (d *DockerClient) FindContainer(ctr string) (*dockerclient.Container, error) {
	return d.dc.InspectContainer(ctr)
}

func (d *DockerClient) CommitContainer(ctr string, image string) (*dockerclient.Image, error) {
	imageID, err := commons.ParseImageID(image)
	if err != nil {
		return nil, err
	}
	opts := dockerclient.CommitContainerOptions{
		Container:  ctr,
		Repository: imageID.BaseName(),
		Tag:        imageID.Tag,
	}
	return d.dc.CommitContainer(opts)
}

func (d *DockerClient) fetchCreds(registry string) (auth dockerclient.AuthConfiguration) {
	if registry = strings.TrimSpace(registry); registry == "" {
		registry = DefaultRegistry
	}
	auths, err := dockerclient.NewAuthConfigurationsFromDockerCfg()
	if err != nil {
		return
	}
	auth, ok := auths.Configs[registry]
	if ok {
		glog.V(1).Infof("Authorized as %s in registry %s", auth.Email, registry)
	}
	return
}

// Generates a unique hash of an image, based on its top layer (excluding the ID) and the IDs of all other layers
// NOTE:  consider http://localhost:5000/v2/jptest/manifests/latest
func (d *DockerClient) GetImageHash(image string) (string, error) {
	historyList, err := d.dc.ImageHistory(image)
	if err != nil {
		return "", err
	}

	var buffer bytes.Buffer
	for _, history := range historyList {
		imageDataString := fmt.Sprintf("%d-%s-%d\n", history.Created, history.CreatedBy, history.Size)
		buffer.WriteString(imageDataString)
	}

	h := sha256.New()
	sha := base64.URLEncoding.EncodeToString(h.Sum(buffer.Bytes()))

	return sha, nil
}
<commit_msg>Exclude layer size from image hash calculation<commit_after>// Copyright 2015 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package docker

import (
	"bytes"
	"crypto/sha256"
	"encoding/base64"
	"fmt"
	"io"
	"regexp"
	"strings"

	"github.com/control-center/serviced/commons"
	dockerclient "github.com/fsouza/go-dockerclient"
	"github.com/zenoss/glog"
)

const (
	DefaultSocket   = "unix:///var/run/docker.sock"
	DefaultRegistry = "https://index.docker.io/v1/"
	Latest          = "latest"
	MaxLayers       = 127 - 2
)

// IsImageNotFound parses an err to determine whether the image is not found
func IsImageNotFound(err error) bool {
	if err != nil {
		if err == dockerclient.ErrNoSuchImage {
			return true
		}
		var checks = []*regexp.Regexp{
			regexp.MustCompile("Tag .* not found in repository .*"),
			regexp.MustCompile("Error: image .* not found"),
		}
		for _, check := range checks {
			if ok := check.MatchString(err.Error()); ok {
				return true
			}
		}
	}
	return false
}

// Docker is the docker client for the dfs
type Docker interface {
	FindImage(image string) (*dockerclient.Image, error)
	SaveImages(images []string, writer io.Writer) error
	LoadImage(reader io.Reader) error
	PushImage(image string) error
	PullImage(image string) error
	TagImage(oldImage, newImage string) error
	RemoveImage(image string) error
	FindContainer(ctr string) (*dockerclient.Container, error)
	CommitContainer(ctr, image string) (*dockerclient.Image, error)
	GetImageHash(image string) (string, error)
}

type DockerClient struct {
	dc *dockerclient.Client
}

func NewDockerClient() (*DockerClient, error) {
	dc, err := dockerclient.NewClient(DefaultSocket)
	if err != nil {
		return nil, err
	}
	return &DockerClient{dc}, nil
}

func (d *DockerClient) FindImage(image string) (*dockerclient.Image, error) {
	return d.dc.InspectImage(image)
}

func (d *DockerClient) SaveImages(images []string, writer io.Writer) error {
	opts := dockerclient.ExportImagesOptions{
		Names:        images,
		OutputStream: writer,
	}
	return d.dc.ExportImages(opts)
}

func (d *DockerClient) LoadImage(reader io.Reader) error {
	opts := dockerclient.LoadImageOptions{
		InputStream: reader,
	}
	return d.dc.LoadImage(opts)
}

func (d *DockerClient) PushImage(image string) error {
	imageID, err := commons.ParseImageID(image)
	if err != nil {
		return err
	}
	opts := dockerclient.PushImageOptions{
		Name:     imageID.BaseName(),
		Tag:      imageID.Tag,
		Registry: imageID.Registry(),
	}
	creds := d.fetchCreds(imageID.Registry())
	return d.dc.PushImage(opts, creds)
}

func (d *DockerClient) PullImage(image string) error {
	imageID, err := commons.ParseImageID(image)
	if err != nil {
		return err
	}
	opts := dockerclient.PullImageOptions{
		Repository: imageID.BaseName(),
		Registry:   imageID.Registry(),
		Tag:        imageID.Tag,
	}
	creds := d.fetchCreds(imageID.Registry())
	return d.dc.PullImage(opts, creds)
}

func (d *DockerClient) TagImage(oldImage, newImage string) error {
	newImageID, err := commons.ParseImageID(newImage)
	if err != nil {
		return err
	}
	opts := dockerclient.TagImageOptions{
		Repo:  newImageID.BaseName(),
		Tag:   newImageID.Tag,
		Force: true,
	}
	return d.dc.TagImage(oldImage, opts)
}

func (d *DockerClient) RemoveImage(image string) error {
	return d.dc.RemoveImage(image)
}

func (d *DockerClient) FindContainer(ctr string) (*dockerclient.Container, error) {
	return d.dc.InspectContainer(ctr)
}

func (d *DockerClient) CommitContainer(ctr string, image string) (*dockerclient.Image, error) {
	imageID, err := commons.ParseImageID(image)
	if err != nil {
		return nil, err
	}
	opts := dockerclient.CommitContainerOptions{
		Container:  ctr,
		Repository: imageID.BaseName(),
		Tag:        imageID.Tag,
	}
	return d.dc.CommitContainer(opts)
}

func (d *DockerClient) fetchCreds(registry string) (auth dockerclient.AuthConfiguration) {
	if registry = strings.TrimSpace(registry); registry == "" {
		registry = DefaultRegistry
	}
	auths, err := dockerclient.NewAuthConfigurationsFromDockerCfg()
	if err != nil {
		return
	}
	auth, ok := auths.Configs[registry]
	if ok {
		glog.V(1).Infof("Authorized as %s in registry %s", auth.Email, registry)
	}
	return
}

// Generates a unique hash of an image, based on the creation time and command of each layer.
// CC-1750: the hash does NOT include the layer size because during HA testing we ran into
//          an edge case where 2 copies of the same image on different machines had different
//          layer sizes.
func (d *DockerClient) GetImageHash(image string) (string, error) {
	historyList, err := d.dc.ImageHistory(image)
	if err != nil {
		return "", err
	}

	var buffer bytes.Buffer
	for _, history := range historyList {
		imageDataString := fmt.Sprintf("%d-%s\n", history.Created, history.CreatedBy)
		buffer.WriteString(imageDataString)
	}

	h := sha256.New()
	sha := base64.URLEncoding.EncodeToString(h.Sum(buffer.Bytes()))

	return sha, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package pkiutil

import (
	"crypto/rsa"
	"crypto/x509"
	"fmt"
	"net"
	"os"
	"path/filepath"
	"time"

	"github.com/pkg/errors"

	"k8s.io/apimachinery/pkg/util/validation"
	certutil "k8s.io/client-go/util/cert"
	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
	"k8s.io/kubernetes/pkg/registry/core/service/ipallocator"
)

// NewCertificateAuthority creates new certificate and private key for the certificate authority
func NewCertificateAuthority(config *certutil.Config) (*x509.Certificate, *rsa.PrivateKey, error) {
	key, err := certutil.NewPrivateKey()
	if err != nil {
		return nil, nil, errors.Wrap(err, "unable to create private key")
	}

	cert, err := certutil.NewSelfSignedCACert(*config, key)
	if err != nil {
		return nil, nil, errors.Wrap(err, "unable to create self-signed certificate")
	}

	return cert, key, nil
}

// NewCertAndKey creates new certificate and key by passing the certificate authority certificate and key
func NewCertAndKey(caCert *x509.Certificate, caKey *rsa.PrivateKey, config *certutil.Config) (*x509.Certificate, *rsa.PrivateKey, error) {
	key, err := certutil.NewPrivateKey()
	if err != nil {
		return nil, nil, errors.Wrap(err, "unable to create private key")
	}

	cert, err := certutil.NewSignedCert(*config, key, caCert, caKey)
	if err != nil {
		return nil, nil, errors.Wrap(err, "unable to sign certificate")
	}

	return cert, key, nil
}

// HasServerAuth returns true if the given certificate is a ServerAuth
func HasServerAuth(cert *x509.Certificate) bool {
	for i := range cert.ExtKeyUsage {
		if cert.ExtKeyUsage[i] == x509.ExtKeyUsageServerAuth {
			return true
		}
	}
	return false
}

// WriteCertAndKey stores certificate and key at the specified location
func WriteCertAndKey(pkiPath string, name string, cert *x509.Certificate, key *rsa.PrivateKey) error {
	if err := WriteKey(pkiPath, name, key); err != nil {
		return err
	}

	return WriteCert(pkiPath, name, cert)
}

// WriteCert stores the given certificate at the given location
func WriteCert(pkiPath, name string, cert *x509.Certificate) error {
	if cert == nil {
		return errors.New("certificate cannot be nil when writing to file")
	}

	certificatePath := pathForCert(pkiPath, name)
	if err := certutil.WriteCert(certificatePath, certutil.EncodeCertPEM(cert)); err != nil {
		return errors.Wrapf(err, "unable to write certificate to file %s", certificatePath)
	}

	return nil
}

// WriteKey stores the given key at the given location
func WriteKey(pkiPath, name string, key *rsa.PrivateKey) error {
	if key == nil {
		return errors.New("private key cannot be nil when writing to file")
	}

	privateKeyPath := pathForKey(pkiPath, name)
	if err := certutil.WriteKey(privateKeyPath, certutil.EncodePrivateKeyPEM(key)); err != nil {
		return errors.Wrapf(err, "unable to write private key to file %s", privateKeyPath)
	}

	return nil
}

// WritePublicKey stores the given public key at the given location
func WritePublicKey(pkiPath, name string, key *rsa.PublicKey) error {
	if key == nil {
		return errors.New("public key cannot be nil when writing to file")
	}

	publicKeyBytes, err := certutil.EncodePublicKeyPEM(key)
	if err != nil {
		return err
	}
	publicKeyPath := pathForPublicKey(pkiPath, name)
	if err := certutil.WriteKey(publicKeyPath, publicKeyBytes); err != nil {
		return errors.Wrapf(err, "unable to write public key to file %s", publicKeyPath)
	}

	return nil
}

// CertOrKeyExist returns a boolean whether the cert or the key exists
func CertOrKeyExist(pkiPath, name string) bool {
	certificatePath, privateKeyPath := PathsForCertAndKey(pkiPath, name)

	_, certErr := os.Stat(certificatePath)
	_, keyErr := os.Stat(privateKeyPath)
	if os.IsNotExist(certErr) && os.IsNotExist(keyErr) {
		// The cert or the key did not exist
		return false
	}

	// Both files exist or one of them
	return true
}

// TryLoadCertAndKeyFromDisk tries to load a cert and a key from the disk and validates that they are valid
func TryLoadCertAndKeyFromDisk(pkiPath, name string) (*x509.Certificate, *rsa.PrivateKey, error) {
	cert, err := TryLoadCertFromDisk(pkiPath, name)
	if err != nil {
		return nil, nil, err
	}

	key, err := TryLoadKeyFromDisk(pkiPath, name)
	if err != nil {
		return nil, nil, err
	}

	return cert, key, nil
}

// TryLoadCertFromDisk tries to load the cert from the disk and validates that it is valid
func TryLoadCertFromDisk(pkiPath, name string) (*x509.Certificate, error) {
	certificatePath := pathForCert(pkiPath, name)

	certs, err := certutil.CertsFromFile(certificatePath)
	if err != nil {
		return nil, errors.Wrapf(err, "couldn't load the certificate file %s", certificatePath)
	}

	// We are only putting one certificate in the certificate pem file, so it's safe to just pick the first one
	// TODO: Support multiple certs here in order to be able to rotate certs
	cert := certs[0]

	// Check so that the certificate is valid now
	now := time.Now()
	if now.Before(cert.NotBefore) {
		return nil, errors.New("the certificate is not valid yet")
	}
	if now.After(cert.NotAfter) {
		return nil, errors.New("the certificate has expired")
	}

	return cert, nil
}

// TryLoadKeyFromDisk tries to load the key from the disk and validates that it is valid
func TryLoadKeyFromDisk(pkiPath, name string) (*rsa.PrivateKey, error) {
	privateKeyPath := pathForKey(pkiPath, name)

	// Parse the private key from a file
	privKey, err := certutil.PrivateKeyFromFile(privateKeyPath)
	if err != nil {
		return nil, errors.Wrapf(err, "couldn't load the private key file %s", privateKeyPath)
	}

	// Allow RSA format only
	var key *rsa.PrivateKey
	switch k := privKey.(type) {
	case *rsa.PrivateKey:
		key = k
	default:
		return nil, errors.Wrapf(err, "the private key file %s isn't in RSA format", privateKeyPath)
	}

	return key, nil
}

// TryLoadPrivatePublicKeyFromDisk tries to load the key from the disk and validates that it is valid
func TryLoadPrivatePublicKeyFromDisk(pkiPath, name string) (*rsa.PrivateKey, *rsa.PublicKey, error) {
	privateKeyPath := pathForKey(pkiPath, name)

	// Parse the private key from a file
	privKey, err := certutil.PrivateKeyFromFile(privateKeyPath)
	if err != nil {
		return nil, nil, errors.Wrapf(err, "couldn't load the private key file %s", privateKeyPath)
	}

	publicKeyPath := pathForPublicKey(pkiPath, name)

	// Parse the public key from a file
	pubKeys, err := certutil.PublicKeysFromFile(publicKeyPath)
	if err != nil {
		return nil, nil, errors.Wrapf(err, "couldn't load the public key file %s", publicKeyPath)
	}

	// Allow RSA format only
	k, ok := privKey.(*rsa.PrivateKey)
	if !ok {
		return nil, nil, errors.Wrapf(err, "the private key file %s isn't in RSA format", privateKeyPath)
	}

	p := pubKeys[0].(*rsa.PublicKey)

	return k, p, nil
}

// PathsForCertAndKey returns the paths for the certificate and key given the path and basename.
func PathsForCertAndKey(pkiPath, name string) (string, string) {
	return pathForCert(pkiPath, name), pathForKey(pkiPath, name)
}

func pathForCert(pkiPath, name string) string {
	return filepath.Join(pkiPath, fmt.Sprintf("%s.crt", name))
}

func pathForKey(pkiPath, name string) string {
	return filepath.Join(pkiPath, fmt.Sprintf("%s.key", name))
}

func pathForPublicKey(pkiPath, name string) string {
	return filepath.Join(pkiPath, fmt.Sprintf("%s.pub", name))
}

// GetAPIServerAltNames builds an AltNames object for to be used when generating apiserver certificate
func GetAPIServerAltNames(cfg *kubeadmapi.InitConfiguration) (*certutil.AltNames, error) {
	// advertise address
	advertiseAddress := net.ParseIP(cfg.APIEndpoint.AdvertiseAddress)
	if advertiseAddress == nil {
		return nil, errors.Errorf("error parsing APIEndpoint AdvertiseAddress %v: is not a valid textual representation of an IP address",
			cfg.APIEndpoint.AdvertiseAddress)
	}

	// internal IP address for the API server
	_, svcSubnet, err := net.ParseCIDR(cfg.Networking.ServiceSubnet)
	if err != nil {
		return nil, errors.Wrapf(err, "error parsing CIDR %q", cfg.Networking.ServiceSubnet)
	}

	internalAPIServerVirtualIP, err := ipallocator.GetIndexedIP(svcSubnet, 1)
	if err != nil {
		return nil, errors.Wrapf(err, "unable to get first IP address from the given CIDR (%s)", svcSubnet.String())
	}

	// create AltNames with defaults DNSNames/IPs
	altNames := &certutil.AltNames{
		DNSNames: []string{
			cfg.NodeRegistration.Name,
			"kubernetes",
			"kubernetes.default",
			"kubernetes.default.svc",
			fmt.Sprintf("kubernetes.default.svc.%s", cfg.Networking.DNSDomain),
		},
		IPs: []net.IP{
			internalAPIServerVirtualIP,
			advertiseAddress,
		},
	}

	// add cluster controlPlaneEndpoint if present (dns or ip)
	if len(cfg.ControlPlaneEndpoint) > 0 {
		if host, _, err := kubeadmutil.ParseHostPort(cfg.ControlPlaneEndpoint); err == nil {
			if ip := net.ParseIP(host); ip != nil {
				altNames.IPs = append(altNames.IPs, ip)
			} else {
				altNames.DNSNames = append(altNames.DNSNames, host)
			}
		} else {
			return nil, errors.Wrapf(err, "error parsing cluster controlPlaneEndpoint %q", cfg.ControlPlaneEndpoint)
		}
	}

	appendSANsToAltNames(altNames, cfg.APIServer.CertSANs, kubeadmconstants.APIServerCertName)

	return altNames, nil
}

// GetEtcdAltNames builds an AltNames object for generating the etcd server certificate.
// `advertise address` and localhost are included in the SAN since this is the interfaces the etcd static pod listens on.
// The user can override the listen address with `Etcd.ExtraArgs` and add SANs with `Etcd.ServerCertSANs`.
func GetEtcdAltNames(cfg *kubeadmapi.InitConfiguration) (*certutil.AltNames, error) {
	// advertise address
	advertiseAddress := net.ParseIP(cfg.APIEndpoint.AdvertiseAddress)
	if advertiseAddress == nil {
		return nil, errors.Errorf("error parsing APIEndpoint AdvertiseAddress %q: is not a valid textual representation of an IP address", cfg.APIEndpoint.AdvertiseAddress)
	}

	// create AltNames with defaults DNSNames/IPs
	altNames := &certutil.AltNames{
		DNSNames: []string{cfg.NodeRegistration.Name, "localhost"},
		IPs:      []net.IP{advertiseAddress, net.IPv4(127, 0, 0, 1), net.IPv6loopback},
	}

	if cfg.Etcd.Local != nil {
		appendSANsToAltNames(altNames, cfg.Etcd.Local.ServerCertSANs, kubeadmconstants.EtcdServerCertName)
	}

	return altNames, nil
}

// GetEtcdPeerAltNames builds an AltNames object for generating the etcd peer certificate.
// Hostname and `API.AdvertiseAddress` are included if the user chooses to promote the single node etcd cluster into a multi-node one (stacked etcd).
// The user can override the listen address with `Etcd.ExtraArgs` and add SANs with `Etcd.PeerCertSANs`.
func GetEtcdPeerAltNames(cfg *kubeadmapi.InitConfiguration) (*certutil.AltNames, error) {
	// advertise address
	advertiseAddress := net.ParseIP(cfg.APIEndpoint.AdvertiseAddress)
	if advertiseAddress == nil {
		return nil, errors.Errorf("error parsing APIEndpoint AdvertiseAddress %v: is not a valid textual representation of an IP address",
			cfg.APIEndpoint.AdvertiseAddress)
	}

	// create AltNames with defaults DNSNames/IPs
	altNames := &certutil.AltNames{
		DNSNames: []string{cfg.NodeRegistration.Name, "localhost"},
		IPs:      []net.IP{advertiseAddress, net.IPv4(127, 0, 0, 1), net.IPv6loopback},
	}

	if cfg.Etcd.Local != nil {
		appendSANsToAltNames(altNames, cfg.Etcd.Local.PeerCertSANs, kubeadmconstants.EtcdPeerCertName)
	}

	return altNames, nil
}

// appendSANsToAltNames parses SANs from as list of strings and adds them to altNames for use on a specific cert
// altNames is passed in with a pointer, and the struct is modified
// valid IP address strings are parsed and added to altNames.IPs as net.IP's
// RFC-1123 compliant DNS strings are added to altNames.DNSNames as strings
// certNames is used to print user facing warnings and should be the name of the cert the altNames will be used for
func appendSANsToAltNames(altNames *certutil.AltNames, SANs []string, certName string) {
	for _, altname := range SANs {
		if ip := net.ParseIP(altname); ip != nil {
			altNames.IPs = append(altNames.IPs, ip)
		} else if len(validation.IsDNS1123Subdomain(altname)) == 0 {
			altNames.DNSNames = append(altNames.DNSNames, altname)
		} else {
			fmt.Printf(
				"[certificates] WARNING: '%s' was not added to the '%s' SAN, because it is not a valid IP or RFC-1123 compliant DNS entry\n",
				altname,
				certName,
			)
		}
	}
}
<commit_msg>Fix error wrap on pki_helpers<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package pkiutil

import (
	"crypto/rsa"
	"crypto/x509"
	"fmt"
	"net"
	"os"
	"path/filepath"
	"time"

	"github.com/pkg/errors"

	"k8s.io/apimachinery/pkg/util/validation"
	certutil "k8s.io/client-go/util/cert"
	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
	"k8s.io/kubernetes/pkg/registry/core/service/ipallocator"
)

// NewCertificateAuthority creates new certificate and private key for the certificate authority
func NewCertificateAuthority(config *certutil.Config) (*x509.Certificate, *rsa.PrivateKey, error) {
	key, err := certutil.NewPrivateKey()
	if err != nil {
		return nil, nil, errors.Wrap(err, "unable to create private key")
	}

	cert, err := certutil.NewSelfSignedCACert(*config, key)
	if err != nil {
		return nil, nil, errors.Wrap(err, "unable to create self-signed certificate")
	}

	return cert, key, nil
}

// NewCertAndKey creates new certificate and key by passing the certificate authority certificate and key
func NewCertAndKey(caCert *x509.Certificate, caKey *rsa.PrivateKey, config *certutil.Config) (*x509.Certificate, *rsa.PrivateKey, error) {
	key, err := certutil.NewPrivateKey()
	if err != nil {
		return nil, nil, errors.Wrap(err, "unable to create private key")
	}

	cert, err := certutil.NewSignedCert(*config, key, caCert, caKey)
	if err != nil {
		return nil, nil, errors.Wrap(err, "unable to sign certificate")
	}

	return cert, key, nil
}

// HasServerAuth returns true if the given certificate is a ServerAuth
func HasServerAuth(cert *x509.Certificate) bool {
	for i := range cert.ExtKeyUsage {
		if cert.ExtKeyUsage[i] == x509.ExtKeyUsageServerAuth {
			return true
		}
	}
	return false
}

// WriteCertAndKey stores certificate and key at the specified location
func WriteCertAndKey(pkiPath string, name string, cert *x509.Certificate, key *rsa.PrivateKey) error {
	if err := WriteKey(pkiPath, name, key); err != nil {
		return err
	}

	return WriteCert(pkiPath, name, cert)
}

// WriteCert stores the given certificate at the given location
func WriteCert(pkiPath, name string, cert *x509.Certificate) error {
	if cert == nil {
		return errors.New("certificate cannot be nil when writing to file")
	}

	certificatePath := pathForCert(pkiPath, name)
	if err := certutil.WriteCert(certificatePath, certutil.EncodeCertPEM(cert)); err != nil {
		return errors.Wrapf(err, "unable to write certificate to file %s", certificatePath)
	}

	return nil
}

// WriteKey stores the given key at the given location
func WriteKey(pkiPath, name string, key *rsa.PrivateKey) error {
	if key == nil {
		return errors.New("private key cannot be nil when writing to file")
	}

	privateKeyPath := pathForKey(pkiPath, name)
	if err := certutil.WriteKey(privateKeyPath, certutil.EncodePrivateKeyPEM(key)); err != nil {
		return errors.Wrapf(err, "unable to write private key to file %s", privateKeyPath)
	}

	return nil
}

// WritePublicKey stores the given public key at the given location
func WritePublicKey(pkiPath, name string, key *rsa.PublicKey) error {
	if key == nil {
		return errors.New("public key cannot be nil when writing to file")
	}

	publicKeyBytes, err := certutil.EncodePublicKeyPEM(key)
	if err != nil {
		return err
	}
	publicKeyPath := pathForPublicKey(pkiPath, name)
	if err := certutil.WriteKey(publicKeyPath, publicKeyBytes); err != nil {
		return errors.Wrapf(err, "unable to write public key to file %s", publicKeyPath)
	}

	return nil
}

// CertOrKeyExist returns a boolean whether the cert or the key exists
func CertOrKeyExist(pkiPath, name string) bool {
	certificatePath, privateKeyPath := PathsForCertAndKey(pkiPath, name)

	_, certErr := os.Stat(certificatePath)
	_, keyErr := os.Stat(privateKeyPath)
	if os.IsNotExist(certErr) && os.IsNotExist(keyErr) {
		// The cert or the key did not exist
		return false
	}

	// Both files exist or one of them
	return true
}

// TryLoadCertAndKeyFromDisk tries to load a cert and a key from the disk and validates that they are valid
func TryLoadCertAndKeyFromDisk(pkiPath, name string) (*x509.Certificate, *rsa.PrivateKey, error) {
	cert, err := TryLoadCertFromDisk(pkiPath, name)
	if err != nil {
		return nil, nil, err
	}

	key, err := TryLoadKeyFromDisk(pkiPath, name)
	if err != nil {
		return nil, nil, err
	}

	return cert, key, nil
}

// TryLoadCertFromDisk tries to load the cert from the disk and validates that it is valid
func TryLoadCertFromDisk(pkiPath, name string) (*x509.Certificate, error) {
	certificatePath := pathForCert(pkiPath, name)

	certs, err := certutil.CertsFromFile(certificatePath)
	if err != nil {
		return nil, errors.Wrapf(err, "couldn't load the certificate file %s", certificatePath)
	}

	// We are only putting one certificate in the certificate pem file, so it's safe to just pick the first one
	// TODO: Support multiple certs here in order to be able to rotate certs
	cert := certs[0]

	// Check so that the certificate is valid now
	now := time.Now()
	if now.Before(cert.NotBefore) {
		return nil, errors.New("the certificate is not valid yet")
	}
	if now.After(cert.NotAfter) {
		return nil, errors.New("the certificate has expired")
	}

	return cert, nil
}

// TryLoadKeyFromDisk tries to load the key from the disk and validates that it is valid
func TryLoadKeyFromDisk(pkiPath, name string) (*rsa.PrivateKey, error) {
	privateKeyPath := pathForKey(pkiPath, name)

	// Parse the private key from a file
	privKey, err := certutil.PrivateKeyFromFile(privateKeyPath)
	if err != nil {
		return nil, errors.Wrapf(err, "couldn't load the private key file %s", privateKeyPath)
	}

	// Allow RSA format only
	var key *rsa.PrivateKey
	switch k := privKey.(type) {
	case *rsa.PrivateKey:
		key = k
	default:
		return nil, errors.Errorf("the private key file %s isn't in RSA format", privateKeyPath)
	}

	return key, nil
}

// TryLoadPrivatePublicKeyFromDisk tries to load the key from the disk and validates that it is valid
func TryLoadPrivatePublicKeyFromDisk(pkiPath, name string) (*rsa.PrivateKey, *rsa.PublicKey, error) {
	privateKeyPath := pathForKey(pkiPath, name)

	// Parse the private key from a file
	privKey, err := certutil.PrivateKeyFromFile(privateKeyPath)
	if err != nil {
		return nil, nil, errors.Wrapf(err, "couldn't load the private key file %s", privateKeyPath)
	}

	publicKeyPath := pathForPublicKey(pkiPath, name)

	// Parse the public key from a file
	pubKeys, err := certutil.PublicKeysFromFile(publicKeyPath)
	if err != nil {
		return nil, nil, errors.Wrapf(err, "couldn't load the public key file %s", publicKeyPath)
	}

	// Allow RSA format only
	k, ok := privKey.(*rsa.PrivateKey)
	if !ok {
		return nil, nil, errors.Errorf("the private key file %s isn't in RSA format", privateKeyPath)
	}

	p := pubKeys[0].(*rsa.PublicKey)

	return k, p, nil
}

// PathsForCertAndKey returns the paths for the certificate and key given the path and basename.
func PathsForCertAndKey(pkiPath, name string) (string, string) {
	return pathForCert(pkiPath, name), pathForKey(pkiPath, name)
}

func pathForCert(pkiPath, name string) string {
	return filepath.Join(pkiPath, fmt.Sprintf("%s.crt", name))
}

func pathForKey(pkiPath, name string) string {
	return filepath.Join(pkiPath, fmt.Sprintf("%s.key", name))
}

func pathForPublicKey(pkiPath, name string) string {
	return filepath.Join(pkiPath, fmt.Sprintf("%s.pub", name))
}

// GetAPIServerAltNames builds an AltNames object for to be used when generating apiserver certificate
func GetAPIServerAltNames(cfg *kubeadmapi.InitConfiguration) (*certutil.AltNames, error) {
	// advertise address
	advertiseAddress := net.ParseIP(cfg.APIEndpoint.AdvertiseAddress)
	if advertiseAddress == nil {
		return nil, errors.Errorf("error parsing APIEndpoint AdvertiseAddress %v: is not a valid textual representation of an IP address",
			cfg.APIEndpoint.AdvertiseAddress)
	}

	// internal IP address for the API server
	_, svcSubnet, err := net.ParseCIDR(cfg.Networking.ServiceSubnet)
	if err != nil {
		return nil, errors.Wrapf(err, "error parsing CIDR %q", cfg.Networking.ServiceSubnet)
	}

	internalAPIServerVirtualIP, err := ipallocator.GetIndexedIP(svcSubnet, 1)
	if err != nil {
		return nil, errors.Wrapf(err, "unable to get first IP address from the given CIDR (%s)", svcSubnet.String())
	}

	// create AltNames with defaults DNSNames/IPs
	altNames := &certutil.AltNames{
		DNSNames: []string{
			cfg.NodeRegistration.Name,
			"kubernetes",
			"kubernetes.default",
			"kubernetes.default.svc",
			fmt.Sprintf("kubernetes.default.svc.%s", cfg.Networking.DNSDomain),
		},
		IPs: []net.IP{
			internalAPIServerVirtualIP,
			advertiseAddress,
		},
	}

	// add cluster controlPlaneEndpoint if present (dns or ip)
	if len(cfg.ControlPlaneEndpoint) > 0 {
		if host, _, err := kubeadmutil.ParseHostPort(cfg.ControlPlaneEndpoint); err == nil {
			if ip := net.ParseIP(host); ip != nil {
				altNames.IPs = append(altNames.IPs, ip)
			} else {
				altNames.DNSNames = append(altNames.DNSNames, host)
			}
		} else {
			return nil, errors.Wrapf(err, "error parsing cluster controlPlaneEndpoint %q", cfg.ControlPlaneEndpoint)
		}
	}

	appendSANsToAltNames(altNames, cfg.APIServer.CertSANs, kubeadmconstants.APIServerCertName)

	return altNames, nil
}

// GetEtcdAltNames builds an AltNames object for generating the etcd server certificate.
// `advertise address` and localhost are included in the SAN since this is the interfaces the etcd static pod listens on.
// The user can override the listen address with `Etcd.ExtraArgs` and add SANs with `Etcd.ServerCertSANs`.
func GetEtcdAltNames(cfg *kubeadmapi.InitConfiguration) (*certutil.AltNames, error) {
	// advertise address
	advertiseAddress := net.ParseIP(cfg.APIEndpoint.AdvertiseAddress)
	if advertiseAddress == nil {
		return nil, errors.Errorf("error parsing APIEndpoint AdvertiseAddress %q: is not a valid textual representation of an IP address", cfg.APIEndpoint.AdvertiseAddress)
	}

	// create AltNames with defaults DNSNames/IPs
	altNames := &certutil.AltNames{
		DNSNames: []string{cfg.NodeRegistration.Name, "localhost"},
		IPs:      []net.IP{advertiseAddress, net.IPv4(127, 0, 0, 1), net.IPv6loopback},
	}

	if cfg.Etcd.Local != nil {
		appendSANsToAltNames(altNames, cfg.Etcd.Local.ServerCertSANs, kubeadmconstants.EtcdServerCertName)
	}

	return altNames, nil
}

// GetEtcdPeerAltNames builds an AltNames object for generating the etcd peer certificate.
// Hostname and `API.AdvertiseAddress` are included if the user chooses to promote the single node etcd cluster into a multi-node one (stacked etcd).
// The user can override the listen address with `Etcd.ExtraArgs` and add SANs with `Etcd.PeerCertSANs`.
func GetEtcdPeerAltNames(cfg *kubeadmapi.InitConfiguration) (*certutil.AltNames, error) {
	// advertise address
	advertiseAddress := net.ParseIP(cfg.APIEndpoint.AdvertiseAddress)
	if advertiseAddress == nil {
		return nil, errors.Errorf("error parsing APIEndpoint AdvertiseAddress %v: is not a valid textual representation of an IP address",
			cfg.APIEndpoint.AdvertiseAddress)
	}

	// create AltNames with defaults DNSNames/IPs
	altNames := &certutil.AltNames{
		DNSNames: []string{cfg.NodeRegistration.Name, "localhost"},
		IPs:      []net.IP{advertiseAddress, net.IPv4(127, 0, 0, 1), net.IPv6loopback},
	}

	if cfg.Etcd.Local != nil {
		appendSANsToAltNames(altNames, cfg.Etcd.Local.PeerCertSANs, kubeadmconstants.EtcdPeerCertName)
	}

	return altNames, nil
}

// appendSANsToAltNames parses SANs from as list of strings and adds them to altNames for use on a specific cert
// altNames is passed in with a pointer, and the struct is modified
// valid IP address strings are parsed and added to altNames.IPs as net.IP's
// RFC-1123 compliant DNS strings are added to altNames.DNSNames as strings
// certNames is used to print user facing warnings and should be the name of the cert the altNames will be used for
func appendSANsToAltNames(altNames *certutil.AltNames, SANs []string, certName string) {
	for _, altname := range SANs {
		if ip := net.ParseIP(altname); ip != nil {
			altNames.IPs = append(altNames.IPs, ip)
		} else if len(validation.IsDNS1123Subdomain(altname)) == 0 {
			altNames.DNSNames = append(altNames.DNSNames, altname)
		} else {
			fmt.Printf(
				"[certificates] WARNING: '%s' was not added to the '%s' SAN, because it is not a valid IP or RFC-1123 compliant DNS entry\n",
				altname,
				certName,
			)
		}
	}
}
<|endoftext|>


<commit_before>/*
http://www.apache.org/licenses/LICENSE-2.0.txt

Copyright 2015 Intel Corporation

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package scheduler

import (
	"errors"
	"fmt"
	"sync"

	"github.com/intelsdi-x/gomit"

	"github.com/intelsdi-x/snap/control/plugin"
	"github.com/intelsdi-x/snap/core"
	"github.com/intelsdi-x/snap/core/cdata"
	"github.com/intelsdi-x/snap/core/scheduler_event"
	"github.com/intelsdi-x/snap/scheduler/wmap"
)

// WorkflowState int type
type WorkflowState int

// Workflow state constants
const (
	WorkflowStopped WorkflowState = iota
	WorkflowStarted
)

// WorkflowStateLookup map and error vars
var (
	WorkflowStateLookup = map[WorkflowState]string{
		WorkflowStopped: "Stopped",
		WorkflowStarted: "Started",
	}

	ErrNullCollectNode        = errors.New("Missing collection node in workflow map")
	ErrNoMetricsInCollectNode = errors.New("Collection node has not metrics defined to collect")
)

// WmapToWorkflow attempts to convert a wmap.WorkflowMap to a schedulerWorkflow instance.
func wmapToWorkflow(wfMap *wmap.WorkflowMap) (*schedulerWorkflow, error) {
	wf := &schedulerWorkflow{}
	err := convertCollectionNode(wfMap.CollectNode, wf)
	if err != nil {
		return nil, err
	}
	// ***
	// TODO validate workflow makes sense here
	// - flows that don't end in publishers?
	// - duplicate child nodes anywhere?
	//***
	// Retain a copy of the original workflow map
	wf.workflowMap = wfMap
	return wf, nil
}

func convertCollectionNode(cnode *wmap.CollectWorkflowMapNode, wf *schedulerWorkflow) error {
	// Collection root
	// Validate collection node exists
	if cnode == nil {
		return ErrNullCollectNode
	}
	// Collection node has at least one metric in it
	if len(cnode.Metrics) < 1 {
		return ErrNoMetricsInCollectNode
	}
	// Get core.RequestedMetric metrics
	mts := cnode.GetMetrics()
	wf.metrics = make([]core.RequestedMetric, len(mts))
	for i, m := range mts {
		wf.metrics[i] = m
	}

	// Get our config data tree
	cdt, err := cnode.GetConfigTree()
	if err != nil {
		return err
	}
	wf.configTree = cdt
	// Iterate over first level process nodes
	pr, err := convertProcessNode(cnode.ProcessNodes)
	if err != nil {
		return err
	}
	wf.processNodes = pr
	// Iterate over first level publish nodes
	pu, err := convertPublishNode(cnode.PublishNodes)
	if err != nil {
		return err
	}
	wf.publishNodes = pu
	return nil
}

func convertProcessNode(pr []wmap.ProcessWorkflowMapNode) ([]*processNode, error) {
	prNodes := make([]*processNode, len(pr))
	for i, p := range pr {
		cdn, err := p.GetConfigNode()
		if err != nil {
			return nil, err
		}
		prC, err := convertProcessNode(p.ProcessNodes)
		if err != nil {
			return nil, err
		}
		puC, err := convertPublishNode(p.PublishNodes)
		if err != nil {
			return nil, err
		}

		// If version is not 1+ we use -1 to indicate we want
		// the plugin manager to select the highest version
		// available on plugin calls
		if p.Version < 1 {
			p.Version = -1
		}
		prNodes[i] = &processNode{
			name:         p.Name,
			version:      p.Version,
			config:       cdn,
			ProcessNodes: prC,
			PublishNodes: puC,
		}
	}
	return prNodes, nil
}

func convertPublishNode(pu []wmap.PublishWorkflowMapNode) ([]*publishNode, error) {
	puNodes := make([]*publishNode, len(pu))
	for i, p := range pu {
		cdn, err := p.GetConfigNode()
		if err != nil {
			return nil, err
		}
		// If version is not 1+ we use -1 to indicate we want
		// the plugin manager to select the highest version
		// available on plugin calls
		if p.Version < 1 {
			p.Version = -1
		}
		puNodes[i] = &publishNode{
			name:    p.Name,
			version: p.Version,
			config:  cdn,
		}
	}
	return puNodes, nil
}

type schedulerWorkflow struct {
	state WorkflowState
	// Metrics to collect
	metrics []core.RequestedMetric
	// The config data tree for collectors
	configTree   *cdata.ConfigDataTree
	processNodes []*processNode
	publishNodes []*publishNode
	// workflowMap used to generate this workflow
	workflowMap  *wmap.WorkflowMap
	eventEmitter gomit.Emitter
}

type processNode struct {
	name               string
	version            int
	config             *cdata.ConfigDataNode
	ProcessNodes       []*processNode
	PublishNodes       []*publishNode
	InboundContentType string
}

func (p *processNode) Name() string {
	return p.name
}

func (p *processNode) Version() int {
	return p.version
}

func (p *processNode) Config() *cdata.ConfigDataNode {
	return p.config
}

func (p *processNode) TypeName() string {
	return "processor"
}

type publishNode struct {
	name               string
	version            int
	config             *cdata.ConfigDataNode
	InboundContentType string
}

func (p *publishNode) Name() string {
	return p.name
}

func (p *publishNode) Version() int {
	return p.version
}

func (p *publishNode) Config() *cdata.ConfigDataNode {
	return p.config
}

func (p *publishNode) TypeName() string {
	return "publisher"
}

type wfContentTypes map[string]map[string][]string

// BindPluginContentTypes
func (s *schedulerWorkflow) BindPluginContentTypes(mm managesPluginContentTypes) error {
	bindPluginContentTypes(s.publishNodes, s.processNodes, mm, []string{plugin.SnapGOBContentType})
	return nil
}

func bindPluginContentTypes(pus []*publishNode, prs []*processNode, mm managesPluginContentTypes, lct []string) error {
	for _, pr := range prs {
		act, rct, err := mm.GetPluginContentTypes(pr.Name(), core.ProcessorPluginType, pr.Version())
		if err != nil {
			return err
		}

		for _, ac := range act {
			for _, lc := range lct {
				// if the return contenet type from the previous node matches
				// the accept content type for this node set it as the
				// inbound content type
				if ac == lc {
					pr.InboundContentType = ac
				}
			}
		}
		// if the inbound content type isn't set yet snap may be able to do
		// the conversion
		if pr.InboundContentType == "" {
			for _, ac := range act {
				switch ac {
				case plugin.SnapGOBContentType:
					pr.InboundContentType = plugin.SnapGOBContentType
				case plugin.SnapJSONContentType:
					pr.InboundContentType = plugin.SnapJSONContentType
				case plugin.SnapAllContentType:
					pr.InboundContentType = plugin.SnapGOBContentType
				}
			}
			// else we return an error
			if pr.InboundContentType == "" {
				return fmt.Errorf("Invalid workflow.  Plugin '%s' does not accept the snap content types or the types '%v' returned from the previous node.", pr.Name(), lct)
			}
		}
		//continue the walk down the nodes
		bindPluginContentTypes(pr.PublishNodes, pr.ProcessNodes, mm, rct)
	}
	for _, pu := range pus {
		act, _, err := mm.GetPluginContentTypes(pu.Name(), core.PublisherPluginType, pu.Version())
		if err != nil {
			return err
		}
		// if the inbound content type isn't set yet snap may be able to do
		// the conversion
		if pu.InboundContentType == "" {
			for _, ac := range act {
				switch ac {
				case plugin.SnapGOBContentType:
					pu.InboundContentType = plugin.SnapGOBContentType
				case plugin.SnapJSONContentType:
					pu.InboundContentType = plugin.SnapJSONContentType
				case plugin.SnapAllContentType:
					pu.InboundContentType = plugin.SnapGOBContentType
				}
			}
			// else we return an error
			if pu.InboundContentType == "" {
				return fmt.Errorf("Invalid workflow.  Plugin '%s' does not accept the snap content types or the types '%v' returned from the previous node.", pu.Name(), lct)
			}
		}
	}
	return nil
}

// Start starts a workflow
func (s *schedulerWorkflow) Start(t *task) {
	s.state = WorkflowStarted
	j := newCollectorJob(s.metrics, t.deadlineDuration, t.metricsManager, t.workflow.configTree, t.id)

	// dispatch 'collect' job to be worked
	// Block until the job has been either run or skipped.
	errors := t.manager.Work(j).Promise().Await()

	if len(errors) != 0 {
		t.RecordFailure(j.Errors())
		event := new(scheduler_event.MetricCollectionFailedEvent)
		event.TaskID = t.id
		event.Errors = errors
		defer s.eventEmitter.Emit(event)
		return
	}

	// Send event
	event := new(scheduler_event.MetricCollectedEvent)
	event.TaskID = t.id
	event.Metrics = j.(*collectorJob).metrics
	defer s.eventEmitter.Emit(event)

	// walk through the tree and dispatch work
	workJobs(s.processNodes, s.publishNodes, t, j)
}

func (s *schedulerWorkflow) State() WorkflowState {
	return s.state
}

func (s *schedulerWorkflow) StateString() string {
	return WorkflowStateLookup[s.state]
}

// workJobs takes a slice of proccess and publish nodes and submits jobs for each for a task.
// It then iterates down any process nodes to submit their child node jobs for the task
func workJobs(prs []*processNode, pus []*publishNode, t *task, pj job) {
	// Create waitgroup to block until all jobs are submitted
	wg := &sync.WaitGroup{}
	// range over the process jobs and call submitProcessJob
	for _, pr := range prs {
		// increment the wait group (before starting goroutine to prevent a race condition)
		wg.Add(1)
		// Start goroutine to submit the process job
		go submitProcessJob(pj, t, wg, pr)
	}
	// range over the publish jobs and call submitPublishJob
	for _, pu := range pus {
		// increment the wait group (before starting goroutine to prevent a race condition)
		wg.Add(1)
		// Start goroutine to submit the process job
		go submitPublishJob(pj, t, wg, pu)
	}
	// Wait until all job submisson goroutines are done
	wg.Wait()
}

func submitProcessJob(pj job, t *task, wg *sync.WaitGroup, pr *processNode) {
	// Decrement the waitgroup
	defer wg.Done()
	// Create a new process job
	j := newProcessJob(pj, pr.Name(), pr.Version(), pr.InboundContentType, pr.config.Table(), t.metricsManager, t.id)
	// Submit the job against the task.managesWork
	errors := t.manager.Work(j).Promise().Await()
	// Check for errors and update the task
	if len(errors) != 0 {
		// Record the failures in the task
		// note: this function is thread safe against t
		t.RecordFailure(errors)
		return
	}
	// Iterate into any child process or publish nodes
	workJobs(pr.ProcessNodes, pr.PublishNodes, t, j)
}

func submitPublishJob(pj job, t *task, wg *sync.WaitGroup, pu *publishNode) {
	// Decrement the waitgroup
	defer wg.Done()
	// Create a new process job
	j := newPublishJob(pj, pu.Name(), pu.Version(), pu.InboundContentType, pu.config.Table(), t.metricsManager, t.id)
	// Submit the job against the task.managesWork
	errors := t.manager.Work(j).Promise().Await()
	// Check for errors and update the task
	if len(errors) != 0 {
		// Record the failures in the task
		// note: this function is thread safe against t
		t.RecordFailure(errors)
		return
	}
	// Publish nodes cannot contain child nodes (publish is a terminal node)
	// so unlike process nodes there is not a call to workJobs here for child nodes.
}
<commit_msg>Adds debug logging to scheduler.workflow.<commit_after>/*
http://www.apache.org/licenses/LICENSE-2.0.txt

Copyright 2015 Intel Corporation

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package scheduler

import (
	"errors"
	"fmt"
	"sync"

	log "github.com/Sirupsen/logrus"
	"github.com/intelsdi-x/gomit"

	"github.com/intelsdi-x/snap/control/plugin"
	"github.com/intelsdi-x/snap/core"
	"github.com/intelsdi-x/snap/core/cdata"
	"github.com/intelsdi-x/snap/core/scheduler_event"
	"github.com/intelsdi-x/snap/scheduler/wmap"
)

// WorkflowState int type
type WorkflowState int

// Workflow state constants
const (
	WorkflowStopped WorkflowState = iota
	WorkflowStarted
)

// WorkflowStateLookup map and error vars
var (
	workflowLogger = schedulerLogger.WithField("_module", "scheduler-workflow")

	WorkflowStateLookup = map[WorkflowState]string{
		WorkflowStopped: "Stopped",
		WorkflowStarted: "Started",
	}

	ErrNullCollectNode        = errors.New("Missing collection node in workflow map")
	ErrNoMetricsInCollectNode = errors.New("Collection node has not metrics defined to collect")
)

// WmapToWorkflow attempts to convert a wmap.WorkflowMap to a schedulerWorkflow instance.
func wmapToWorkflow(wfMap *wmap.WorkflowMap) (*schedulerWorkflow, error) {
	wf := &schedulerWorkflow{}
	err := convertCollectionNode(wfMap.CollectNode, wf)
	if err != nil {
		return nil, err
	}
	// ***
	// TODO validate workflow makes sense here
	// - flows that don't end in publishers?
	// - duplicate child nodes anywhere?
	//***
	// Retain a copy of the original workflow map
	wf.workflowMap = wfMap
	return wf, nil
}

func convertCollectionNode(cnode *wmap.CollectWorkflowMapNode, wf *schedulerWorkflow) error {
	// Collection root
	// Validate collection node exists
	if cnode == nil {
		return ErrNullCollectNode
	}
	// Collection node has at least one metric in it
	if len(cnode.Metrics) < 1 {
		return ErrNoMetricsInCollectNode
	}
	// Get core.RequestedMetric metrics
	mts := cnode.GetMetrics()
	wf.metrics = make([]core.RequestedMetric, len(mts))
	for i, m := range mts {
		wf.metrics[i] = m
	}

	// Get our config data tree
	cdt, err := cnode.GetConfigTree()
	if err != nil {
		return err
	}
	wf.configTree = cdt
	// Iterate over first level process nodes
	pr, err := convertProcessNode(cnode.ProcessNodes)
	if err != nil {
		return err
	}
	wf.processNodes = pr
	// Iterate over first level publish nodes
	pu, err := convertPublishNode(cnode.PublishNodes)
	if err != nil {
		return err
	}
	wf.publishNodes = pu
	return nil
}

func convertProcessNode(pr []wmap.ProcessWorkflowMapNode) ([]*processNode, error) {
	prNodes := make([]*processNode, len(pr))
	for i, p := range pr {
		cdn, err := p.GetConfigNode()
		if err != nil {
			return nil, err
		}
		prC, err := convertProcessNode(p.ProcessNodes)
		if err != nil {
			return nil, err
		}
		puC, err := convertPublishNode(p.PublishNodes)
		if err != nil {
			return nil, err
		}

		// If version is not 1+ we use -1 to indicate we want
		// the plugin manager to select the highest version
		// available on plugin calls
		if p.Version < 1 {
			p.Version = -1
		}
		prNodes[i] = &processNode{
			name:         p.Name,
			version:      p.Version,
			config:       cdn,
			ProcessNodes: prC,
			PublishNodes: puC,
		}
	}
	return prNodes, nil
}

func convertPublishNode(pu []wmap.PublishWorkflowMapNode) ([]*publishNode, error) {
	puNodes := make([]*publishNode, len(pu))
	for i, p := range pu {
		cdn, err := p.GetConfigNode()
		if err != nil {
			return nil, err
		}
		// If version is not 1+ we use -1 to indicate we want
		// the plugin manager to select the highest version
		// available on plugin calls
		if p.Version < 1 {
			p.Version = -1
		}
		puNodes[i] = &publishNode{
			name:    p.Name,
			version: p.Version,
			config:  cdn,
		}
	}
	return puNodes, nil
}

type schedulerWorkflow struct {
	state WorkflowState
	// Metrics to collect
	metrics []core.RequestedMetric
	// The config data tree for collectors
	configTree   *cdata.ConfigDataTree
	processNodes []*processNode
	publishNodes []*publishNode
	// workflowMap used to generate this workflow
	workflowMap  *wmap.WorkflowMap
	eventEmitter gomit.Emitter
}

type processNode struct {
	name               string
	version            int
	config             *cdata.ConfigDataNode
	ProcessNodes       []*processNode
	PublishNodes       []*publishNode
	InboundContentType string
}

func (p *processNode) Name() string {
	return p.name
}

func (p *processNode) Version() int {
	return p.version
}

func (p *processNode) Config() *cdata.ConfigDataNode {
	return p.config
}

func (p *processNode) TypeName() string {
	return "processor"
}

type publishNode struct {
	name               string
	version            int
	config             *cdata.ConfigDataNode
	InboundContentType string
}

func (p *publishNode) Name() string {
	return p.name
}

func (p *publishNode) Version() int {
	return p.version
}

func (p *publishNode) Config() *cdata.ConfigDataNode {
	return p.config
}

func (p *publishNode) TypeName() string {
	return "publisher"
}

type wfContentTypes map[string]map[string][]string

// BindPluginContentTypes
func (s *schedulerWorkflow) BindPluginContentTypes(mm managesPluginContentTypes) error {
	bindPluginContentTypes(s.publishNodes, s.processNodes, mm, []string{plugin.SnapGOBContentType})
	return nil
}

func bindPluginContentTypes(pus []*publishNode, prs []*processNode, mm managesPluginContentTypes, lct []string) error {
	for _, pr := range prs {
		act, rct, err := mm.GetPluginContentTypes(pr.Name(), core.ProcessorPluginType, pr.Version())
		if err != nil {
			return err
		}

		for _, ac := range act {
			for _, lc := range lct {
				// if the return contenet type from the previous node matches
				// the accept content type for this node set it as the
				// inbound content type
				if ac == lc {
					pr.InboundContentType = ac
				}
			}
		}
		// if the inbound content type isn't set yet snap may be able to do
		// the conversion
		if pr.InboundContentType == "" {
			for _, ac := range act {
				switch ac {
				case plugin.SnapGOBContentType:
					pr.InboundContentType = plugin.SnapGOBContentType
				case plugin.SnapJSONContentType:
					pr.InboundContentType = plugin.SnapJSONContentType
				case plugin.SnapAllContentType:
					pr.InboundContentType = plugin.SnapGOBContentType
				}
			}
			// else we return an error
			if pr.InboundContentType == "" {
				return fmt.Errorf("Invalid workflow.  Plugin '%s' does not accept the snap content types or the types '%v' returned from the previous node.", pr.Name(), lct)
			}
		}
		//continue the walk down the nodes
		bindPluginContentTypes(pr.PublishNodes, pr.ProcessNodes, mm, rct)
	}
	for _, pu := range pus {
		act, _, err := mm.GetPluginContentTypes(pu.Name(), core.PublisherPluginType, pu.Version())
		if err != nil {
			return err
		}
		// if the inbound content type isn't set yet snap may be able to do
		// the conversion
		if pu.InboundContentType == "" {
			for _, ac := range act {
				switch ac {
				case plugin.SnapGOBContentType:
					pu.InboundContentType = plugin.SnapGOBContentType
				case plugin.SnapJSONContentType:
					pu.InboundContentType = plugin.SnapJSONContentType
				case plugin.SnapAllContentType:
					pu.InboundContentType = plugin.SnapGOBContentType
				}
			}
			// else we return an error
			if pu.InboundContentType == "" {
				return fmt.Errorf("Invalid workflow.  Plugin '%s' does not accept the snap content types or the types '%v' returned from the previous node.", pu.Name(), lct)
			}
		}
	}
	return nil
}

// Start starts a workflow
func (s *schedulerWorkflow) Start(t *task) {
	workflowLogger.WithFields(log.Fields{
		"_block":    "workflow-start",
		"task-id":   t.id,
		"task-name": t.name,
	}).Info(fmt.Sprintf("Starting workflow for task (%s\\%s)", t.id, t.name))
	s.state = WorkflowStarted
	j := newCollectorJob(s.metrics, t.deadlineDuration, t.metricsManager, t.workflow.configTree, t.id)

	// dispatch 'collect' job to be worked
	// Block until the job has been either run or skipped.
	errors := t.manager.Work(j).Promise().Await()

	if len(errors) != 0 {
		t.RecordFailure(j.Errors())
		event := new(scheduler_event.MetricCollectionFailedEvent)
		event.TaskID = t.id
		event.Errors = errors
		defer s.eventEmitter.Emit(event)
		return
	}

	// Send event
	event := new(scheduler_event.MetricCollectedEvent)
	event.TaskID = t.id
	event.Metrics = j.(*collectorJob).metrics
	defer s.eventEmitter.Emit(event)

	// walk through the tree and dispatch work
	workJobs(s.processNodes, s.publishNodes, t, j)
}

func (s *schedulerWorkflow) State() WorkflowState {
	return s.state
}

func (s *schedulerWorkflow) StateString() string {
	return WorkflowStateLookup[s.state]
}

// workJobs takes a slice of proccess and publish nodes and submits jobs for each for a task.
// It then iterates down any process nodes to submit their child node jobs for the task
func workJobs(prs []*processNode, pus []*publishNode, t *task, pj job) {
	// optimize for no jobs
	if len(prs) == 0 && len(pus) == 0 {
		return
	}
	// Create waitgroup to block until all jobs are submitted
	wg := &sync.WaitGroup{}
	workflowLogger.WithFields(log.Fields{
		"_block":              "work-jobs",
		"task-id":             t.id,
		"task-name":           t.name,
		"count-process-nodes": len(prs),
		"count-publish-nodes": len(pus),
		"parent-node-type":    pj.TypeString(),
	}).Debug("Batch submission of process and publish nodes")
	// range over the process jobs and call submitProcessJob
	for _, pr := range prs {
		// increment the wait group (before starting goroutine to prevent a race condition)
		wg.Add(1)
		// Start goroutine to submit the process job
		go submitProcessJob(pj, t, wg, pr)
	}
	// range over the publish jobs and call submitPublishJob
	for _, pu := range pus {
		// increment the wait group (before starting goroutine to prevent a race condition)
		wg.Add(1)
		// Start goroutine to submit the process job
		go submitPublishJob(pj, t, wg, pu)
	}
	// Wait until all job submisson goroutines are done
	wg.Wait()
	workflowLogger.WithFields(log.Fields{
		"_block":              "work-jobs",
		"task-id":             t.id,
		"task-name":           t.name,
		"count-process-nodes": len(prs),
		"count-publish-nodes": len(pus),
		"parent-node-type":    pj.TypeString(),
	}).Debug("Batch submission complete")
}

func submitProcessJob(pj job, t *task, wg *sync.WaitGroup, pr *processNode) {
	// Decrement the waitgroup
	defer wg.Done()
	// Create a new process job
	j := newProcessJob(pj, pr.Name(), pr.Version(), pr.InboundContentType, pr.config.Table(), t.metricsManager, t.id)
	workflowLogger.WithFields(log.Fields{
		"_block":           "submit-process-job",
		"task-id":          t.id,
		"task-name":        t.name,
		"process-name":     pr.Name(),
		"process-version":  pr.Version(),
		"parent-node-type": pj.TypeString(),
	}).Debug("Submitting process job")
	// Submit the job against the task.managesWork
	errors := t.manager.Work(j).Promise().Await()
	// Check for errors and update the task
	if len(errors) != 0 {
		// Record the failures in the task
		// note: this function is thread safe against t
		t.RecordFailure(errors)
		workflowLogger.WithFields(log.Fields{
			"_block":           "submit-process-job",
			"task-id":          t.id,
			"task-name":        t.name,
			"process-name":     pr.Name(),
			"process-version":  pr.Version(),
			"parent-node-type": pj.TypeString(),
		}).Warn("Process job failed")
		return
	}
	workflowLogger.WithFields(log.Fields{
		"_block":           "submit-process-job",
		"task-id":          t.id,
		"task-name":        t.name,
		"process-name":     pr.Name(),
		"process-version":  pr.Version(),
		"parent-node-type": pj.TypeString(),
	}).Debug("Process job completed")
	// Iterate into any child process or publish nodes
	workJobs(pr.ProcessNodes, pr.PublishNodes, t, j)
}

func submitPublishJob(pj job, t *task, wg *sync.WaitGroup, pu *publishNode) {
	// Decrement the waitgroup
	defer wg.Done()
	// Create a new process job
	j := newPublishJob(pj, pu.Name(), pu.Version(), pu.InboundContentType, pu.config.Table(), t.metricsManager, t.id)
	workflowLogger.WithFields(log.Fields{
		"_block":           "submit-publish-job",
		"task-id":          t.id,
		"task-name":        t.name,
		"publish-name":     pu.Name(),
		"publish-version":  pu.Version(),
		"parent-node-type": pj.TypeString(),
	}).Debug("Submitting publish job")
	// Submit the job against the task.managesWork
	errors := t.manager.Work(j).Promise().Await()
	// Check for errors and update the task
	if len(errors) != 0 {
		// Record the failures in the task
		// note: this function is thread safe against t
		t.RecordFailure(errors)
		workflowLogger.WithFields(log.Fields{
			"_block":           "submit-publish-job",
			"task-id":          t.id,
			"task-name":        t.name,
			"publish-name":     pu.Name(),
			"publish-version":  pu.Version(),
			"parent-node-type": pj.TypeString(),
		}).Warn("Publish job failed")
		return
	}
	workflowLogger.WithFields(log.Fields{
		"_block":           "submit-publish-job",
		"task-id":          t.id,
		"task-name":        t.name,
		"publish-name":     pu.Name(),
		"publish-version":  pu.Version(),
		"parent-node-type": pj.TypeString(),
	}).Debug("Publish job completed")
	// Publish nodes cannot contain child nodes (publish is a terminal node)
	// so unlike process nodes there is not a call to workJobs here for child nodes.
}
<|endoftext|>