<commit_before>package auditzip

import (
	"fmt"
	"io"
	"io/ioutil"

	upstreamzip "archive/zip"

	itchiozip "github.com/itchio/arkive/zip"

	humanize "github.com/dustin/go-humanize"

	"github.com/go-errors/errors"
	"github.com/itchio/butler/archive"
	"github.com/itchio/butler/comm"
	"github.com/itchio/butler/mansion"
	"github.com/itchio/wharf/eos"
	"github.com/itchio/wharf/state"
)

var args = struct {
	file     *string
	upstream *bool
}{}

func Register(ctx *mansion.Context) {
	cmd := ctx.App.Command("auditzip", "Audit a zip file for common errors")
	args.file = cmd.Arg("file", ".zip file to audit").Required().String()
	args.upstream = cmd.Flag("upstream", "Use upstream zip implementation (archive/zip)").Bool()
	ctx.Register(cmd, do)
}

func do(ctx *mansion.Context) {
	consumer := comm.NewStateConsumer()
	ctx.Must(Do(consumer, *args.file))
}

func Do(consumer *state.Consumer, file string) error {
	f, err := eos.Open(file)
	if err != nil {
		return errors.Wrap(err, 0)
	}
	defer f.Close()

	stats, err := f.Stat()
	if err != nil {
		return errors.Wrap(err, 0)
	}

	consumer.Opf("Auditing (%s)...", stats.Name())

	var impl ZipImpl
	if *args.upstream {
		consumer.Opf("Using upstream zip implementation")
		impl = &upstreamImpl{}
	} else {
		consumer.Opf("Using itchio/arkive zip implementation")
		impl = &itchioImpl{}
	}

	var foundErrors []string

	markError := func(path string, message string, args ...interface{}) {
		formatted := fmt.Sprintf(message, args...)
		fullMessage := fmt.Sprintf("(%s): %s", path, formatted)
		foundErrors = append(foundErrors, fullMessage)
	}

	paths := make(map[string]int)
	started := false

	err = impl.EachEntry(consumer, f, stats.Size(), func(index int, name string, uncompressedSize int64, rc io.ReadCloser, numEntries int) error {
		if !started {
			comm.StartProgress()
			started = true
		}
		path := archive.CleanFileName(name)

		comm.Progress(float64(index) / float64(numEntries))
		comm.ProgressLabel(path)

		if previousIndex, ok := paths[path]; ok {
			consumer.Warnf("Duplicate path (%s) at indices (%d) and (%d)", path, index, previousIndex)
		}
		paths[path] = index

		actualSize, err := io.Copy(ioutil.Discard, rc)
		if err != nil {
			markError("while extracting: %s", err.Error())
			return nil
		}

		if actualSize != uncompressedSize {
			err := fmt.Errorf("Dictionary says (%s) is %s (%d bytes), but it's actually %s (%d bytes)",
				path,
				humanize.IBytes(uint64(uncompressedSize)),
				uncompressedSize,
				humanize.IBytes(uint64(actualSize)),
				actualSize,
			)
			return errors.Wrap(err, 0)
		}
		return nil
	})
	comm.EndProgress()
	if err != nil {
		return errors.Wrap(err, 0)
	}

	if len(foundErrors) > 0 {
		consumer.Statf("Found %d errors, see above", len(foundErrors))
		return fmt.Errorf("Found %d errors in zip file", len(foundErrors))
	}

	consumer.Statf("Everything checks out!")

	return nil
}

// zip implementation types

type EachEntryFunc func(index int, name string, uncompressedSize int64, rc io.ReadCloser, numEntries int) error

type ZipImpl interface {
	EachEntry(consumer *state.Consumer, r io.ReaderAt, size int64, cb EachEntryFunc) error
}

// itchio zip impl

type itchioImpl struct{}

var _ ZipImpl = (*itchioImpl)(nil)

func (a *itchioImpl) EachEntry(consumer *state.Consumer, r io.ReaderAt, size int64, cb EachEntryFunc) error {
	zr, err := itchiozip.NewReader(r, size)
	if err != nil {
		return errors.Wrap(err, 0)
	}

	var compressedSize int64
	var uncompressedSize int64
	for _, entry := range zr.File {
		compressedSize += int64(entry.CompressedSize64)
		uncompressedSize += int64(entry.UncompressedSize64)
	}
	printExtras(consumer, size, compressedSize, uncompressedSize, zr.Comment)

	foundMethods := make(map[uint16]int)
	for _, entry := range zr.File {
		foundMethods[entry.Method] = foundMethods[entry.Method] + 1
	}
	printFoundMethods(consumer, foundMethods)

	numEntries := len(zr.File)
	for index, entry := range zr.File {
		rc, err := entry.Open()
		if err != nil {
			return errors.Wrap(err, 0)
		}

		err = cb(index, entry.Name, int64(entry.UncompressedSize64), rc, numEntries)
		rc.Close()
		if err != nil {
			return errors.Wrap(err, 0)
		}
	}

	return nil
}

// upstream zip impl

type upstreamImpl struct{}

var _ ZipImpl = (*upstreamImpl)(nil)

func (a *upstreamImpl) EachEntry(consumer *state.Consumer, r io.ReaderAt, size int64, cb EachEntryFunc) error {
	zr, err := upstreamzip.NewReader(r, size)
	if err != nil {
		return errors.Wrap(err, 0)
	}

	var compressedSize int64
	var uncompressedSize int64
	for _, entry := range zr.File {
		compressedSize += int64(entry.CompressedSize64)
		uncompressedSize += int64(entry.UncompressedSize64)
	}
	printExtras(consumer, size, compressedSize, uncompressedSize, zr.Comment)

	foundMethods := make(map[uint16]int)
	for _, entry := range zr.File {
		foundMethods[entry.Method] = foundMethods[entry.Method] + 1
	}
	printFoundMethods(consumer, foundMethods)

	numEntries := len(zr.File)
	for index, entry := range zr.File {
		rc, err := entry.Open()
		if err != nil {
			return errors.Wrap(err, 0)
		}

		err = cb(index, entry.Name, int64(entry.UncompressedSize64), rc, numEntries)
		rc.Close()
		if err != nil {
			return errors.Wrap(err, 0)
		}
	}

	return nil
}

// utils

func printExtras(consumer *state.Consumer, size int64, compressedSize int64, uncompressedSize int64, comment string) {
	consumer.Infof("Comment: (%s)", comment)
	consumer.Infof("Sizes: ")
	consumer.Infof(" → Archive size      : %s (%d bytes)", humanize.IBytes(uint64(size)), size)
	consumer.Infof(" → Sum (compressed)  : %s (%d bytes)", humanize.IBytes(uint64(compressedSize)), compressedSize)
	consumer.Infof(" → Sum (uncompressed): %s (%d bytes)", humanize.IBytes(uint64(uncompressedSize)), uncompressedSize)
	if compressedSize > uncompressedSize {
		consumer.Warnf("Compressed size is larger than uncompressed, that's suspicious.")
	}
}

func printFoundMethods(consumer *state.Consumer, foundMethods map[uint16]int) {
	consumer.Infof("Entries: ")
	for method, count := range foundMethods {
		switch method {
		case itchiozip.Store:
			consumer.Infof(" → %d STORE entries", count)
		case itchiozip.Deflate:
			consumer.Infof(" → %d DEFLATE entries", count)
		case itchiozip.LZMA:
			consumer.Infof(" → %d LZMA entries", count)
		default:
			consumer.Infof(" → %d entries with unknown method (%d)", count, method)
		}
	}
}
<commit_msg>Actually show error<commit_after>package auditzip

import (
	"fmt"
	"io"
	"io/ioutil"

	upstreamzip "archive/zip"

	itchiozip "github.com/itchio/arkive/zip"

	humanize "github.com/dustin/go-humanize"

	"github.com/go-errors/errors"
	"github.com/itchio/butler/archive"
	"github.com/itchio/butler/comm"
	"github.com/itchio/butler/mansion"
	"github.com/itchio/wharf/eos"
	"github.com/itchio/wharf/state"
)

var args = struct {
	file     *string
	upstream *bool
}{}

func Register(ctx *mansion.Context) {
	cmd := ctx.App.Command("auditzip", "Audit a zip file for common errors")
	args.file = cmd.Arg("file", ".zip file to audit").Required().String()
	args.upstream = cmd.Flag("upstream", "Use upstream zip implementation (archive/zip)").Bool()
	ctx.Register(cmd, do)
}

func do(ctx *mansion.Context) {
	consumer := comm.NewStateConsumer()
	ctx.Must(Do(consumer, *args.file))
}

func Do(consumer *state.Consumer, file string) error {
	f, err := eos.Open(file)
	if err != nil {
		return errors.Wrap(err, 0)
	}
	defer f.Close()

	stats, err := f.Stat()
	if err != nil {
		return errors.Wrap(err, 0)
	}

	consumer.Opf("Auditing (%s)...", stats.Name())

	var impl ZipImpl
	if *args.upstream {
		consumer.Opf("Using upstream zip implementation")
		impl = &upstreamImpl{}
	} else {
		consumer.Opf("Using itchio/arkive zip implementation")
		impl = &itchioImpl{}
	}

	var foundErrors []string

	markError := func(path string, message string, args ...interface{}) {
		formatted := fmt.Sprintf(message, args...)
		fullMessage := fmt.Sprintf("(%s): %s", path, formatted)
		consumer.Errorf(fullMessage)
		foundErrors = append(foundErrors, fullMessage)
	}

	paths := make(map[string]int)
	started := false

	err = impl.EachEntry(consumer, f, stats.Size(), func(index int, name string, uncompressedSize int64, rc io.ReadCloser, numEntries int) error {
		if !started {
			comm.StartProgress()
			started = true
		}
		path := archive.CleanFileName(name)

		comm.Progress(float64(index) / float64(numEntries))
		comm.ProgressLabel(path)

		if previousIndex, ok := paths[path]; ok {
			consumer.Warnf("Duplicate path (%s) at indices (%d) and (%d)", path, index, previousIndex)
		}
		paths[path] = index

		actualSize, err := io.Copy(ioutil.Discard, rc)
		if err != nil {
			markError("while extracting: %s", err.Error())
			return nil
		}

		if actualSize != uncompressedSize {
			err := fmt.Errorf("Dictionary says (%s) is %s (%d bytes), but it's actually %s (%d bytes)",
				path,
				humanize.IBytes(uint64(uncompressedSize)),
				uncompressedSize,
				humanize.IBytes(uint64(actualSize)),
				actualSize,
			)
			return errors.Wrap(err, 0)
		}
		return nil
	})
	comm.EndProgress()
	if err != nil {
		return errors.Wrap(err, 0)
	}

	if len(foundErrors) > 0 {
		consumer.Statf("Found %d errors, see above", len(foundErrors))
		return fmt.Errorf("Found %d errors in zip file", len(foundErrors))
	}

	consumer.Statf("Everything checks out!")

	return nil
}

// zip implementation types

type EachEntryFunc func(index int, name string, uncompressedSize int64, rc io.ReadCloser, numEntries int) error

type ZipImpl interface {
	EachEntry(consumer *state.Consumer, r io.ReaderAt, size int64, cb EachEntryFunc) error
}

// itchio zip impl

type itchioImpl struct{}

var _ ZipImpl = (*itchioImpl)(nil)

func (a *itchioImpl) EachEntry(consumer *state.Consumer, r io.ReaderAt, size int64, cb EachEntryFunc) error {
	zr, err := itchiozip.NewReader(r, size)
	if err != nil {
		return errors.Wrap(err, 0)
	}

	var compressedSize int64
	var uncompressedSize int64
	for _, entry := range zr.File {
		compressedSize += int64(entry.CompressedSize64)
		uncompressedSize += int64(entry.UncompressedSize64)
	}
	printExtras(consumer, size, compressedSize, uncompressedSize, zr.Comment)

	foundMethods := make(map[uint16]int)
	for _, entry := range zr.File {
		foundMethods[entry.Method] = foundMethods[entry.Method] + 1
	}
	printFoundMethods(consumer, foundMethods)

	numEntries := len(zr.File)
	for index, entry := range zr.File {
		rc, err := entry.Open()
		if err != nil {
			return errors.Wrap(err, 0)
		}

		err = cb(index, entry.Name, int64(entry.UncompressedSize64), rc, numEntries)
		rc.Close()
		if err != nil {
			return errors.Wrap(err, 0)
		}
	}

	return nil
}

// upstream zip impl

type upstreamImpl struct{}

var _ ZipImpl = (*upstreamImpl)(nil)

func (a *upstreamImpl) EachEntry(consumer *state.Consumer, r io.ReaderAt, size int64, cb EachEntryFunc) error {
	zr, err := upstreamzip.NewReader(r, size)
	if err != nil {
		return errors.Wrap(err, 0)
	}

	var compressedSize int64
	var uncompressedSize int64
	for _, entry := range zr.File {
		compressedSize += int64(entry.CompressedSize64)
		uncompressedSize += int64(entry.UncompressedSize64)
	}
	printExtras(consumer, size, compressedSize, uncompressedSize, zr.Comment)

	foundMethods := make(map[uint16]int)
	for _, entry := range zr.File {
		foundMethods[entry.Method] = foundMethods[entry.Method] + 1
	}
	printFoundMethods(consumer, foundMethods)

	numEntries := len(zr.File)
	for index, entry := range zr.File {
		rc, err := entry.Open()
		if err != nil {
			return errors.Wrap(err, 0)
		}

		err = cb(index, entry.Name, int64(entry.UncompressedSize64), rc, numEntries)
		rc.Close()
		if err != nil {
			return errors.Wrap(err, 0)
		}
	}

	return nil
}

// utils

func printExtras(consumer *state.Consumer, size int64, compressedSize int64, uncompressedSize int64, comment string) {
	consumer.Infof("Comment: (%s)", comment)
	consumer.Infof("Sizes: ")
	consumer.Infof(" → Archive size      : %s (%d bytes)", humanize.IBytes(uint64(size)), size)
	consumer.Infof(" → Sum (compressed)  : %s (%d bytes)", humanize.IBytes(uint64(compressedSize)), compressedSize)
	consumer.Infof(" → Sum (uncompressed): %s (%d bytes)", humanize.IBytes(uint64(uncompressedSize)), uncompressedSize)
	if compressedSize > uncompressedSize {
		consumer.Warnf("Compressed size is larger than uncompressed, that's suspicious.")
	}
}

func printFoundMethods(consumer *state.Consumer, foundMethods map[uint16]int) {
	consumer.Infof("Entries: ")
	for method, count := range foundMethods {
		switch method {
		case itchiozip.Store:
			consumer.Infof(" → %d STORE entries", count)
		case itchiozip.Deflate:
			consumer.Infof(" → %d DEFLATE entries", count)
		case itchiozip.LZMA:
			consumer.Infof(" → %d LZMA entries", count)
		default:
			consumer.Infof(" → %d entries with unknown method (%d)", count, method)
		}
	}
}
<|endoftext|>


<commit_before>package webapp

import (
	"github.com/skygeario/skygear-server/pkg/core/config"
	"github.com/skygeario/skygear-server/pkg/core/template"
)

const (
	TemplateItemTypeAuthUISignInHTML config.TemplateItemType = "auth_ui_sign_in.html"
	// nolint
	TemplateItemTypeAuthUISignInPasswordHTML config.TemplateItemType = "auth_ui_sign_in_password.html"
)

const defineHead = `
{{ define "HEAD" }}

<title>{{ .client_name }}</title>
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="stylesheet" href="{{ .x_static_asset_url_prefix }}/css/main.css">
{{ if .x_css }}

{{ end }}

{{ end }}
`

const defineHidden = `
{{ define "HIDDEN" }}
<input type="hidden" name="x_login_id_input_type" value="{{ .x_login_id_input_type }}">
{{ end }}
`

const defineLogo = `
{{ define "LOGO" }}
{{ if .logo_uri }}

{{ else }}

{{ end }}
{{ end }}
`

const defineError = `
{{ define "ERROR" }}
{{ if .x_error }}{{ if eq .x_error.reason "ValidationFailed" }}

{{ range .x_error.info.causes }}

{{ .message }}

{{ end }}

{{ else }}

{{ .x_error.message }}

{{ end }}{{ end }}
{{ end }}
`

const defineSkygearLogo = `
{{ define "SKYGEAR_LOGO" }}

{{ end }}
`

var defines = []string{
	defineHead,
	defineHidden,
	defineLogo,
	defineError,
	defineSkygearLogo,
}

var TemplateAuthUISignInHTML = template.Spec{
	Type:    TemplateItemTypeAuthUISignInHTML,
	IsHTML:  true,
	Defines: defines,
	Default: `<!DOCTYPE html>

{{ template "HEAD" . }}

		{{ template "LOGO" . }}

			<form class="authorize-idp-form" method="post">
				<input type="hidden" name="x_step" value="choose_idp">
				{{ range .x_idp_providers }}
				<button class="btn sso-btn {{ .type }}" type="submit" name="x_idp_id" value="{{ .id }}">
					{{- if eq .type "apple" -}}
					Sign in with Apple
					{{- end -}}
					{{- if eq .type "google" -}}
					Sign in with Google
					{{- end -}}
					{{- if eq .type "facebook" -}}
					Sign in with Facebook
					{{- end -}}
					{{- if eq .type "instagram" -}}
					Sign in with Instagram
					{{- end -}}
					{{- if eq .type "linkedin" -}}
					Sign in with LinkedIn
					{{- end -}}
					{{- if eq .type "azureadv2" -}}
					Sign in with Azure AD
					{{- end -}}
				</button>
				{{ end }}
			</form>

or

			{{ template "ERROR" . }}

			<form class="authorize-loginid-form" method="post">
				{{ template "HIDDEN" . }}

				{{ if .x_login_id_input_type }}{{ if and (eq .x_login_id_input_type "phone") .x_login_id_input_type_has_phone }}

					<select class="input select" name="x_calling_code">
						<option value="">Code</option>
						{{ range .x_calling_codes }}
						<option
							value="{{ . }}"
							{{ if $.x_calling_code }}{{ if eq $.x_calling_code . }}
							selected
							{{ end }}{{ end }}
							>
							+{{ . }}
						</option>
						{{ end }}
					</select>
					<input class="input text-input" type="tel" name="x_national_number" placeholder="Phone number" value="{{ .x_national_number }}">

				{{ end }}{{ end }}

				{{ if .x_login_id_input_type }}{{ if and (not (eq .x_login_id_input_type "phone")) .x_login_id_input_type_has_text }}
				<input class="input text-input" type="text" name="x_login_id" placeholder="Email or Username" value="{{ .x_login_id }}">
				{{ end }}{{ end }}

				{{ if .x_login_id_input_type }}{{ if and (eq .x_login_id_input_type "phone") .x_login_id_input_type_has_text }}
				Use an email or username instead
				{{ end }}{{ end }}
				{{ if .x_login_id_input_type }}{{ if and (not (eq .x_login_id_input_type "phone")) .x_login_id_input_type_has_phone }}
				Use a phone number instead
				{{ end }}{{ end }}

Don't have an account yet? Create one!

				Can't access your account?

				{{ if or .x_login_id_input_type_has_phone .x_login_id_input_type_has_text }}
				<button class="btn primary-btn" type="submit" name="x_step" value="submit_login_id">Next</button>
				{{ end }}
			</form>

		{{ template "SKYGEAR_LOGO" . }}

`,
}

var TemplateAuthUISignInPasswordHTML = template.Spec{
	Type:    TemplateItemTypeAuthUISignInPasswordHTML,
	IsHTML:  true,
	Defines: defines,
	Default: `<!DOCTYPE html>

{{ template "HEAD" . }}

{{ template "LOGO" . }}

<form class="enter-password-form" method="post">

{{ template "HIDDEN" . }}

	<button class="btn back-btn" onclick="window.history.back()" title="Back"></button>

	{{ if .x_calling_code }}
		+{{ .x_calling_code}} {{ .x_national_number }}
	{{ else }}
		{{ .x_login_id }}
	{{ end }}

Enter password

{{ template "ERROR" . }}

<input type="hidden" name="x_calling_code" value="{{ .x_calling_code }}">
<input type="hidden" name="x_national_number" value="{{ .x_national_number }}">
<input type="hidden" name="x_login_id" value="{{ .x_login_id }}">

<input id="password" class="input text-input" type="password" name="x_password" placeholder="Password" value="{{ .x_password }}">

Forgot Password?

<button class="btn primary-btn" type="submit" name="x_step" value="submit_password">Next</button>

</form>
{{ template "SKYGEAR_LOGO" . }}

`,
}
<commit_msg>Specialize login id required error<commit_after>package webapp

import (
	"github.com/skygeario/skygear-server/pkg/core/config"
	"github.com/skygeario/skygear-server/pkg/core/template"
)

const (
	TemplateItemTypeAuthUISignInHTML config.TemplateItemType = "auth_ui_sign_in.html"
	// nolint
	TemplateItemTypeAuthUISignInPasswordHTML config.TemplateItemType = "auth_ui_sign_in_password.html"
)

const defineHead = `
{{ define "HEAD" }}

<title>{{ .client_name }}</title>
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="stylesheet" href="{{ .x_static_asset_url_prefix }}/css/main.css">
{{ if .x_css }}

{{ end }}

{{ end }}
`

const defineHidden = `
{{ define "HIDDEN" }}
<input type="hidden" name="x_login_id_input_type" value="{{ .x_login_id_input_type }}">
{{ end }}
`

const defineLogo = `
{{ define "LOGO" }}
{{ if .logo_uri }}

{{ else }}

{{ end }}
{{ end }}
`

const defineError = `
{{ define "ERROR" }}
{{ if .x_error }}{{ if eq .x_error.reason "ValidationFailed" }}

{{ range .x_error.info.causes }}
{{ if and (eq .kind "Required") (eq .pointer "/x_login_id" ) }}

Email or Username is required

{{ else }}

{{ .message }}

{{ end }}
{{ end }}

{{ else }}

{{ .x_error.message }}

{{ end }}{{ end }}
{{ end }}
`

const defineSkygearLogo = `
{{ define "SKYGEAR_LOGO" }}

{{ end }}
`

var defines = []string{
	defineHead,
	defineHidden,
	defineLogo,
	defineError,
	defineSkygearLogo,
}

var TemplateAuthUISignInHTML = template.Spec{
	Type:    TemplateItemTypeAuthUISignInHTML,
	IsHTML:  true,
	Defines: defines,
	Default: `<!DOCTYPE html>

{{ template "HEAD" . }}

		{{ template "LOGO" . }}

			<form class="authorize-idp-form" method="post">
				<input type="hidden" name="x_step" value="choose_idp">
				{{ range .x_idp_providers }}
				<button class="btn sso-btn {{ .type }}" type="submit" name="x_idp_id" value="{{ .id }}">
					{{- if eq .type "apple" -}}
					Sign in with Apple
					{{- end -}}
					{{- if eq .type "google" -}}
					Sign in with Google
					{{- end -}}
					{{- if eq .type "facebook" -}}
					Sign in with Facebook
					{{- end -}}
					{{- if eq .type "instagram" -}}
					Sign in with Instagram
					{{- end -}}
					{{- if eq .type "linkedin" -}}
					Sign in with LinkedIn
					{{- end -}}
					{{- if eq .type "azureadv2" -}}
					Sign in with Azure AD
					{{- end -}}
				</button>
				{{ end }}
			</form>

or

			{{ template "ERROR" . }}

			<form class="authorize-loginid-form" method="post">
				{{ template "HIDDEN" . }}

				{{ if .x_login_id_input_type }}{{ if and (eq .x_login_id_input_type "phone") .x_login_id_input_type_has_phone }}

					<select class="input select" name="x_calling_code">
						<option value="">Code</option>
						{{ range .x_calling_codes }}
						<option
							value="{{ . }}"
							{{ if $.x_calling_code }}{{ if eq $.x_calling_code . }}
							selected
							{{ end }}{{ end }}
							>
							+{{ . }}
						</option>
						{{ end }}
					</select>
					<input class="input text-input" type="tel" name="x_national_number" placeholder="Phone number" value="{{ .x_national_number }}">

				{{ end }}{{ end }}

				{{ if .x_login_id_input_type }}{{ if and (not (eq .x_login_id_input_type "phone")) .x_login_id_input_type_has_text }}
				<input class="input text-input" type="text" name="x_login_id" placeholder="Email or Username" value="{{ .x_login_id }}">
				{{ end }}{{ end }}

				{{ if .x_login_id_input_type }}{{ if and (eq .x_login_id_input_type "phone") .x_login_id_input_type_has_text }}
				Use an email or username instead
				{{ end }}{{ end }}
				{{ if .x_login_id_input_type }}{{ if and (not (eq .x_login_id_input_type "phone")) .x_login_id_input_type_has_phone }}
				Use a phone number instead
				{{ end }}{{ end }}

Don't have an account yet? Create one!

				Can't access your account?

				{{ if or .x_login_id_input_type_has_phone .x_login_id_input_type_has_text }}
				<button class="btn primary-btn" type="submit" name="x_step" value="submit_login_id">Next</button>
				{{ end }}
			</form>

		{{ template "SKYGEAR_LOGO" . }}

`,
}

var TemplateAuthUISignInPasswordHTML = template.Spec{
	Type:    TemplateItemTypeAuthUISignInPasswordHTML,
	IsHTML:  true,
	Defines: defines,
	Default: `<!DOCTYPE html>

{{ template "HEAD" . }}

{{ template "LOGO" . }}

<form class="enter-password-form" method="post">

{{ template "HIDDEN" . }}

	<button class="btn back-btn" onclick="window.history.back()" title="Back"></button>

	{{ if .x_calling_code }}
		+{{ .x_calling_code}} {{ .x_national_number }}
	{{ else }}
		{{ .x_login_id }}
	{{ end }}

Enter password

{{ template "ERROR" . }}

<input type="hidden" name="x_calling_code" value="{{ .x_calling_code }}">
<input type="hidden" name="x_national_number" value="{{ .x_national_number }}">
<input type="hidden" name="x_login_id" value="{{ .x_login_id }}">

<input id="password" class="input text-input" type="password" name="x_password" placeholder="Password" value="{{ .x_password }}">

Forgot Password?

<button class="btn primary-btn" type="submit" name="x_step" value="submit_password">Next</button>

</form>
{{ template "SKYGEAR_LOGO" . }}

`,
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"

	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/http_server"
	"github.com/tedsuo/ifrit/sigmon"

	"github.com/concourse/baggageclaim/api"
	"github.com/concourse/baggageclaim/volume"
	"github.com/concourse/baggageclaim/volume/driver"
)

var listenAddress = flag.String(
	"listenAddress",
	"0.0.0.0",
	"address to listen on",
)

var listenPort = flag.Int(
	"listenPort",
	7788,
	"port for the server to listen on",
)

var volumeDir = flag.String(
	"volumeDir",
	"",
	"directory where volumes and metadata will be stored",
)

func main() {
	flag.Parse()
	if *volumeDir == "" {
		fmt.Fprintln(os.Stderr, "-volumeDir must be specified")
		os.Exit(1)
	}

	logger := lager.NewLogger("baggageclaim")
	sink := lager.NewReconfigurableSink(lager.NewWriterSink(os.Stdout, lager.DEBUG), lager.INFO)
	logger.RegisterSink(sink)

	listenAddr := fmt.Sprintf("%s:%d", *listenAddress, *listenPort)

	volumeRepo := volume.NewRepository(
		logger.Session("repository"),
		*volumeDir,
		&driver.NaiveDriver{},
	)

	apiHandler, err := api.NewHandler(
		logger.Session("api"),
		volumeRepo,
	)
	if err != nil {
		logger.Fatal("failed-to-create-handler", err)
	}

	memberGrouper := []grouper.Member{
		{"api", http_server.New(listenAddr, apiHandler)},
	}

	group := grouper.NewParallel(os.Interrupt, memberGrouper)
	running := ifrit.Invoke(sigmon.New(group))

	logger.Info("listening", lager.Data{
		"addr": listenAddr,
	})

	err = <-running.Wait()
	if err != nil {
		logger.Error("exited-with-failure", err)
		os.Exit(1)
	}
}
<commit_msg>allow driver to be configured by flag<commit_after>package main

import (
	"flag"
	"fmt"
	"os"

	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/http_server"
	"github.com/tedsuo/ifrit/sigmon"

	"github.com/concourse/baggageclaim/api"
	"github.com/concourse/baggageclaim/volume"
	"github.com/concourse/baggageclaim/volume/driver"
)

var listenAddress = flag.String(
	"listenAddress",
	"0.0.0.0",
	"address to listen on",
)

var listenPort = flag.Int(
	"listenPort",
	7788,
	"port for the server to listen on",
)

var volumeDir = flag.String(
	"volumeDir",
	"",
	"directory where volumes and metadata will be stored",
)

var driverType = flag.String(
	"driverType",
	"",
	"the backend driver to use for filesystems",
)

func main() {
	flag.Parse()
	if *volumeDir == "" {
		fmt.Fprintln(os.Stderr, "-volumeDir must be specified")
		os.Exit(1)
	}

	logger := lager.NewLogger("baggageclaim")
	sink := lager.NewReconfigurableSink(lager.NewWriterSink(os.Stdout, lager.DEBUG), lager.INFO)
	logger.RegisterSink(sink)

	listenAddr := fmt.Sprintf("%s:%d", *listenAddress, *listenPort)

	var volumeDriver volume.Driver

	if *driverType == "btrfs" {
		volumeDriver = driver.NewBtrFSDriver(logger.Session("driver"))
	} else {
		volumeDriver = &driver.NaiveDriver{}
	}

	volumeRepo := volume.NewRepository(
		logger.Session("repository"),
		*volumeDir,
		volumeDriver,
	)

	apiHandler, err := api.NewHandler(
		logger.Session("api"),
		volumeRepo,
	)
	if err != nil {
		logger.Fatal("failed-to-create-handler", err)
	}

	memberGrouper := []grouper.Member{
		{"api", http_server.New(listenAddr, apiHandler)},
	}

	group := grouper.NewParallel(os.Interrupt, memberGrouper)
	running := ifrit.Invoke(sigmon.New(group))

	logger.Info("listening", lager.Data{
		"addr": listenAddr,
	})

	err = <-running.Wait()
	if err != nil {
		logger.Error("exited-with-failure", err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>/**

Parallel Data Mover is scalable system to copy or migrate data between
various storage systems. It supports multliple types of sources and
destinations, including POSIX, S3, HPSS, etc.

Use cases include:
  * Data movement for Lustre HSM.
  * Offsite replication for DR
  * Lustre file-level replication
  * Storage rebalancing within a single tier
  * Migration between filesytems (e.g GPFS - > Lustre)

Initially the main focus is for HSM.
*/

package agent

import (
	"fmt"
	"sync"
	"time"

	"golang.org/x/net/context"

	"github.intel.com/hpdd/logging/alert"
	"github.intel.com/hpdd/logging/debug"
	"github.intel.com/hpdd/lustre/fs"
	"github.intel.com/hpdd/lustre/hsm"
	"github.intel.com/hpdd/policy/pkg/client"
)

type (
	// HsmAgent for a single filesytem and a collection of backends.
	HsmAgent struct {
		config    *Config
		client    *client.Client
		wg        sync.WaitGroup
		Endpoints *Endpoints
		mu        sync.Mutex // Protject the agent
		agent     hsm.Agent
		monitor   *PluginMonitor
	}

	// Transport for backend plugins
	Transport interface {
		Init(*Config, *HsmAgent) error
	}
)

// New accepts a config and returns a *HsmAgent
func New(cfg *Config) (*HsmAgent, error) {
	client, err := client.New(cfg.AgentMountpoint)
	if err != nil {
		return nil, err
	}

	ct := &HsmAgent{
		config:    cfg,
		client:    client,
		monitor:   NewMonitor(),
		Endpoints: NewEndpoints(),
	}

	return ct, nil
}

// Start backgrounds the agent and starts backend data movers
func (ct *HsmAgent) Start(ctx context.Context) error {
	for _, t := range transports {
		if err := t.Init(ct.config, ct); err != nil {
			return err
		}
	}

	if err := ct.initAgent(); err != nil {
		return err
	}

	for i := 0; i < ct.config.Processes; i++ {
		ct.addHandler(fmt.Sprintf("handler-%d", i))
	}

	ct.monitor.Start(ctx)
	for _, pluginConf := range ct.config.Plugins() {
		err := ct.monitor.StartPlugin(pluginConf)
		if err != nil {
			return err
		}
	}

	ct.wg.Wait()
	return nil
}

// Stop shuts down all backend data movers and kills the agent
func (ct *HsmAgent) Stop() {
	ct.mu.Lock()
	defer ct.mu.Unlock()
	if ct.agent != nil {
		ct.agent.Stop()
	}
}

// Root returns a fs.RootDir representing the Lustre filesystem root
func (ct *HsmAgent) Root() fs.RootDir {
	return ct.client.Root()
}

func (ct *HsmAgent) initAgent() (err error) {
	ct.mu.Lock()
	defer ct.mu.Unlock()
	ct.agent, err = hsm.Start(ct.client.Root())
	return
}

func (ct *HsmAgent) newAction(aih hsm.ActionHandle) *Action {
	return &Action{
		id:    NextActionID(),
		aih:   aih,
		start: time.Now(),
		agent: ct,
	}
}

func (ct *HsmAgent) handleActions(tag string) {
	ch := ct.agent.Actions()
	for ai := range ch {
		debug.Printf("%s: incoming: %s", tag, ai)
		aih, err := ai.Begin(0, false)
		if err != nil {
			alert.Warnf("%s: begin failed: %v: %s", tag, err, ai)
			continue
		}
		action := ct.newAction(aih)
		if e, ok := ct.Endpoints.Get(uint32(aih.ArchiveID())); ok {
			debug.Printf("%s: id:%d new %s %x %v", tag, action.id,
				action.aih.Action(),
				action.aih.Cookie(),
				action.aih.Fid())
			e.Send(action)
		} else {
			alert.Warnf("no handler for archive %d", aih.ArchiveID())
			action.Fail(-1)
		}
	}
}

func (ct *HsmAgent) addHandler(tag string) {
	ct.wg.Add(1)
	go func() {
		ct.handleActions(tag)
		ct.wg.Done()
	}()
}

var transports []Transport

// RegisterTransport registers the transport in the list of known transports
func RegisterTransport(t Transport) {
	transports = append(transports, t)
}
<commit_msg>Fail CANCEL requests immediately.<commit_after>/**

Parallel Data Mover is scalable system to copy or migrate data between
various storage systems. It supports multliple types of sources and
destinations, including POSIX, S3, HPSS, etc.

Use cases include:
  * Data movement for Lustre HSM.
  * Offsite replication for DR
  * Lustre file-level replication
  * Storage rebalancing within a single tier
  * Migration between filesytems (e.g GPFS - > Lustre)

Initially the main focus is for HSM.
*/

package agent

import (
	"fmt"
	"sync"
	"time"

	"golang.org/x/net/context"
	"golang.org/x/sys/unix"

	"github.intel.com/hpdd/logging/alert"
	"github.intel.com/hpdd/logging/debug"
	"github.intel.com/hpdd/lustre/fs"
	"github.intel.com/hpdd/lustre/hsm"
	"github.intel.com/hpdd/policy/pkg/client"
)

type (
	// HsmAgent for a single filesytem and a collection of backends.
	HsmAgent struct {
		config    *Config
		client    *client.Client
		wg        sync.WaitGroup
		Endpoints *Endpoints
		mu        sync.Mutex // Protject the agent
		agent     hsm.Agent
		monitor   *PluginMonitor
	}

	// Transport for backend plugins
	Transport interface {
		Init(*Config, *HsmAgent) error
	}
)

// New accepts a config and returns a *HsmAgent
func New(cfg *Config) (*HsmAgent, error) {
	client, err := client.New(cfg.AgentMountpoint)
	if err != nil {
		return nil, err
	}

	ct := &HsmAgent{
		config:    cfg,
		client:    client,
		monitor:   NewMonitor(),
		Endpoints: NewEndpoints(),
	}

	return ct, nil
}

// Start backgrounds the agent and starts backend data movers
func (ct *HsmAgent) Start(ctx context.Context) error {
	for _, t := range transports {
		if err := t.Init(ct.config, ct); err != nil {
			return err
		}
	}

	if err := ct.initAgent(); err != nil {
		return err
	}

	for i := 0; i < ct.config.Processes; i++ {
		ct.addHandler(fmt.Sprintf("handler-%d", i))
	}

	ct.monitor.Start(ctx)
	for _, pluginConf := range ct.config.Plugins() {
		err := ct.monitor.StartPlugin(pluginConf)
		if err != nil {
			return err
		}
	}

	ct.wg.Wait()
	return nil
}

// Stop shuts down all backend data movers and kills the agent
func (ct *HsmAgent) Stop() {
	ct.mu.Lock()
	defer ct.mu.Unlock()
	if ct.agent != nil {
		ct.agent.Stop()
	}
}

// Root returns a fs.RootDir representing the Lustre filesystem root
func (ct *HsmAgent) Root() fs.RootDir {
	return ct.client.Root()
}

func (ct *HsmAgent) initAgent() (err error) {
	ct.mu.Lock()
	defer ct.mu.Unlock()
	ct.agent, err = hsm.Start(ct.client.Root())
	return
}

func (ct *HsmAgent) newAction(aih hsm.ActionHandle) *Action {
	return &Action{
		id:    NextActionID(),
		aih:   aih,
		start: time.Now(),
		agent: ct,
	}
}

func (ct *HsmAgent) handleActions(tag string) {
	ch := ct.agent.Actions()
	for ai := range ch {
		debug.Printf("%s: incoming: %s", tag, ai)
		// AFAICT, this is how the copytool is expected to handle cancels.
		if ai.Action == llapi.HSM_CANCEL {
			ai.FailImmediately(unix.ENOSYS)
			// TODO: send out of band cancel message to the mover
			continue
		}
		aih, err := ai.Begin(0, false)
		if err != nil {
			alert.Warnf("%s: begin failed: %v: %s", tag, err, ai)
			continue
		}
		action := ct.newAction(aih)
		if e, ok := ct.Endpoints.Get(uint32(aih.ArchiveID())); ok {
			debug.Printf("%s: id:%d new %s %x %v", tag, action.id,
				action.aih.Action(),
				action.aih.Cookie(),
				action.aih.Fid())
			e.Send(action)
		} else {
			alert.Warnf("no handler for archive %d", aih.ArchiveID())
			action.Fail(-1)
		}
	}
}

func (ct *HsmAgent) addHandler(tag string) {
	ct.wg.Add(1)
	go func() {
		ct.handleActions(tag)
		ct.wg.Done()
	}()
}

var transports []Transport

// RegisterTransport registers the transport in the list of known transports
func RegisterTransport(t Transport) {
	transports = append(transports, t)
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"time"

	ethereum "github.com/ethereum/go-ethereum"
	"github.com/ethereum/go-ethereum/accounts"
	"github.com/ethereum/go-ethereum/accounts/keystore"
	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/core/types"
	"github.com/ethereum/go-ethereum/ethclient"
	"github.com/golang/glog"
	crypto "github.com/libp2p/go-libp2p-crypto"
	"github.com/livepeer/golp/core"
	"github.com/livepeer/golp/eth"
	"github.com/livepeer/golp/mediaserver"
	"github.com/livepeer/golp/net"
)

var ErrKeygen = errors.New("ErrKeygen")
var EthRpcTimeout = 10 * time.Second
var EthEventTimeout = 30 * time.Second
var EthMinedTxTimeout = 60 * time.Second

func main() {
	flag.Set("logtostderr", "true")

	port := flag.Int("p", 15000, "port")
	httpPort := flag.String("http", "8935", "http port")
	rtmpPort := flag.String("rtmp", "1935", "rtmp port")
	datadir := flag.String("datadir", "./data", "data directory")
	bootID := flag.String("bootID", "122074003534f659626514b1ceb29d750a07f595db6619724576088df8380e1b3d8e", "Bootstrap node ID")
	bootAddr := flag.String("bootAddr", "/ip4/127.0.0.1/tcp/15000", "Bootstrap node addr")
	bootnode := flag.Bool("bootnode", false, "Set to true if starting bootstrap node")
	transcoder := flag.Bool("transcoder", false, "Set to true to be a transcoder")
	newEthAccount := flag.Bool("newEthAccount", false, "Create an eth account")
	ethPassword := flag.String("ethPassword", "", "New Eth account password")
	gethipc := flag.String("gethipc", "", "Geth ipc file location")
	protocolAddr := flag.String("protocolAddr", "", "Protocol smart contract address")

	flag.Parse()

	if *port == 0 {
		glog.Fatalf("Please provide port")
	}
	if *httpPort == "" {
		glog.Fatalf("Please provide http port")
	}
	if *rtmpPort == "" {
		glog.Fatalf("Please provide rtmp port")
	}

	if _, err := os.Stat(*datadir); os.IsNotExist(err) {
		os.Mkdir(*datadir, 0755)
	}

	priv, pub, err := getLPKeys(*datadir)
	if err != nil {
		glog.Errorf("Error getting keys: %v", err)
		return
	}

	n, err := core.NewLivepeerNode(*port, priv, pub, nil)
	if err != nil {
		glog.Errorf("Error creating livepeer node: %v", err)
	}

	if *bootnode {
		glog.Infof("Setting up bootnode")
		//Setup boostrap node
		if err := n.VideoNetwork.SetupProtocol(); err != nil {
			glog.Errorf("Cannot set up protocol:%v", err)
			return
		}
	} else {
		if err := n.Start(*bootID, *bootAddr); err != nil {
			glog.Errorf("Cannot connect to bootstrap node: %v", err)
			return
		}
	}

	//Set up ethereum-related stuff
	if *gethipc != "" {
		var backend *ethclient.Client
		var acct accounts.Account

		if *newEthAccount {
			keyStore := keystore.NewKeyStore(filepath.Join(*datadir, "keystore"), keystore.StandardScryptN, keystore.StandardScryptP)
			acct, err = keyStore.NewAccount(*ethPassword)
			if err != nil {
				glog.Errorf("Error creating new eth account: %v", err)
				return
			}
		} else {
			acct, err = getEthAccount(*datadir)
			if err != nil {
				glog.Errorf("Error getting Eth account: %v", err)
				return
			}
		}
		glog.Infof("Connecting to geth @ %v", *gethipc)
		backend, err = ethclient.Dial(*gethipc)
		if err != nil {
			glog.Errorf("Failed to connect to Ethereum client: %v", err)
			return
		}

		client, err := eth.NewClient(acct, *ethPassword, *datadir, backend, common.HexToAddress(*protocolAddr), EthRpcTimeout, EthEventTimeout)
		if err != nil {
			glog.Errorf("Error creating Eth client: %v", err)
			return
		}
		n.Eth = client
		n.EthPassword = *ethPassword

		if *transcoder {
			logsSub, err := setupTranscoder(n, acct)

			if err != nil {
				glog.Errorf("Error subscribing to job event: %v", err)
			}
			defer logsSub.Unsubscribe()
			// defer close(logsChan)
		}
	} else {
		glog.Infof("***Livepeer is in off-chain mode***")
	}

	//Set up the media server
	glog.Infof("Setting up Media Server")
	s := mediaserver.NewLivepeerMediaServer(*rtmpPort, *httpPort, "", n)
	ec := make(chan error)
	msCtx, cancel := context.WithCancel(context.Background())
	go func() {
		ec <- s.StartMediaServer(msCtx)
	}()

	select {
	case err := <-ec:
		glog.Infof("Error from media server: %v", err)
		cancel()
		return
	case <-msCtx.Done():
		glog.Infof("MediaServer Done()")
		cancel()
		return
	}
	// if err := s.StartMediaServer(context.Background()); err != nil {
	// 	glog.Errorf("Failed to start LPMS: %v", err)
	// 	return
	// }

	// select {}
}

type LPKeyFile struct {
	Pub  string
	Priv string
}

func getLPKeys(datadir string) (crypto.PrivKey, crypto.PubKey, error) {
	gen := false
	var priv crypto.PrivKey
	var pub crypto.PubKey
	var privb []byte
	var pubb []byte
	var err error

	if datadir != "" {
		f, e := ioutil.ReadFile(path.Join(datadir, "keys.json"))
		if e != nil {
			gen = true
		}

		var keyf LPKeyFile
		if gen == false {
			if err := json.Unmarshal(f, &keyf); err != nil {
				gen = true
			}
		}

		if gen == false {
			privb, err = crypto.ConfigDecodeKey(keyf.Priv)
			if err != nil {
				gen = true
			}
		}

		if gen == false {
			pubb, err = crypto.ConfigDecodeKey(keyf.Pub)
			if err != nil {
				gen = true
			}
		}

		if gen == false {
			priv, err = crypto.UnmarshalPrivateKey(privb)
			if err != nil {
				gen = true
			}

		}

		if gen == false {
			pub, err = crypto.UnmarshalPublicKey(pubb)
			if err != nil {
				gen = true
			}
		}
	}

	if gen == true || pub == nil || priv == nil {
		glog.Errorf("Cannot file keys in data dir %v, creating new keys", datadir)
		priv, pub, err := crypto.GenerateKeyPair(crypto.RSA, 2048)
		if err != nil {
			glog.Errorf("Error generating keypair: %v", err)
			return nil, nil, ErrKeygen
		}

		privb, _ := priv.Bytes()
		pubb, _ := pub.Bytes()

		//Write keys to datadir
		if datadir != "" {
			kf := LPKeyFile{Priv: crypto.ConfigEncodeKey(privb), Pub: crypto.ConfigEncodeKey(pubb)}
			kfb, err := json.Marshal(kf)
			if err != nil {
				glog.Errorf("Error writing keyfile to datadir: %v", err)
			} else {
				if err := ioutil.WriteFile(path.Join(datadir, "keys.json"), kfb, 0644); err != nil {
					glog.Errorf("Error writing keyfile to datadir: %v", err)
				}
			}
		}

		return priv, pub, nil
	}

	return priv, pub, nil
}

func getEthAccount(datadir string) (accounts.Account, error) {
	keyStore := keystore.NewKeyStore(filepath.Join(datadir, "keystore"), keystore.StandardScryptN, keystore.StandardScryptP)
	accounts := keyStore.Accounts()
	if len(accounts) == 0 {
		glog.Errorf("Cannot find geth account, creating a new one")
		return accounts[0], fmt.Errorf("ErrGeth")
	}

	return accounts[0], nil
}

func setupTranscoder(n *core.LivepeerNode, acct accounts.Account) (ethereum.Subscription, error) {
	//Check if transcoder is active
	active, err := n.Eth.IsActiveTranscoder()
	if err != nil {
		glog.Errorf("Error getting transcoder state: %v", err)
	}

	if !active {
		glog.Infof("Transcoder %v is inactive", acct.Address.Hex())
	} else {
		s, err := n.Eth.TranscoderStake()
		if err != nil {
			glog.Errorf("Error getting transcoder stake: %v", err)
		}
		glog.Infof("Transcoder Active. Total Stake: %v", s)
	}

	rm := core.NewRewardManager(time.Second*5, n.Eth)
	go rm.Start()

	//Subscribe to when a job is assigned to us
	logsCh := make(chan types.Log)
	sub, err := n.Eth.SubscribeToJobEvent(context.Background(), logsCh)
	if err != nil {
		glog.Errorf("Error subscribing to job event: %v", err)
	}
	go func() error {
		select {
		case l := <-logsCh:
			tx, _, err := n.Eth.Backend().TransactionByHash(context.Background(), l.TxHash)
			if err != nil {
				glog.Errorf("Error getting transaction data: %v", err)
			}
			strmId, tData, err := eth.ParseJobTxData(tx.Data())
			if err != nil {
				glog.Errorf("Error parsing job tx data: %v", err)
			}

			jid, _, _, _, err := eth.GetInfoFromJobEvent(l, n.Eth)
			if err != nil {
				glog.Errorf("Error getting info from job event: %v", err)
			}

			//Create Transcode Config
			//TODO: profile should contain multiple video profiles.  Waiting for a protocol change.
			profile, ok := net.VideoProfileLookup[tData]
			if !ok {
				glog.Errorf("Cannot find video profile for job: %v", tData)
				return core.ErrTranscode
			}

			tProfiles := []net.VideoProfile{profile}
			config := net.TranscodeConfig{StrmID: strmId, Profiles: tProfiles, JobID: jid, PerformOnchainClaim: true}
			glog.Infof("Transcoder got job %v - strmID: %v, tData: %v, config: %v", tx.Hash(), strmId, tData, config)

			//Do The Transcoding
			cm := core.NewClaimManager(strmId, jid, tProfiles, n.Eth)
			strmIDs, err := n.Transcode(config, cm)
			if err != nil {
				glog.Errorf("Transcode Error: %v", err)
			}

			//Notify Broadcaster
			sid := core.StreamID(strmId)
			err = n.NotifyBroadcaster(sid.GetNodeID(), sid, map[core.StreamID]net.VideoProfile{strmIDs[0]: net.VideoProfileLookup[tData]})
			if err != nil {
				glog.Errorf("Notify Broadcaster Error: %v", err)
			}

			return nil

		}
	}()

	return sub, nil
}
<commit_msg>add stream command<commit_after>package main

import (
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"time"

	ethereum "github.com/ethereum/go-ethereum"
	"github.com/ethereum/go-ethereum/accounts"
	"github.com/ethereum/go-ethereum/accounts/keystore"
	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/core/types"
	"github.com/ethereum/go-ethereum/ethclient"
	"github.com/golang/glog"
	crypto "github.com/libp2p/go-libp2p-crypto"
	"github.com/livepeer/golp/core"
	"github.com/livepeer/golp/eth"
	"github.com/livepeer/golp/mediaserver"
	"github.com/livepeer/golp/net"
)

var ErrKeygen = errors.New("ErrKeygen")
var EthRpcTimeout = 10 * time.Second
var EthEventTimeout = 30 * time.Second
var EthMinedTxTimeout = 60 * time.Second

func main() {
	flag.Set("logtostderr", "true")

	//Stream Command
	streamCmd := flag.NewFlagSet("stream", flag.ExitOnError)
	streamHLS := streamCmd.Bool("hls", false, "Set to true to indicate hls streaming")
	streamID := streamCmd.String("id", "", "Stream ID")
	srPort := streamCmd.String("port", "8935", "Port for the video")

	if len(os.Args) > 1 {
		if os.Args[1] == "stream" {
			streamCmd.Parse(os.Args[2:])
			stream(*streamHLS, *srPort, *streamID)
			return
		}
	}

	port := flag.Int("p", 15000, "port")
	httpPort := flag.String("http", "8935", "http port")
	rtmpPort := flag.String("rtmp", "1935", "rtmp port")
	datadir := flag.String("datadir", "./data", "data directory")
	bootID := flag.String("bootID", "122074003534f659626514b1ceb29d750a07f595db6619724576088df8380e1b3d8e", "Bootstrap node ID")
	bootAddr := flag.String("bootAddr", "/ip4/127.0.0.1/tcp/15000", "Bootstrap node addr")
	bootnode := flag.Bool("bootnode", false, "Set to true if starting bootstrap node")
	transcoder := flag.Bool("transcoder", false, "Set to true to be a transcoder")
	newEthAccount := flag.Bool("newEthAccount", false, "Create an eth account")
	ethPassword := flag.String("ethPassword", "", "New Eth account password")
	gethipc := flag.String("gethipc", "", "Geth ipc file location")
	protocolAddr := flag.String("protocolAddr", "", "Protocol smart contract address")

	flag.Parse()

	if *port == 0 {
		glog.Fatalf("Please provide port")
	}
	if *httpPort == "" {
		glog.Fatalf("Please provide http port")
	}
	if *rtmpPort == "" {
		glog.Fatalf("Please provide rtmp port")
	}

	if _, err := os.Stat(*datadir); os.IsNotExist(err) {
		os.Mkdir(*datadir, 0755)
	}

	priv, pub, err := getLPKeys(*datadir)
	if err != nil {
		glog.Errorf("Error getting keys: %v", err)
		return
	}

	n, err := core.NewLivepeerNode(*port, priv, pub, nil)
	if err != nil {
		glog.Errorf("Error creating livepeer node: %v", err)
	}

	if *bootnode {
		glog.Infof("Setting up bootnode")
		//Setup boostrap node
		if err := n.VideoNetwork.SetupProtocol(); err != nil {
			glog.Errorf("Cannot set up protocol:%v", err)
			return
		}
	} else {
		if err := n.Start(*bootID, *bootAddr); err != nil {
			glog.Errorf("Cannot connect to bootstrap node: %v", err)
			return
		}
	}

	//Set up ethereum-related stuff
	if *gethipc != "" {
		var backend *ethclient.Client
		var acct accounts.Account

		if *newEthAccount {
			keyStore := keystore.NewKeyStore(filepath.Join(*datadir, "keystore"), keystore.StandardScryptN, keystore.StandardScryptP)
			acct, err = keyStore.NewAccount(*ethPassword)
			if err != nil {
				glog.Errorf("Error creating new eth account: %v", err)
				return
			}
		} else {
			acct, err = getEthAccount(*datadir)
			if err != nil {
				glog.Errorf("Error getting Eth account: %v", err)
				return
			}
		}
		glog.Infof("Connecting to geth @ %v", *gethipc)
		backend, err = ethclient.Dial(*gethipc)
		if err != nil {
			glog.Errorf("Failed to connect to Ethereum client: %v", err)
			return
		}

		client, err := eth.NewClient(acct, *ethPassword, *datadir, backend, common.HexToAddress(*protocolAddr), EthRpcTimeout, EthEventTimeout)
		if err != nil {
			glog.Errorf("Error creating Eth client: %v", err)
			return
		}
		n.Eth = client
		n.EthPassword = *ethPassword

		if *transcoder {
			logsSub, err := setupTranscoder(n, acct)

			if err != nil {
				glog.Errorf("Error subscribing to job event: %v", err)
			}
			defer logsSub.Unsubscribe()
			// defer close(logsChan)
		}
	} else {
		glog.Infof("***Livepeer is in off-chain mode***")
	}

	//Set up the media server
	glog.Infof("Setting up Media Server")
	s := mediaserver.NewLivepeerMediaServer(*rtmpPort, *httpPort, "", n)
	ec := make(chan error)
	msCtx, cancel := context.WithCancel(context.Background())
	go func() {
		ec <- s.StartMediaServer(msCtx)
	}()

	select {
	case err := <-ec:
		glog.Infof("Error from media server: %v", err)
		cancel()
		return
	case <-msCtx.Done():
		glog.Infof("MediaServer Done()")
		cancel()
		return
	}
	// if err := s.StartMediaServer(context.Background()); err != nil {
	// 	glog.Errorf("Failed to start LPMS: %v", err)
	// 	return
	// }

	// select {}
}

type LPKeyFile struct {
	Pub  string
	Priv string
}

func getLPKeys(datadir string) (crypto.PrivKey, crypto.PubKey, error) {
	gen := false
	var priv crypto.PrivKey
	var pub crypto.PubKey
	var privb []byte
	var pubb []byte
	var err error

	if datadir != "" {
		f, e := ioutil.ReadFile(path.Join(datadir, "keys.json"))
		if e != nil {
			gen = true
		}

		var keyf LPKeyFile
		if gen == false {
			if err := json.Unmarshal(f, &keyf); err != nil {
				gen = true
			}
		}

		if gen == false {
			privb, err = crypto.ConfigDecodeKey(keyf.Priv)
			if err != nil {
				gen = true
			}
		}

		if gen == false {
			pubb, err = crypto.ConfigDecodeKey(keyf.Pub)
			if err != nil {
				gen = true
			}
		}

		if gen == false {
			priv, err = crypto.UnmarshalPrivateKey(privb)
			if err != nil {
				gen = true
			}

		}

		if gen == false {
			pub, err = crypto.UnmarshalPublicKey(pubb)
			if err != nil {
				gen = true
			}
		}
	}

	if gen == true || pub == nil || priv == nil {
		glog.Errorf("Cannot file keys in data dir %v, creating new keys", datadir)
		priv, pub, err := crypto.GenerateKeyPair(crypto.RSA, 2048)
		if err != nil {
			glog.Errorf("Error generating keypair: %v", err)
			return nil, nil, ErrKeygen
		}

		privb, _ := priv.Bytes()
		pubb, _ := pub.Bytes()

		//Write keys to datadir
		if datadir != "" {
			kf := LPKeyFile{Priv: crypto.ConfigEncodeKey(privb), Pub: crypto.ConfigEncodeKey(pubb)}
			kfb, err := json.Marshal(kf)
			if err != nil {
				glog.Errorf("Error writing keyfile to datadir: %v", err)
			} else {
				if err := ioutil.WriteFile(path.Join(datadir, "keys.json"), kfb, 0644); err != nil {
					glog.Errorf("Error writing keyfile to datadir: %v", err)
				}
			}
		}

		return priv, pub, nil
	}

	return priv, pub, nil
}

func getEthAccount(datadir string) (accounts.Account, error) {
	keyStore := keystore.NewKeyStore(filepath.Join(datadir, "keystore"), keystore.StandardScryptN, keystore.StandardScryptP)
	accounts := keyStore.Accounts()
	if len(accounts) == 0 {
		glog.Errorf("Cannot find geth account, creating a new one")
		return accounts[0], fmt.Errorf("ErrGeth")
	}

	return accounts[0], nil
}

func setupTranscoder(n *core.LivepeerNode, acct accounts.Account) (ethereum.Subscription, error) {
	//Check if transcoder is active
	active, err := n.Eth.IsActiveTranscoder()
	if err != nil {
		glog.Errorf("Error getting transcoder state: %v", err)
	}

	if !active {
		glog.Infof("Transcoder %v is inactive", acct.Address.Hex())
	} else {
		s, err := n.Eth.TranscoderStake()
		if err != nil {
			glog.Errorf("Error getting transcoder stake: %v", err)
		}
		glog.Infof("Transcoder Active. Total Stake: %v", s)
	}

	rm := core.NewRewardManager(time.Second*5, n.Eth)
	go rm.Start()

	//Subscribe to when a job is assigned to us
	logsCh := make(chan types.Log)
	sub, err := n.Eth.SubscribeToJobEvent(context.Background(), logsCh)
	if err != nil {
		glog.Errorf("Error subscribing to job event: %v", err)
	}
	go func() error {
		select {
		case l := <-logsCh:
			tx, _, err := n.Eth.Backend().TransactionByHash(context.Background(), l.TxHash)
			if err != nil {
				glog.Errorf("Error getting transaction data: %v", err)
			}
			strmId, tData, err := eth.ParseJobTxData(tx.Data())
			if err != nil {
				glog.Errorf("Error parsing job tx data: %v", err)
			}

			jid, _, _, _, err := eth.GetInfoFromJobEvent(l, n.Eth)
			if err != nil {
				glog.Errorf("Error getting info from job event: %v", err)
			}

			//Create Transcode Config
			//TODO: profile should contain multiple video profiles.  Waiting for a protocol change.
			profile, ok := net.VideoProfileLookup[tData]
			if !ok {
				glog.Errorf("Cannot find video profile for job: %v", tData)
				return core.ErrTranscode
			}

			tProfiles := []net.VideoProfile{profile}
			config := net.TranscodeConfig{StrmID: strmId, Profiles: tProfiles, JobID: jid, PerformOnchainClaim: true}
			glog.Infof("Transcoder got job %v - strmID: %v, tData: %v, config: %v", tx.Hash(), strmId, tData, config)

			//Do The Transcoding
			cm := core.NewClaimManager(strmId, jid, tProfiles, n.Eth)
			strmIDs, err := n.Transcode(config, cm)
			if err != nil {
				glog.Errorf("Transcode Error: %v", err)
			}

			//Notify Broadcaster
			sid := core.StreamID(strmId)
			err = n.NotifyBroadcaster(sid.GetNodeID(), sid, map[core.StreamID]net.VideoProfile{strmIDs[0]: net.VideoProfileLookup[tData]})
			if err != nil {
				glog.Errorf("Notify Broadcaster Error: %v", err)
			}

			return nil

		}
	}()

	return sub, nil
}

func stream(hlsRequest bool, port string, streamID string) {
	var url string

	// Determine if you are streaming the HLS or RTMP version. If --hls is passed in, stream HLS
	if hlsRequest == true {
		url = fmt.Sprintf("http://localhost:%v/stream/%v.m3u8", port, streamID)
	} else {
		url = fmt.Sprintf("rtmp://localhost:%v/stream/%v", port, streamID)
	}

	cmd := exec.Command("ffplay", url)
	glog.Infof("url: %v", url)
	err := cmd.Start()
	if err != nil {
		fmt.Println("Couldn't start the stream")
		os.Exit(1)
	}
	fmt.Println("Now streaming")
	err = cmd.Wait()
	fmt.Println("Finished the stream")
}
<|endoftext|>


<commit_before>/*
Copyright 2019 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package decorator

import (
	"context"
	"knative.dev/pkg/resolver"

	"github.com/kelseyhightower/envconfig"
	"go.uber.org/zap"
	"k8s.io/client-go/tools/cache"
	"knative.dev/pkg/configmap"
	"knative.dev/pkg/controller"
	"knative.dev/pkg/logging"

	"github.com/google/knative-gcp/pkg/apis/messaging/v1alpha1"
	"github.com/google/knative-gcp/pkg/reconciler"

	serviceinformer "knative.dev/serving/pkg/client/injection/informers/serving/v1/service"

	decoratorinformer "github.com/google/knative-gcp/pkg/client/injection/informers/messaging/v1alpha1/decorator"
)

const (
	// controllerAgentName is the string used by this controller to identify
	// itself when creating events.
	controllerAgentName = "cloud-run-events-decorator-controller"
)

type envConfig struct {
	// Decorator is the image used to run the decorator. Required.
	Decorator string `envconfig:"DECORATOR_IMAGE" required:"true"`
}

// NewController initializes the controller and is called by the generated code
// Registers event handlers to enqueue events
func NewController(
	ctx context.Context,
	cmw configmap.Watcher,
) *controller.Impl {
	decoratorInformer := decoratorinformer.Get(ctx)
	serviceinformer := serviceinformer.Get(ctx)

	logger := logging.FromContext(ctx).Named(controllerAgentName)

	var env envConfig
	if err := envconfig.Process("", &env); err != nil {
		logger.Fatal("Failed to process env var", zap.Error(err))
	}

	c := &Reconciler{
		Base:            reconciler.NewBase(ctx, controllerAgentName, cmw),
		decoratorLister: decoratorInformer.Lister(),
		serviceLister:   serviceinformer.Lister(),
		decoratorImage:  env.Decorator,
	}

	impl := controller.NewImpl(c, c.Logger, ReconcilerName)

	c.Logger.Info("Setting up event handlers")
	decoratorInformer.Informer().AddEventHandler(controller.HandleAll(impl.Enqueue))

	serviceinformer.Informer().AddEventHandler(cache.FilteringResourceEventHandler{
		FilterFunc: controller.Filter(v1alpha1.SchemeGroupVersion.WithKind("Decorator")),
		Handler:    controller.HandleAll(impl.EnqueueControllerOf),
	})

	c.uriResolver = resolver.NewURIResolver(ctx, impl.EnqueueKey)

	return impl
}
<commit_msg>golang format tools (#382)<commit_after>/*
Copyright 2019 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package decorator

import (
	"context"

	"knative.dev/pkg/resolver"

	"github.com/kelseyhightower/envconfig"
	"go.uber.org/zap"
	"k8s.io/client-go/tools/cache"
	"knative.dev/pkg/configmap"
	"knative.dev/pkg/controller"
	"knative.dev/pkg/logging"

	"github.com/google/knative-gcp/pkg/apis/messaging/v1alpha1"
	"github.com/google/knative-gcp/pkg/reconciler"

	serviceinformer "knative.dev/serving/pkg/client/injection/informers/serving/v1/service"

	decoratorinformer "github.com/google/knative-gcp/pkg/client/injection/informers/messaging/v1alpha1/decorator"
)

const (
	// controllerAgentName is the string used by this controller to identify
	// itself when creating events.
	controllerAgentName = "cloud-run-events-decorator-controller"
)

type envConfig struct {
	// Decorator is the image used to run the decorator. Required.
	Decorator string `envconfig:"DECORATOR_IMAGE" required:"true"`
}

// NewController initializes the controller and is called by the generated code
// Registers event handlers to enqueue events
func NewController(
	ctx context.Context,
	cmw configmap.Watcher,
) *controller.Impl {
	decoratorInformer := decoratorinformer.Get(ctx)
	serviceinformer := serviceinformer.Get(ctx)

	logger := logging.FromContext(ctx).Named(controllerAgentName)

	var env envConfig
	if err := envconfig.Process("", &env); err != nil {
		logger.Fatal("Failed to process env var", zap.Error(err))
	}

	c := &Reconciler{
		Base:            reconciler.NewBase(ctx, controllerAgentName, cmw),
		decoratorLister: decoratorInformer.Lister(),
		serviceLister:   serviceinformer.Lister(),
		decoratorImage:  env.Decorator,
	}

	impl := controller.NewImpl(c, c.Logger, ReconcilerName)

	c.Logger.Info("Setting up event handlers")
	decoratorInformer.Informer().AddEventHandler(controller.HandleAll(impl.Enqueue))

	serviceinformer.Informer().AddEventHandler(cache.FilteringResourceEventHandler{
		FilterFunc: controller.Filter(v1alpha1.SchemeGroupVersion.WithKind("Decorator")),
		Handler:    controller.HandleAll(impl.EnqueueControllerOf),
	})

	c.uriResolver = resolver.NewURIResolver(ctx, impl.EnqueueKey)

	return impl
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/grafana/metrictank/clock"
	"github.com/grafana/metrictank/stacktest/graphite"
	"github.com/grafana/metrictank/stats"
	log "github.com/sirupsen/logrus"
	"math"
	"net/http"
	"strconv"
	"time"
)

var (
	httpError    = stats.NewCounter32("parrot.monitoring.error;error=http")
	decodeError  = stats.NewCounter32("parrot.monitoring.error;error=decode")
	invalidError = stats.NewCounter32("parrot.monitoring.error;error=invalid")
)

var metricsBySeries []partitionMetrics

type seriesStats struct {
	lastTs uint32
	//the partition currently being checked
	nans int32
	//the sum of abs(value - ts) across the time series
	deltaSum float64
	//the number of timestamps where value != ts
	numNonMatching int32
	//tracks the last seen non-NaN time stamp (useful for lag
	lastSeen uint32
}

type partitionMetrics struct {
	//number of missing values for each series
	nanCount *stats.Gauge32
	//time since the last value was recorded
	lag *stats.Gauge32
	//total amount of drift between expected value and actual values
	deltaSum *stats.Gauge32
	//total number of entries where drift occurred
	nonMatching *stats.Gauge32
}

func monitor() {
	initMetricsBySeries()
	for tick := range clock.AlignedTickLossless(queryInterval) {

		query := graphite.ExecuteRenderQuery(buildRequest(tick))
		if query.HTTPErr != nil {
			httpError.Inc()
			continue
		}
		if query.DecodeErr != nil {
			decodeError.Inc()
			continue
		}

		for _, s := range query.Decoded {
			processPartitionSeries(s)
		}
	}
}

func processPartitionSeries(s graphite.Series) {
	log.Infof("%d - %d", s.Datapoints[0].Ts, s.Datapoints[len(s.Datapoints)-1].Ts)
	partition, err := strconv.Atoi(s.Target)
	if err != nil {
		log.Debug("unable to parse partition", err)
		invalidError.Inc()
		return
	}
	serStats := seriesStats{}
	serStats.lastTs = s.Datapoints[len(s.Datapoints)-1].Ts

	for _, dp := range s.Datapoints {

		if math.IsNaN(dp.Val) {
			serStats.nans += 1
			continue
		}
		serStats.lastSeen = dp.Ts
		if diff := dp.Val - float64(dp.Ts); diff != 0 {
			log.Debugf("partition=%d dp.Val=%f dp.Ts=%d diff=%f", partition, dp.Val, dp.Ts, diff)
			serStats.deltaSum += diff
			serStats.numNonMatching += 1
		}
	}

	metrics := metricsBySeries[partition]
	metrics.nanCount.Set(int(serStats.nans))
	metrics.lag.Set(int(serStats.lastTs - serStats.lastSeen))
	metrics.deltaSum.Set(int(serStats.deltaSum))
	metrics.nonMatching.Set(int(serStats.numNonMatching))
}

func initMetricsBySeries() {
	for p := 0; p < int(partitionCount); p++ {
		metrics := partitionMetrics{
			nanCount:    stats.NewGauge32(fmt.Sprintf("parrot.monitoring.nancount;partition=%d", p)),
			lag:         stats.NewGauge32(fmt.Sprintf("parrot.monitoring.lag;partition=%d", p)),
			deltaSum:    stats.NewGauge32(fmt.Sprintf("parrot.monitoring.deltaSum;partition=%d", p)),
			nonMatching: stats.NewGauge32(fmt.Sprintf("parrot.monitoring.nonMatching;partition=%d", p)),
		}
		metricsBySeries = append(metricsBySeries, metrics)
	}
}

func buildRequest(now time.Time) *http.Request {
	req, _ := http.NewRequest("GET", fmt.Sprintf("%s/render", gatewayAddress), nil)
	q := req.URL.Query()
	q.Set("target", "aliasByNode(parrot.testdata.*.generated.*, 2)")
	q.Set("from", strconv.Itoa(int(now.Add(-1*lookbackPeriod).Unix()-1)))
	q.Set("until", strconv.Itoa(int(now.Unix())))
	q.Set("format", "json")
	q.Set("X-Org-Id", strconv.Itoa(orgId))
	req.URL.RawQuery = q.Encode()
	if len(gatewayKey) != 0 {
		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", gatewayKey))
	}
	return req
}
<commit_msg>Add additional validation for correct number of points that are correctly spaced etc<commit_after>package main

import (
	"fmt"
	"github.com/grafana/metrictank/clock"
	"github.com/grafana/metrictank/stacktest/graphite"
	"github.com/grafana/metrictank/stats"
	log "github.com/sirupsen/logrus"
	"math"
	"net/http"
	"strconv"
	"time"
)

var (
	httpError    = stats.NewCounter32("parrot.monitoring.error;error=http")
	decodeError  = stats.NewCounter32("parrot.monitoring.error;error=decode")
	invalidError = stats.NewCounter32("parrot.monitoring.error;error=invalid")
)

var metricsBySeries []partitionMetrics

type seriesStats struct {
	lastTs uint32
	//the partition currently being checked
	nans int32
	//the sum of abs(value - ts) across the time series
	deltaSum float64
	//the number of timestamps where value != ts
	numNonMatching int32
	//tracks the last seen non-NaN time stamp (useful for lag
	lastSeen uint32
	//the expected number of points were received
	correctNumPoints bool
	//the last ts matches `now`
	correctAlignment bool
	//all points are sorted and 1 period apart
	correctSpacing bool
}

type partitionMetrics struct {
	//number of missing values for each series
	nanCount *stats.Gauge32
	//time since the last value was recorded
	lag *stats.Gauge32
	//total amount of drift between expected value and actual values
	deltaSum *stats.Gauge32
	//total number of entries where drift occurred
	nonMatching *stats.Gauge32
}

func monitor() {
	initMetricsBySeries()
	for tick := range clock.AlignedTickLossless(queryInterval) {

		query := graphite.ExecuteRenderQuery(buildRequest(tick))
		if query.HTTPErr != nil {
			httpError.Inc()
			continue
		}
		if query.DecodeErr != nil {
			decodeError.Inc()
			continue
		}

		for _, s := range query.Decoded {
			processPartitionSeries(s, tick)
		}
	}
}

func processPartitionSeries(s graphite.Series, now time.Time) {
	partition, err := strconv.Atoi(s.Target)
	if err != nil {
		log.Debug("unable to parse partition", err)
		invalidError.Inc()
		return
	}
	if len(s.Datapoints) < 2 {
		log.Debugf("partition has invalid number of datapoints: %d", len(s.Datapoints))
		invalidError.Inc()
		return
	}

	serStats := seriesStats{}
	serStats.lastTs = s.Datapoints[len(s.Datapoints)-1].Ts
	serStats.correctAlignment = int64(serStats.lastTs) == now.Unix()
	serStats.correctNumPoints = len(s.Datapoints) == int(lookbackPeriod/testMetricsInterval)+1
	serStats.correctSpacing = checkSpacing(s.Datapoints)

	for _, dp := range s.Datapoints {
		if math.IsNaN(dp.Val) {
			serStats.nans += 1
			continue
		}
		serStats.lastSeen = dp.Ts
		if diff := dp.Val - float64(dp.Ts); diff != 0 {
			log.Debugf("partition=%d dp.Val=%f dp.Ts=%d diff=%f", partition, dp.Val, dp.Ts, diff)
			serStats.deltaSum += diff
			serStats.numNonMatching += 1
		}
	}

	metrics := metricsBySeries[partition]
	metrics.nanCount.Set(int(serStats.nans))
	metrics.lag.Set(int(serStats.lastTs - serStats.lastSeen))
	metrics.deltaSum.Set(int(serStats.deltaSum))
	metrics.nonMatching.Set(int(serStats.numNonMatching))
}

func checkSpacing(points []graphite.Point) bool {
	previous := points[0].Ts
	for i := 1; i < len(points); i++ {
		current := points[i].Ts
		if current-previous != uint32(testMetricsInterval.Seconds()) {
			return false
		}
		previous = current
	}
	return true
}

func initMetricsBySeries() {
	for p := 0; p < int(partitionCount); p++ {
		metrics := partitionMetrics{
			nanCount:    stats.NewGauge32(fmt.Sprintf("parrot.monitoring.nancount;partition=%d", p)),
			lag:         stats.NewGauge32(fmt.Sprintf("parrot.monitoring.lag;partition=%d", p)),
			deltaSum:    stats.NewGauge32(fmt.Sprintf("parrot.monitoring.deltaSum;partition=%d", p)),
			nonMatching: stats.NewGauge32(fmt.Sprintf("parrot.monitoring.nonMatching;partition=%d", p)),
		}
		metricsBySeries = append(metricsBySeries, metrics)
	}
}

func buildRequest(now time.Time) *http.Request {
	req, _ := http.NewRequest("GET", fmt.Sprintf("%s/render", gatewayAddress), nil)
	q := req.URL.Query()
	q.Set("target", "aliasByNode(parrot.testdata.*.generated.*, 2)")
	q.Set("from", strconv.Itoa(int(now.Add(-1*lookbackPeriod).Unix()-1)))
	q.Set("until", strconv.Itoa(int(now.Unix())))
	q.Set("format", "json")
	q.Set("X-Org-Id", strconv.Itoa(orgId))
	req.URL.RawQuery = q.Encode()
	if len(gatewayKey) != 0 {
		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", gatewayKey))
	}
	return req
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"database/sql"
	"flag"
	"fmt"
	"os"
	"time"

	"github.com/jmhodges/clock"
	akamaipb "github.com/letsencrypt/boulder/akamai/proto"
	capb "github.com/letsencrypt/boulder/ca/proto"
	"github.com/letsencrypt/boulder/cmd"
	"github.com/letsencrypt/boulder/core"
	"github.com/letsencrypt/boulder/db"
	"github.com/letsencrypt/boulder/features"
	bgrpc "github.com/letsencrypt/boulder/grpc"
	blog "github.com/letsencrypt/boulder/log"
	"github.com/letsencrypt/boulder/sa"
	"github.com/prometheus/client_golang/prometheus"
)

/*
 * ocspDB is an interface collecting the gorp.DbMap functions that the
 * various parts of OCSPUpdater rely on. Using this adapter shim allows tests to
 * swap out the dbMap implementation.
 */
type ocspDB interface {
	Select(i interface{}, query string, args ...interface{}) ([]interface{}, error)
	SelectOne(holder interface{}, query string, args ...interface{}) error
	Exec(query string, args ...interface{}) (sql.Result, error)
}

// OCSPUpdater contains the useful objects for the Updater
type OCSPUpdater struct {
	log blog.Logger
	clk clock.Clock

	dbMap ocspDB

	ogc capb.OCSPGeneratorClient

	tickWindow    time.Duration
	batchSize     int
	tickHistogram *prometheus.HistogramVec

	maxBackoff    time.Duration
	backoffFactor float64
	tickFailures  int

	// Used to calculate how far back stale OCSP responses should be looked for
	ocspMinTimeToExpiry time.Duration
	// Maximum number of individual OCSP updates to attempt in parallel. Making
	// these requests in parallel allows us to get higher total throughput.
	parallelGenerateOCSPRequests int

	purgerService akamaipb.AkamaiPurgerClient
	// issuer is used to generate OCSP request URLs to purge
	issuer *x509.Certificate

	genStoreHistogram prometheus.Histogram
	generatedCounter  *prometheus.CounterVec
	storedCounter     *prometheus.CounterVec
}

func newUpdater(
	stats prometheus.Registerer,
	clk clock.Clock,
	dbMap ocspDB,
	ogc capb.OCSPGeneratorClient,
	apc akamaipb.AkamaiPurgerClient,
	config OCSPUpdaterConfig,
	issuerPath string,
	log blog.Logger,
) (*OCSPUpdater, error) {
	if config.OldOCSPBatchSize == 0 {
		return nil, fmt.Errorf("Loop batch sizes must be non-zero")
	}
	if config.OldOCSPWindow.Duration == 0 {
		return nil, fmt.Errorf("Loop window sizes must be non-zero")
	}
	if config.ParallelGenerateOCSPRequests == 0 {
		// Default to 1
		config.ParallelGenerateOCSPRequests = 1
	}

	genStoreHistogram := prometheus.NewHistogram(prometheus.HistogramOpts{
		Name: "ocsp_updater_generate_and_store",
		Help: "A histogram of latencies of OCSP generation and storage latencies",
	})
	stats.MustRegister(genStoreHistogram)
	generatedCounter := prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "ocsp_updater_generated",
		Help: "A counter of OCSP response generation calls labelled by result",
	}, []string{"result"})
	stats.MustRegister(generatedCounter)
	storedCounter := prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "ocsp_updater_stored",
		Help: "A counter of OCSP response storage calls labelled by result",
	}, []string{"result"})
	stats.MustRegister(storedCounter)
	tickHistogram := prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Name: "ocsp_updater_ticks",
		Help: "A histogram of ocsp-updater tick latencies labelled by result and whether the tick was considered longer than expected",
	}, []string{"result", "long"})
	stats.MustRegister(tickHistogram)

	updater := OCSPUpdater{
		clk:                          clk,
		dbMap:                        dbMap,
		ogc:                          ogc,
		log:                          log,
		ocspMinTimeToExpiry:          config.OCSPMinTimeToExpiry.Duration,
		parallelGenerateOCSPRequests: config.ParallelGenerateOCSPRequests,
		purgerService:                apc,
		genStoreHistogram:            genStoreHistogram,
		generatedCounter:             generatedCounter,
		storedCounter:                storedCounter,
		tickHistogram:                tickHistogram,
		tickWindow:                   config.OldOCSPWindow.Duration,
		batchSize:                    config.OldOCSPBatchSize,
		maxBackoff:                   config.SignFailureBackoffMax.Duration,
		backoffFactor:                config.SignFailureBackoffFactor,
	}

	if updater.purgerService != nil {
		issuer, err := core.LoadCert(issuerPath)
		if err != nil {
			return nil, err
		}
		updater.issuer = issuer
	}

	return &updater, nil
}

func (updater *OCSPUpdater) findStaleOCSPResponses(oldestLastUpdatedTime time.Time, batchSize int) ([]core.CertificateStatus, error) {
	statuses, err := sa.SelectCertificateStatuses(
		updater.dbMap,
		`WHERE ocspLastUpdated < :lastUpdate
		 AND NOT isExpired
		 ORDER BY ocspLastUpdated ASC
		 LIMIT :limit`,
		map[string]interface{}{
			"lastUpdate": oldestLastUpdatedTime,
			"limit":      batchSize,
		},
	)
	if db.IsNoRows(err) {
		return statuses, nil
	}
	return statuses, err
}

func getCertDER(selector ocspDB, serial string) ([]byte, error) {
	cert, err := sa.SelectCertificate(selector, serial)
	if err != nil {
		if db.IsNoRows(err) {
			cert, err = sa.SelectPrecertificate(selector, serial)
			// If there was still a non-nil error return it. If we can't find
			// a precert row something is amiss, we have a certificateStatus row with
			// no matching certificate or precertificate.
			if err != nil {
				return nil, err
			}
		} else {
			return nil, err
		}
	}
	return cert.DER, nil
}

func (updater *OCSPUpdater) generateResponse(ctx context.Context, status core.CertificateStatus) (*core.CertificateStatus, error) {
	ocspReq := capb.GenerateOCSPRequest{
		Reason:    int32(status.RevokedReason),
		Status:    string(status.Status),
		RevokedAt: status.RevokedDate.UnixNano(),
	}
	if status.IssuerID != nil {
		ocspReq.Serial = status.Serial
		ocspReq.IssuerID = *status.IssuerID
	} else {
		certDER, err := getCertDER(updater.dbMap, status.Serial)
		if err != nil {
			return nil, err
		}
		ocspReq.CertDER = certDER
	}

	ocspResponse, err := updater.ogc.GenerateOCSP(ctx, &ocspReq)
	if err != nil {
		return nil, err
	}

	status.OCSPLastUpdated = updater.clk.Now()
	status.OCSPResponse = ocspResponse.Response

	return &status, nil
}

func (updater *OCSPUpdater) storeResponse(status *core.CertificateStatus) error {
	// Update the certificateStatus table with the new OCSP response, the status
	// WHERE is used make sure we don't overwrite a revoked response with a one
	// containing a 'good' status.
	_, err := updater.dbMap.Exec(
		`UPDATE certificateStatus
		 SET ocspResponse=?,ocspLastUpdated=?
		 WHERE serial=?
		 AND status=?`,
		status.OCSPResponse,
		status.OCSPLastUpdated,
		status.Serial,
		string(status.Status),
	)
	return err
}

// markExpired updates a given CertificateStatus to have `isExpired` set.
func (updater *OCSPUpdater) markExpired(status core.CertificateStatus) error {
	_, err := updater.dbMap.Exec(
		`UPDATE certificateStatus
 		SET isExpired = TRUE
 		WHERE serial = ?`,
		status.Serial,
	)
	return err
}

func (updater *OCSPUpdater) generateOCSPResponses(ctx context.Context, statuses []core.CertificateStatus) error {
	// Use the semaphore pattern from
	// https://github.com/golang/go/wiki/BoundingResourceUse to send a number of
	// GenerateOCSP / storeResponse requests in parallel, while limiting the total number of
	// outstanding requests. The number of outstanding requests equals the
	// capacity of the channel.
	sem := make(chan int, updater.parallelGenerateOCSPRequests)
	wait := func() {
		sem <- 1 // Block until there's capacity.
	}
	done := func(start time.Time) {
		<-sem // Indicate there's more capacity.
		updater.genStoreHistogram.Observe(time.Since(start).Seconds())
	}

	work := func(status core.CertificateStatus) {
		defer done(updater.clk.Now())
		meta, err := updater.generateResponse(ctx, status)
		if err != nil {
			updater.log.AuditErrf("Failed to generate OCSP response: %s", err)
			updater.generatedCounter.WithLabelValues("failed").Inc()
			return
		}
		updater.generatedCounter.WithLabelValues("success").Inc()
		err = updater.storeResponse(meta)
		if err != nil {
			updater.log.AuditErrf("Failed to store OCSP response: %s", err)
			updater.storedCounter.WithLabelValues("failed").Inc()
			return
		}
		updater.storedCounter.WithLabelValues("success").Inc()
	}

	for _, status := range statuses {
		wait()
		go work(status)
	}
	// Block until the channel reaches its full capacity again, indicating each
	// goroutine has completed.
	for i := 0; i < updater.parallelGenerateOCSPRequests; i++ {
		wait()
	}
	return nil
}

// updateOCSPResponses looks for certificates with stale OCSP responses and
// generates/stores new ones
func (updater *OCSPUpdater) updateOCSPResponses(ctx context.Context, batchSize int) error {
	tickStart := updater.clk.Now()
	statuses, err := updater.findStaleOCSPResponses(tickStart.Add(-updater.ocspMinTimeToExpiry), batchSize)
	if err != nil {
		updater.log.AuditErrf("Failed to find stale OCSP responses: %s", err)
		return err
	}

	for _, s := range statuses {
		if !s.IsExpired && tickStart.After(s.NotAfter) {
			err := updater.markExpired(s)
			if err != nil {
				return err
			}
		}
	}

	return updater.generateOCSPResponses(ctx, statuses)
}

type config struct {
	OCSPUpdater OCSPUpdaterConfig

	Syslog cmd.SyslogConfig

	Common struct {
		IssuerCert string
	}
}

// OCSPUpdaterConfig provides the various window tick times and batch sizes needed
// for the OCSP (and SCT) updater
type OCSPUpdaterConfig struct {
	cmd.ServiceConfig
	cmd.DBConfig

	OldOCSPWindow    cmd.ConfigDuration
	OldOCSPBatchSize int

	OCSPMinTimeToExpiry          cmd.ConfigDuration
	ParallelGenerateOCSPRequests int

	AkamaiBaseURL           string
	AkamaiClientToken       string
	AkamaiClientSecret      string
	AkamaiAccessToken       string
	AkamaiV3Network         string
	AkamaiPurgeRetries      int
	AkamaiPurgeRetryBackoff cmd.ConfigDuration

	SignFailureBackoffFactor float64
	SignFailureBackoffMax    cmd.ConfigDuration

	SAService            *cmd.GRPCClientConfig
	OCSPGeneratorService *cmd.GRPCClientConfig
	AkamaiPurgerService  *cmd.GRPCClientConfig

	Features map[string]bool
}

func setupClients(c OCSPUpdaterConfig, stats prometheus.Registerer, clk clock.Clock) (
	capb.OCSPGeneratorClient,
	akamaipb.AkamaiPurgerClient,
) {
	var tls *tls.Config
	var err error
	if c.TLS.CertFile != nil {
		tls, err = c.TLS.Load()
		cmd.FailOnError(err, "TLS config")
	}
	clientMetrics := bgrpc.NewClientMetrics(stats)
	caConn, err := bgrpc.ClientSetup(c.OCSPGeneratorService, tls, clientMetrics, clk)
	cmd.FailOnError(err, "Failed to load credentials and create gRPC connection to CA")
	ogc := bgrpc.NewOCSPGeneratorClient(capb.NewOCSPGeneratorClient(caConn))

	var apc akamaipb.AkamaiPurgerClient
	if c.AkamaiPurgerService != nil {
		apcConn, err := bgrpc.ClientSetup(c.AkamaiPurgerService, tls, clientMetrics, clk)
		cmd.FailOnError(err, "Failed to load credentials and create gRPC connection to Akamai Purger service")
		apc = akamaipb.NewAkamaiPurgerClient(apcConn)
	}

	return ogc, apc
}

func (updater *OCSPUpdater) tick() {
	start := updater.clk.Now()
	err := updater.updateOCSPResponses(context.Background(), updater.batchSize)
	end := updater.clk.Now()
	took := end.Sub(start)
	long, state := "false", "success"
	if took > updater.tickWindow {
		long = "true"
	}
	sleepDur := start.Add(updater.tickWindow).Sub(end)
	if err != nil {
		state = "failed"
		updater.tickFailures++
		sleepDur = core.RetryBackoff(
			updater.tickFailures,
			updater.tickWindow,
			updater.maxBackoff,
			updater.backoffFactor,
		)
	} else if updater.tickFailures > 0 {
		updater.tickFailures = 0
	}
	updater.tickHistogram.WithLabelValues(state, long).Observe(took.Seconds())
	updater.clk.Sleep(sleepDur)
}

func main() {
	configFile := flag.String("config", "", "File path to the configuration file for this service")
	flag.Parse()
	if *configFile == "" {
		flag.Usage()
		os.Exit(1)
	}

	var c config
	err := cmd.ReadConfigFile(*configFile, &c)
	cmd.FailOnError(err, "Reading JSON config file into config structure")

	conf := c.OCSPUpdater
	err = features.Set(conf.Features)
	cmd.FailOnError(err, "Failed to set feature flags")

	stats, logger := cmd.StatsAndLogging(c.Syslog, conf.DebugAddr)
	defer logger.AuditPanic()
	logger.Info(cmd.VersionString())

	// Configure DB
	dbURL, err := conf.DBConfig.URL()
	cmd.FailOnError(err, "Couldn't load DB URL")
	dbMap, err := sa.NewDbMap(dbURL, conf.DBConfig.MaxDBConns)
	cmd.FailOnError(err, "Could not connect to database")

	// Collect and periodically report DB metrics using the DBMap and prometheus stats.
	sa.InitDBMetrics(dbMap, stats)

	clk := cmd.Clock()
	ogc, apc := setupClients(conf, stats, clk)

	updater, err := newUpdater(
		stats,
		clk,
		dbMap,
		ogc,
		apc,
		// Necessary evil for now
		conf,
		c.Common.IssuerCert,
		logger,
	)
	cmd.FailOnError(err, "Failed to create updater")

	go cmd.CatchSignals(logger, nil)

	for {
		updater.tick()
	}
}
<commit_msg>ocsp-updater: use certDER codepath when IssuerID is 0 (#5100)<commit_after>package main

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"database/sql"
	"flag"
	"fmt"
	"os"
	"time"

	"github.com/jmhodges/clock"
	akamaipb "github.com/letsencrypt/boulder/akamai/proto"
	capb "github.com/letsencrypt/boulder/ca/proto"
	"github.com/letsencrypt/boulder/cmd"
	"github.com/letsencrypt/boulder/core"
	"github.com/letsencrypt/boulder/db"
	"github.com/letsencrypt/boulder/features"
	bgrpc "github.com/letsencrypt/boulder/grpc"
	blog "github.com/letsencrypt/boulder/log"
	"github.com/letsencrypt/boulder/sa"
	"github.com/prometheus/client_golang/prometheus"
)

/*
 * ocspDB is an interface collecting the gorp.DbMap functions that the
 * various parts of OCSPUpdater rely on. Using this adapter shim allows tests to
 * swap out the dbMap implementation.
 */
type ocspDB interface {
	Select(i interface{}, query string, args ...interface{}) ([]interface{}, error)
	SelectOne(holder interface{}, query string, args ...interface{}) error
	Exec(query string, args ...interface{}) (sql.Result, error)
}

// OCSPUpdater contains the useful objects for the Updater
type OCSPUpdater struct {
	log blog.Logger
	clk clock.Clock

	dbMap ocspDB

	ogc capb.OCSPGeneratorClient

	tickWindow    time.Duration
	batchSize     int
	tickHistogram *prometheus.HistogramVec

	maxBackoff    time.Duration
	backoffFactor float64
	tickFailures  int

	// Used to calculate how far back stale OCSP responses should be looked for
	ocspMinTimeToExpiry time.Duration
	// Maximum number of individual OCSP updates to attempt in parallel. Making
	// these requests in parallel allows us to get higher total throughput.
	parallelGenerateOCSPRequests int

	purgerService akamaipb.AkamaiPurgerClient
	// issuer is used to generate OCSP request URLs to purge
	issuer *x509.Certificate

	genStoreHistogram prometheus.Histogram
	generatedCounter  *prometheus.CounterVec
	storedCounter     *prometheus.CounterVec
}

func newUpdater(
	stats prometheus.Registerer,
	clk clock.Clock,
	dbMap ocspDB,
	ogc capb.OCSPGeneratorClient,
	apc akamaipb.AkamaiPurgerClient,
	config OCSPUpdaterConfig,
	issuerPath string,
	log blog.Logger,
) (*OCSPUpdater, error) {
	if config.OldOCSPBatchSize == 0 {
		return nil, fmt.Errorf("Loop batch sizes must be non-zero")
	}
	if config.OldOCSPWindow.Duration == 0 {
		return nil, fmt.Errorf("Loop window sizes must be non-zero")
	}
	if config.ParallelGenerateOCSPRequests == 0 {
		// Default to 1
		config.ParallelGenerateOCSPRequests = 1
	}

	genStoreHistogram := prometheus.NewHistogram(prometheus.HistogramOpts{
		Name: "ocsp_updater_generate_and_store",
		Help: "A histogram of latencies of OCSP generation and storage latencies",
	})
	stats.MustRegister(genStoreHistogram)
	generatedCounter := prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "ocsp_updater_generated",
		Help: "A counter of OCSP response generation calls labelled by result",
	}, []string{"result"})
	stats.MustRegister(generatedCounter)
	storedCounter := prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "ocsp_updater_stored",
		Help: "A counter of OCSP response storage calls labelled by result",
	}, []string{"result"})
	stats.MustRegister(storedCounter)
	tickHistogram := prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Name: "ocsp_updater_ticks",
		Help: "A histogram of ocsp-updater tick latencies labelled by result and whether the tick was considered longer than expected",
	}, []string{"result", "long"})
	stats.MustRegister(tickHistogram)

	updater := OCSPUpdater{
		clk:                          clk,
		dbMap:                        dbMap,
		ogc:                          ogc,
		log:                          log,
		ocspMinTimeToExpiry:          config.OCSPMinTimeToExpiry.Duration,
		parallelGenerateOCSPRequests: config.ParallelGenerateOCSPRequests,
		purgerService:                apc,
		genStoreHistogram:            genStoreHistogram,
		generatedCounter:             generatedCounter,
		storedCounter:                storedCounter,
		tickHistogram:                tickHistogram,
		tickWindow:                   config.OldOCSPWindow.Duration,
		batchSize:                    config.OldOCSPBatchSize,
		maxBackoff:                   config.SignFailureBackoffMax.Duration,
		backoffFactor:                config.SignFailureBackoffFactor,
	}

	if updater.purgerService != nil {
		issuer, err := core.LoadCert(issuerPath)
		if err != nil {
			return nil, err
		}
		updater.issuer = issuer
	}

	return &updater, nil
}

func (updater *OCSPUpdater) findStaleOCSPResponses(oldestLastUpdatedTime time.Time, batchSize int) ([]core.CertificateStatus, error) {
	statuses, err := sa.SelectCertificateStatuses(
		updater.dbMap,
		`WHERE ocspLastUpdated < :lastUpdate
		 AND NOT isExpired
		 ORDER BY ocspLastUpdated ASC
		 LIMIT :limit`,
		map[string]interface{}{
			"lastUpdate": oldestLastUpdatedTime,
			"limit":      batchSize,
		},
	)
	if db.IsNoRows(err) {
		return statuses, nil
	}
	return statuses, err
}

func getCertDER(selector ocspDB, serial string) ([]byte, error) {
	cert, err := sa.SelectCertificate(selector, serial)
	if err != nil {
		if db.IsNoRows(err) {
			cert, err = sa.SelectPrecertificate(selector, serial)
			// If there was still a non-nil error return it. If we can't find
			// a precert row something is amiss, we have a certificateStatus row with
			// no matching certificate or precertificate.
			if err != nil {
				return nil, err
			}
		} else {
			return nil, err
		}
	}
	return cert.DER, nil
}

func (updater *OCSPUpdater) generateResponse(ctx context.Context, status core.CertificateStatus) (*core.CertificateStatus, error) {
	ocspReq := capb.GenerateOCSPRequest{
		Reason:    int32(status.RevokedReason),
		Status:    string(status.Status),
		RevokedAt: status.RevokedDate.UnixNano(),
	}
	if status.IssuerID != nil && *status.IssuerID != 0 {
		ocspReq.Serial = status.Serial
		ocspReq.IssuerID = *status.IssuerID
	} else {
		certDER, err := getCertDER(updater.dbMap, status.Serial)
		if err != nil {
			return nil, err
		}
		ocspReq.CertDER = certDER
	}

	ocspResponse, err := updater.ogc.GenerateOCSP(ctx, &ocspReq)
	if err != nil {
		return nil, err
	}

	status.OCSPLastUpdated = updater.clk.Now()
	status.OCSPResponse = ocspResponse.Response

	return &status, nil
}

func (updater *OCSPUpdater) storeResponse(status *core.CertificateStatus) error {
	// Update the certificateStatus table with the new OCSP response, the status
	// WHERE is used make sure we don't overwrite a revoked response with a one
	// containing a 'good' status.
	_, err := updater.dbMap.Exec(
		`UPDATE certificateStatus
		 SET ocspResponse=?,ocspLastUpdated=?
		 WHERE serial=?
		 AND status=?`,
		status.OCSPResponse,
		status.OCSPLastUpdated,
		status.Serial,
		string(status.Status),
	)
	return err
}

// markExpired updates a given CertificateStatus to have `isExpired` set.
func (updater *OCSPUpdater) markExpired(status core.CertificateStatus) error {
	_, err := updater.dbMap.Exec(
		`UPDATE certificateStatus
 		SET isExpired = TRUE
 		WHERE serial = ?`,
		status.Serial,
	)
	return err
}

func (updater *OCSPUpdater) generateOCSPResponses(ctx context.Context, statuses []core.CertificateStatus) error {
	// Use the semaphore pattern from
	// https://github.com/golang/go/wiki/BoundingResourceUse to send a number of
	// GenerateOCSP / storeResponse requests in parallel, while limiting the total number of
	// outstanding requests. The number of outstanding requests equals the
	// capacity of the channel.
	sem := make(chan int, updater.parallelGenerateOCSPRequests)
	wait := func() {
		sem <- 1 // Block until there's capacity.
	}
	done := func(start time.Time) {
		<-sem // Indicate there's more capacity.
		updater.genStoreHistogram.Observe(time.Since(start).Seconds())
	}

	work := func(status core.CertificateStatus) {
		defer done(updater.clk.Now())
		meta, err := updater.generateResponse(ctx, status)
		if err != nil {
			updater.log.AuditErrf("Failed to generate OCSP response: %s", err)
			updater.generatedCounter.WithLabelValues("failed").Inc()
			return
		}
		updater.generatedCounter.WithLabelValues("success").Inc()
		err = updater.storeResponse(meta)
		if err != nil {
			updater.log.AuditErrf("Failed to store OCSP response: %s", err)
			updater.storedCounter.WithLabelValues("failed").Inc()
			return
		}
		updater.storedCounter.WithLabelValues("success").Inc()
	}

	for _, status := range statuses {
		wait()
		go work(status)
	}
	// Block until the channel reaches its full capacity again, indicating each
	// goroutine has completed.
	for i := 0; i < updater.parallelGenerateOCSPRequests; i++ {
		wait()
	}
	return nil
}

// updateOCSPResponses looks for certificates with stale OCSP responses and
// generates/stores new ones
func (updater *OCSPUpdater) updateOCSPResponses(ctx context.Context, batchSize int) error {
	tickStart := updater.clk.Now()
	statuses, err := updater.findStaleOCSPResponses(tickStart.Add(-updater.ocspMinTimeToExpiry), batchSize)
	if err != nil {
		updater.log.AuditErrf("Failed to find stale OCSP responses: %s", err)
		return err
	}

	for _, s := range statuses {
		if !s.IsExpired && tickStart.After(s.NotAfter) {
			err := updater.markExpired(s)
			if err != nil {
				return err
			}
		}
	}

	return updater.generateOCSPResponses(ctx, statuses)
}

type config struct {
	OCSPUpdater OCSPUpdaterConfig

	Syslog cmd.SyslogConfig

	Common struct {
		IssuerCert string
	}
}

// OCSPUpdaterConfig provides the various window tick times and batch sizes needed
// for the OCSP (and SCT) updater
type OCSPUpdaterConfig struct {
	cmd.ServiceConfig
	cmd.DBConfig

	OldOCSPWindow    cmd.ConfigDuration
	OldOCSPBatchSize int

	OCSPMinTimeToExpiry          cmd.ConfigDuration
	ParallelGenerateOCSPRequests int

	AkamaiBaseURL           string
	AkamaiClientToken       string
	AkamaiClientSecret      string
	AkamaiAccessToken       string
	AkamaiV3Network         string
	AkamaiPurgeRetries      int
	AkamaiPurgeRetryBackoff cmd.ConfigDuration

	SignFailureBackoffFactor float64
	SignFailureBackoffMax    cmd.ConfigDuration

	SAService            *cmd.GRPCClientConfig
	OCSPGeneratorService *cmd.GRPCClientConfig
	AkamaiPurgerService  *cmd.GRPCClientConfig

	Features map[string]bool
}

func setupClients(c OCSPUpdaterConfig, stats prometheus.Registerer, clk clock.Clock) (
	capb.OCSPGeneratorClient,
	akamaipb.AkamaiPurgerClient,
) {
	var tls *tls.Config
	var err error
	if c.TLS.CertFile != nil {
		tls, err = c.TLS.Load()
		cmd.FailOnError(err, "TLS config")
	}
	clientMetrics := bgrpc.NewClientMetrics(stats)
	caConn, err := bgrpc.ClientSetup(c.OCSPGeneratorService, tls, clientMetrics, clk)
	cmd.FailOnError(err, "Failed to load credentials and create gRPC connection to CA")
	ogc := bgrpc.NewOCSPGeneratorClient(capb.NewOCSPGeneratorClient(caConn))

	var apc akamaipb.AkamaiPurgerClient
	if c.AkamaiPurgerService != nil {
		apcConn, err := bgrpc.ClientSetup(c.AkamaiPurgerService, tls, clientMetrics, clk)
		cmd.FailOnError(err, "Failed to load credentials and create gRPC connection to Akamai Purger service")
		apc = akamaipb.NewAkamaiPurgerClient(apcConn)
	}

	return ogc, apc
}

func (updater *OCSPUpdater) tick() {
	start := updater.clk.Now()
	err := updater.updateOCSPResponses(context.Background(), updater.batchSize)
	end := updater.clk.Now()
	took := end.Sub(start)
	long, state := "false", "success"
	if took > updater.tickWindow {
		long = "true"
	}
	sleepDur := start.Add(updater.tickWindow).Sub(end)
	if err != nil {
		state = "failed"
		updater.tickFailures++
		sleepDur = core.RetryBackoff(
			updater.tickFailures,
			updater.tickWindow,
			updater.maxBackoff,
			updater.backoffFactor,
		)
	} else if updater.tickFailures > 0 {
		updater.tickFailures = 0
	}
	updater.tickHistogram.WithLabelValues(state, long).Observe(took.Seconds())
	updater.clk.Sleep(sleepDur)
}

func main() {
	configFile := flag.String("config", "", "File path to the configuration file for this service")
	flag.Parse()
	if *configFile == "" {
		flag.Usage()
		os.Exit(1)
	}

	var c config
	err := cmd.ReadConfigFile(*configFile, &c)
	cmd.FailOnError(err, "Reading JSON config file into config structure")

	conf := c.OCSPUpdater
	err = features.Set(conf.Features)
	cmd.FailOnError(err, "Failed to set feature flags")

	stats, logger := cmd.StatsAndLogging(c.Syslog, conf.DebugAddr)
	defer logger.AuditPanic()
	logger.Info(cmd.VersionString())

	// Configure DB
	dbURL, err := conf.DBConfig.URL()
	cmd.FailOnError(err, "Couldn't load DB URL")
	dbMap, err := sa.NewDbMap(dbURL, conf.DBConfig.MaxDBConns)
	cmd.FailOnError(err, "Could not connect to database")

	// Collect and periodically report DB metrics using the DBMap and prometheus stats.
	sa.InitDBMetrics(dbMap, stats)

	clk := cmd.Clock()
	ogc, apc := setupClients(conf, stats, clk)

	updater, err := newUpdater(
		stats,
		clk,
		dbMap,
		ogc,
		apc,
		// Necessary evil for now
		conf,
		c.Common.IssuerCert,
		logger,
	)
	cmd.FailOnError(err, "Failed to create updater")

	go cmd.CatchSignals(logger, nil)

	for {
		updater.tick()
	}
}
<|endoftext|>